Ann_Agreement_Round2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## Multi-labeled"
      ],
      "metadata": {
        "id": "V__PlcKKLRGd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "file_path1 = 'Kian.jsonl'\n",
        "file_path2 = 'Annina.jsonl'"
      ],
      "metadata": {
        "id": "hGynXGYADPuA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.metrics import cohen_kappa_score\n",
        "from collections import defaultdict\n",
        "from itertools import combinations\n",
        "\n",
        "# Define separate label sets for each task\n",
        "label_mapping_speaker = ['Speaker 1', 'Speaker 2', 'Interviewee', 'Interviewer', 'Instructor']\n",
        "label_mapping_emotion = ['Joy', 'Sadness', 'Anger', 'Fear', 'Surprise', 'Disgust', 'Trust', 'Anticipation']\n",
        "\n",
        "# Group IDs (modify as needed)\n",
        "group1_ids = {1, 2, 3, 4}\n",
        "group2_ids = {9, 11, 15}\n",
        "group3_ids = {5, 6, 7, 8, 10, 12, 13, 14, 16}\n",
        "group_definitions = {'Group 1': group1_ids, 'Group 2': group2_ids, 'Group 3': group3_ids}\n",
        "\n",
        "# Function to load JSONL data\n",
        "def load_jsonl(file_path):\n",
        "    data = []\n",
        "    with open(file_path, 'r') as f:\n",
        "        for line in f:\n",
        "            data.append(json.loads(line.strip()))\n",
        "    return data\n",
        "\n",
        "# Function to aggregate annotations for each group\n",
        "def aggregate_annotations_multilabel(spans, group_ids):\n",
        "    aggregated_text = \"\"\n",
        "    aggregated_annotations = []\n",
        "    offset = 0\n",
        "\n",
        "    for doc in spans:\n",
        "        if doc['id'] in group_ids:\n",
        "            text = doc['text']\n",
        "            annotations = [(start + offset, end + offset, label) for start, end, label in doc['annotations']]\n",
        "            aggregated_text += text\n",
        "            aggregated_annotations.extend(annotations)\n",
        "            offset += len(text)  # Adjust offset for next document in group\n",
        "\n",
        "    return aggregated_text, aggregated_annotations\n",
        "\n",
        "# Create one-hot encoding for multi-label annotations\n",
        "def one_hot_encode_annotations(annotations, labels):\n",
        "    encoding = []\n",
        "    label_to_idx = {label: i for i, label in enumerate(labels)}\n",
        "    for label_set in annotations:\n",
        "        vec = [0] * len(labels)\n",
        "        for label in label_set:\n",
        "            if label in label_to_idx:\n",
        "                vec[label_to_idx[label]] = 1\n",
        "        encoding.append(vec)\n",
        "    return encoding\n",
        "\n",
        "# Jaccard similarity calculation\n",
        "def jaccard_similarity_for_labels(annotator1, annotator2):\n",
        "    \"\"\"Calculate Jaccard Similarity for overlapping labels.\"\"\"\n",
        "    similarities = []\n",
        "    for labels1, labels2 in zip(annotator1, annotator2):\n",
        "        if labels1 or labels2:  # Only calculate for labeled positions\n",
        "            intersection = len(set(labels1) & set(labels2))\n",
        "            union = len(set(labels1) | set(labels2))\n",
        "            similarity = intersection / union if union > 0 else 0\n",
        "            similarities.append(similarity)\n",
        "    return np.mean(similarities)\n",
        "\n",
        "# Pad vectors to ensure consistent length for comparison\n",
        "def pad_to_match_length(vec1, vec2, pad_value=0):\n",
        "    \"\"\"Pad two lists to match the length of the longer list.\"\"\"\n",
        "    max_len = max(len(vec1), len(vec2))\n",
        "    return vec1 + [pad_value] * (max_len - len(vec1)), vec2 + [pad_value] * (max_len - len(vec2))\n",
        "\n",
        "# Calculate multi-label agreement for grouped data for each task separately\n",
        "def calculate_multi_label_agreement_for_tasks(spans_annotator1, spans_annotator2, labels_speaker, labels_emotion, group_definitions):\n",
        "    results = []\n",
        "    for group_name, group_ids in group_definitions.items():\n",
        "        # Aggregate annotations for each group\n",
        "        text_1, annotations_1 = aggregate_annotations_multilabel(spans_annotator1, group_ids)\n",
        "        text_2, annotations_2 = aggregate_annotations_multilabel(spans_annotator2, group_ids)\n",
        "\n",
        "        # Convert aggregated annotations to list of labels per position for each annotator\n",
        "        annotator1_labels = [[] for _ in range(len(text_1))]\n",
        "        annotator2_labels = [[] for _ in range(len(text_2))]\n",
        "\n",
        "        for start, end, label in annotations_1:\n",
        "            for i in range(start, end):\n",
        "                annotator1_labels[i].append(label)\n",
        "\n",
        "        for start, end, label in annotations_2:\n",
        "            for i in range(start, end):\n",
        "                annotator2_labels[i].append(label)\n",
        "\n",
        "        # Speaker Identification Task\n",
        "        speaker_jaccard = jaccard_similarity_for_labels(\n",
        "            [[label for label in labels if label in labels_speaker] for labels in annotator1_labels],\n",
        "            [[label for label in labels if label in labels_speaker] for labels in annotator2_labels]\n",
        "        )\n",
        "\n",
        "        speaker_encoded1 = one_hot_encode_annotations(\n",
        "            [[label for label in labels if label in labels_speaker] for labels in annotator1_labels],\n",
        "            labels_speaker\n",
        "        )\n",
        "        speaker_encoded2 = one_hot_encode_annotations(\n",
        "            [[label for label in labels if label in labels_speaker] for labels in annotator2_labels],\n",
        "            labels_speaker\n",
        "        )\n",
        "\n",
        "        speaker_kappas = []\n",
        "        for label_idx in range(len(labels_speaker)):\n",
        "            label_vec1 = [pos[label_idx] for pos in speaker_encoded1]\n",
        "            label_vec2 = [pos[label_idx] for pos in speaker_encoded2]\n",
        "            padded_vec1, padded_vec2 = pad_to_match_length(label_vec1, label_vec2)\n",
        "            if any(padded_vec1) or any(padded_vec2):\n",
        "                speaker_kappas.append(cohen_kappa_score(padded_vec1, padded_vec2))\n",
        "        speaker_kappa = np.mean(speaker_kappas) if speaker_kappas else None\n",
        "\n",
        "        # Emotion Detection Task\n",
        "        emotion_jaccard = jaccard_similarity_for_labels(\n",
        "            [[label for label in labels if label in labels_emotion] for labels in annotator1_labels],\n",
        "            [[label for label in labels if label in labels_emotion] for labels in annotator2_labels]\n",
        "        )\n",
        "\n",
        "        emotion_encoded1 = one_hot_encode_annotations(\n",
        "            [[label for label in labels if label in labels_emotion] for labels in annotator1_labels],\n",
        "            labels_emotion\n",
        "        )\n",
        "        emotion_encoded2 = one_hot_encode_annotations(\n",
        "            [[label for label in labels if label in labels_emotion] for labels in annotator2_labels],\n",
        "            labels_emotion\n",
        "        )\n",
        "\n",
        "        emotion_kappas = []\n",
        "        for label_idx in range(len(labels_emotion)):\n",
        "            label_vec1 = [pos[label_idx] for pos in emotion_encoded1]\n",
        "            label_vec2 = [pos[label_idx] for pos in emotion_encoded2]\n",
        "            padded_vec1, padded_vec2 = pad_to_match_length(label_vec1, label_vec2)\n",
        "            if any(padded_vec1) or any(padded_vec2):\n",
        "                emotion_kappas.append(cohen_kappa_score(padded_vec1, padded_vec2))\n",
        "        emotion_kappa = np.mean(emotion_kappas) if emotion_kappas else None\n",
        "\n",
        "        # Append results for this group for each task\n",
        "        results.append({\n",
        "            'group': group_name,\n",
        "            'speaker_jaccard': speaker_jaccard,\n",
        "            'speaker_kappa': speaker_kappa,\n",
        "            'emotion_jaccard': emotion_jaccard,\n",
        "            'emotion_kappa': emotion_kappa\n",
        "        })\n",
        "\n",
        "    return pd.DataFrame(results)\n",
        "\n",
        "# Load your files and run\n",
        "data_annotator1 = load_jsonl(file_path1)\n",
        "data_annotator2 = load_jsonl(file_path2)\n",
        "\n",
        "# Prepare spans for both annotators\n",
        "def get_spans_for_jaccard(data):\n",
        "    spans = []\n",
        "    for doc in data:\n",
        "        text = doc['text']\n",
        "        doc_annotations = [(start, end, label) for start, end, label in doc['label']]\n",
        "        spans.append({'id': doc['id'], 'text': text, 'annotations': doc_annotations})\n",
        "    return spans\n",
        "\n",
        "spans1 = get_spans_for_jaccard(data_annotator1)\n",
        "spans2 = get_spans_for_jaccard(data_annotator2)\n",
        "\n",
        "# Run agreement calculation for each group and task\n",
        "agreement_results_grouped_tasks = calculate_multi_label_agreement_for_tasks(\n",
        "    spans1, spans2, label_mapping_speaker, label_mapping_emotion, group_definitions\n",
        ")\n",
        "\n",
        "# Display the results\n",
        "print(file_path1,file_path2)\n",
        "print(agreement_results_grouped_tasks)\n"
      ],
      "metadata": {
        "id": "5Q5uvBG_LUm_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dfc10db7-af5f-487b-dc25-2aab705100b0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Lea.jsonl Kian.jsonl\n",
            "     group  speaker_jaccard  speaker_kappa  emotion_jaccard  emotion_kappa\n",
            "0  Group 1         0.918472       0.853876         0.089169       0.054961\n",
            "1  Group 2         0.462954       0.424625         0.082166       0.076222\n",
            "2  Group 3         0.000000       0.000000         0.176105       0.180288\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Labeled Only"
      ],
      "metadata": {
        "id": "cS53a_YT7G_d"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from collections import defaultdict\n",
        "from itertools import combinations\n",
        "\n",
        "# Define separate label sets for each task\n",
        "label_mapping_speaker = ['Speaker 1', 'Speaker 2', 'Interviewee', 'Interviewer', 'Instructor']\n",
        "label_mapping_emotion = ['Joy', 'Sadness', 'Anger', 'Fear', 'Surprise', 'Disgust', 'Trust', 'Anticipation']\n",
        "\n",
        "# Group IDs (modify as needed)\n",
        "group1_ids = {1, 2, 3, 4}\n",
        "group2_ids = {9, 11, 15}\n",
        "group3_ids = {5, 6, 7, 8, 10, 12, 13, 14, 16}\n",
        "group_definitions = {'Group 1': group1_ids, 'Group 2': group2_ids, 'Group 3': group3_ids}\n",
        "\n",
        "# Function to load JSONL data\n",
        "def load_jsonl(file_path):\n",
        "    data = []\n",
        "    with open(file_path, 'r') as f:\n",
        "        for line in f:\n",
        "            data.append(json.loads(line.strip()))\n",
        "    return data\n",
        "\n",
        "# Function to aggregate annotations for each group\n",
        "def aggregate_annotations_multilabel(spans, group_ids):\n",
        "    aggregated_text = \"\"\n",
        "    aggregated_annotations = []\n",
        "    offset = 0\n",
        "\n",
        "    for doc in spans:\n",
        "        if doc['id'] in group_ids:\n",
        "            text = doc['text']\n",
        "            annotations = [(start + offset, end + offset, label) for start, end, label in doc['annotations']]\n",
        "            aggregated_text += text\n",
        "            aggregated_annotations.extend(annotations)\n",
        "            offset += len(text)  # Adjust offset for next document in group\n",
        "\n",
        "    return aggregated_text, aggregated_annotations\n",
        "\n",
        "# Create one-hot encoding for multi-label annotations\n",
        "def one_hot_encode_annotations(annotations, labels):\n",
        "    encoding = []\n",
        "    label_to_idx = {label: i for i, label in enumerate(labels)}\n",
        "    for label_set in annotations:\n",
        "        vec = [0] * len(labels)\n",
        "        for label in label_set:\n",
        "            if label in label_to_idx:\n",
        "                vec[label_to_idx[label]] = 1\n",
        "        encoding.append(vec)\n",
        "    return encoding\n",
        "\n",
        "# Jaccard similarity calculation\n",
        "def jaccard_similarity_for_labels(annotator1, annotator2):\n",
        "    \"\"\"Calculate Jaccard Similarity for overlapping labels.\"\"\"\n",
        "    similarities = []\n",
        "    for labels1, labels2 in zip(annotator1, annotator2):\n",
        "        if labels1 and labels2:  # Only calculate for labeled positions\n",
        "            intersection = len(set(labels1) & set(labels2))\n",
        "            union = len(set(labels1) | set(labels2))\n",
        "            similarity = intersection / union if union > 0 else 0\n",
        "            similarities.append(similarity)\n",
        "    return np.mean(similarities)\n",
        "\n",
        "# Calculate multi-label agreement for grouped data for each task separately\n",
        "def calculate_multi_label_agreement_for_tasks(spans_annotator1, spans_annotator2, labels_speaker, labels_emotion, group_definitions):\n",
        "    results = []\n",
        "    for group_name, group_ids in group_definitions.items():\n",
        "        # Aggregate annotations for each group\n",
        "        text_1, annotations_1 = aggregate_annotations_multilabel(spans_annotator1, group_ids)\n",
        "        text_2, annotations_2 = aggregate_annotations_multilabel(spans_annotator2, group_ids)\n",
        "\n",
        "        # Convert aggregated annotations to list of labels per position for each annotator\n",
        "        annotator1_labels = [[] for _ in range(len(text_1))]\n",
        "        annotator2_labels = [[] for _ in range(len(text_2))]\n",
        "\n",
        "        for start, end, label in annotations_1:\n",
        "            for i in range(start, end):\n",
        "                annotator1_labels[i].append(label)\n",
        "\n",
        "        for start, end, label in annotations_2:\n",
        "            for i in range(start, end):\n",
        "                annotator2_labels[i].append(label)\n",
        "\n",
        "        # Speaker Identification Task\n",
        "        speaker_jaccard = jaccard_similarity_for_labels(\n",
        "            [[label for label in labels if label in labels_speaker] for labels in annotator1_labels],\n",
        "            [[label for label in labels if label in labels_speaker] for labels in annotator2_labels]\n",
        "        )\n",
        "\n",
        "        # Emotion Detection Task\n",
        "        emotion_jaccard = jaccard_similarity_for_labels(\n",
        "            [[label for label in labels if label in labels_emotion] for labels in annotator1_labels],\n",
        "            [[label for label in labels if label in labels_emotion] for labels in annotator2_labels]\n",
        "        )\n",
        "\n",
        "        # Append results for this group for each task\n",
        "        results.append({\n",
        "            'group': group_name,\n",
        "            'speaker_jaccard': speaker_jaccard,\n",
        "            'emotion_jaccard': emotion_jaccard\n",
        "        })\n",
        "\n",
        "    return pd.DataFrame(results)\n",
        "\n",
        "# Load your files and run\n",
        "data_annotator1 = load_jsonl(file_path1)\n",
        "data_annotator2 = load_jsonl(file_path2)\n",
        "\n",
        "# Prepare spans for both annotators\n",
        "def get_spans_for_jaccard(data):\n",
        "    spans = []\n",
        "    for doc in data:\n",
        "        text = doc['text']\n",
        "        doc_annotations = [(start, end, label) for start, end, label in doc['label']]\n",
        "        spans.append({'id': doc['id'], 'text': text, 'annotations': doc_annotations})\n",
        "    return spans\n",
        "\n",
        "spans1 = get_spans_for_jaccard(data_annotator1)\n",
        "spans2 = get_spans_for_jaccard(data_annotator2)\n",
        "\n",
        "# Run agreement calculation for each group and task\n",
        "agreement_results_grouped_tasks = calculate_multi_label_agreement_for_tasks(\n",
        "    spans1, spans2, label_mapping_speaker, label_mapping_emotion, group_definitions\n",
        ")\n",
        "\n",
        "# Display the results\n",
        "print(file_path1,file_path2)\n",
        "print(agreement_results_grouped_tasks)\n"
      ],
      "metadata": {
        "id": "kbL2iIAT6Et_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8efd7490-e0e8-448a-f504-856b447e186c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Lea.jsonl Kian.jsonl\n",
            "     group  speaker_jaccard  emotion_jaccard\n",
            "0  Group 1         0.928014         0.386147\n",
            "1  Group 2         0.482335         0.291584\n",
            "2  Group 3              NaN         0.441328\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/numpy/core/fromnumeric.py:3504: RuntimeWarning: Mean of empty slice.\n",
            "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
            "/usr/local/lib/python3.10/dist-packages/numpy/core/_methods.py:129: RuntimeWarning: invalid value encountered in scalar divide\n",
            "  ret = ret.dtype.type(ret / rcount)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Sentiment Only"
      ],
      "metadata": {
        "id": "AZKL5SQs7Lve"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define positive and negative emotions based on the user's instruction\n",
        "positive_emotions = {'Joy', 'Surprise', 'Trust', 'Anticipation'}\n",
        "negative_emotions = {'Sadness', 'Anger', 'Fear', 'Disgust'}\n",
        "\n",
        "# Function to classify labels as positive or negative\n",
        "def classify_sentiment_labels(labels, positive_emotions, negative_emotions):\n",
        "    \"\"\"Classify a list of labels as 'positive', 'negative', or [] if neither.\"\"\"\n",
        "    sentiment_labels = []\n",
        "    for label_set in labels:\n",
        "        if any(label in positive_emotions for label in label_set):\n",
        "            sentiment_labels.append(\"positive\")\n",
        "        elif any(label in negative_emotions for label in label_set):\n",
        "            sentiment_labels.append(\"negative\")\n",
        "        else:\n",
        "            sentiment_labels.append([])  # Empty if no sentiment category applies\n",
        "    return sentiment_labels\n",
        "\n",
        "# Function to calculate sentiment agreement\n",
        "def calculate_sentiment_agreement(spans_annotator1, spans_annotator2, positive_emotions, negative_emotions, group_definitions):\n",
        "    results = []\n",
        "    for group_name, group_ids in group_definitions.items():\n",
        "        # Aggregate annotations for each group\n",
        "        text_1, annotations_1 = aggregate_annotations_multilabel(spans_annotator1, group_ids)\n",
        "        text_2, annotations_2 = aggregate_annotations_multilabel(spans_annotator2, group_ids)\n",
        "\n",
        "        # Convert aggregated annotations to list of labels per position for each annotator\n",
        "        annotator1_labels = [[] for _ in range(len(text_1))]\n",
        "        annotator2_labels = [[] for _ in range(len(text_2))]\n",
        "\n",
        "        for start, end, label in annotations_1:\n",
        "            for i in range(start, end):\n",
        "                annotator1_labels[i].append(label)\n",
        "\n",
        "        for start, end, label in annotations_2:\n",
        "            for i in range(start, end):\n",
        "                annotator2_labels[i].append(label)\n",
        "\n",
        "        # Classify labels by sentiment (positive or negative)\n",
        "        annotator1_sentiment = classify_sentiment_labels(annotator1_labels, positive_emotions, negative_emotions)\n",
        "        annotator2_sentiment = classify_sentiment_labels(annotator2_labels, positive_emotions, negative_emotions)\n",
        "\n",
        "        # Filter to only keep positions where both annotators have classified a sentiment\n",
        "        sentiment_jaccard = jaccard_similarity_for_labels(\n",
        "            [[sentiment] for sentiment in annotator1_sentiment if sentiment],\n",
        "            [[sentiment] for sentiment in annotator2_sentiment if sentiment]\n",
        "        )\n",
        "\n",
        "        # Append results for this group\n",
        "        results.append({\n",
        "            'group': group_name,\n",
        "            'sentiment_jaccard': sentiment_jaccard\n",
        "        })\n",
        "\n",
        "    return pd.DataFrame(results)\n",
        "\n",
        "# Run sentiment agreement calculation\n",
        "sentiment_agreement_results = calculate_sentiment_agreement(\n",
        "    spans1, spans2, positive_emotions, negative_emotions, group_definitions\n",
        ")\n",
        "\n",
        "# Display the results\n",
        "print(file_path1,file_path2)\n",
        "print(sentiment_agreement_results)\n"
      ],
      "metadata": {
        "id": "UrtpUOf77OFb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7e6355e1-bd2c-425f-840b-a564fb5f0edd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Kian.jsonl Inka.jsonl\n",
            "    group  sentiment_jaccard\n",
            "0  Group1           0.634883\n",
            "1  Group2           0.757422\n",
            "2  Group3           0.794830\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Speaker switch"
      ],
      "metadata": {
        "id": "cMeHdRRT77hm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def aggregate_annotations_multilabel(spans, group_ids):\n",
        "    \"\"\"Aggregate annotations from spans for a specific group.\"\"\"\n",
        "    text = ''\n",
        "    annotations = []\n",
        "    for span in spans:\n",
        "        if span['id'] in group_ids:\n",
        "            text += span['text']  # Concatenate text for the group\n",
        "            annotations.extend(span['annotations'])  # Extend annotations list\n",
        "    return text, annotations\n",
        "\n",
        "def identify_speaker_switches(labels):\n",
        "    \"\"\"Identify positions where a speaker switch occurs in the annotation labels.\"\"\"\n",
        "    switch_points = [0] * len(labels)  # Initialize a list to mark switches\n",
        "    for i in range(1, len(labels)):\n",
        "        if labels[i] != labels[i - 1]:  # A switch occurs if the label changes from previous\n",
        "            switch_points[i] = 1\n",
        "    return switch_points\n",
        "\n",
        "def get_spans_for_kappa(data):\n",
        "    \"\"\"Convert JSONL data into spans format expected by calculation functions.\"\"\"\n",
        "    spans = []\n",
        "    for doc in data:\n",
        "        text = doc['text']\n",
        "        doc_annotations = [(start, end, label) for start, end, label in doc['label']]\n",
        "        spans.append({'id': doc.get('id'), 'text': text, 'annotations': doc_annotations})\n",
        "    return spans"
      ],
      "metadata": {
        "id": "kbNTBmvp_yQp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Adjust group definitions based on the user's request\n",
        "group_definitions = {\n",
        "    'Group1': {1, 2, 3, 4},\n",
        "    'Group2': {9, 11, 15},\n",
        "    'Group3': {5, 6, 7, 8, 10, 12, 13, 14, 16},\n",
        "}\n",
        "\n",
        "# List of labels to filter\n",
        "valid_labels = {'Speaker 1', 'Speaker 2', 'Interviewee', 'Interviewer', 'Instructor'}\n",
        "\n",
        "def calculate_speaker_switch_agreement_filtered(spans_annotator1, spans_annotator2, group_definitions, valid_labels):\n",
        "    results = []\n",
        "    for group_name, group_ids in group_definitions.items():\n",
        "        # Aggregate annotations for each group\n",
        "        text_1, annotations_1 = aggregate_annotations_multilabel(spans_annotator1, group_ids)\n",
        "        text_2, annotations_2 = aggregate_annotations_multilabel(spans_annotator2, group_ids)\n",
        "\n",
        "        # Convert aggregated annotations to list of labels per position for each annotator\n",
        "        annotator1_labels = [[] for _ in range(len(text_1))]\n",
        "        annotator2_labels = [[] for _ in range(len(text_2))]\n",
        "\n",
        "        for start, end, label in annotations_1:\n",
        "            if label in valid_labels:  # Only include valid labels\n",
        "                for i in range(start, end):\n",
        "                    annotator1_labels[i].append(label)\n",
        "\n",
        "        for start, end, label in annotations_2:\n",
        "            if label in valid_labels:  # Only include valid labels\n",
        "                for i in range(start, end):\n",
        "                    annotator2_labels[i].append(label)\n",
        "\n",
        "        # Simplify to primary speaker label per position, ignore multi-label\n",
        "        annotator1_primary_labels = [labels[0] if labels else None for labels in annotator1_labels]\n",
        "        annotator2_primary_labels = [labels[0] if labels else None for labels in annotator2_labels]\n",
        "\n",
        "        # Identify switch points for each annotator\n",
        "        annotator1_switches = identify_speaker_switches(annotator1_primary_labels)\n",
        "        annotator2_switches = identify_speaker_switches(annotator2_primary_labels)\n",
        "\n",
        "        # Calculate Cohen's Kappa for switch points\n",
        "        switch_kappa = cohen_kappa_score(annotator1_switches, annotator2_switches)\n",
        "\n",
        "        # Append results for this group\n",
        "        results.append({\n",
        "            'group': group_name,\n",
        "            'switch_kappa': switch_kappa\n",
        "        })\n",
        "\n",
        "    return pd.DataFrame(results)\n",
        "\n",
        "# Calculate Kappa for the filtered data\n",
        "speaker_switch_agreement_results_filtered = calculate_speaker_switch_agreement_filtered(spans1, spans2, group_definitions, valid_labels)\n",
        "\n",
        "# Display the results\n",
        "print(file_path1,file_path2)\n",
        "print(speaker_switch_agreement_results_filtered)\n"
      ],
      "metadata": {
        "id": "hWXHs1wW7_Yg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "016bb6cb-5418-4a82-ab95-1d2c8d473322"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Kian.jsonl Inka.jsonl\n",
            "    group  switch_kappa\n",
            "0  Group1      0.707508\n",
            "1  Group2      0.707756\n",
            "2  Group3      0.000000\n"
          ]
        }
      ]
    }
  ]
}