Fix issue #127: Upload resolver output to Share-OpenHands and also send the URL to the Github issue

openhands_resolver/feedback.py

@@ -0,0 +1,64 @@
+from typing import Any, Literal
+import json
+import logging
+import requests
+from litellm import BaseModel
+
+logger = logging.getLogger(__name__)
+
+class FeedbackDataModel(BaseModel):
+    version: str
+    email: str
+    token: str
+    feedback: Literal['positive', 'negative']
+    permissions: Literal['public', 'private']
+    trajectory: list[dict[str, Any]]
+
+
+FEEDBACK_URL = 'https://share-od-trajectory-3u9bw9tx.uc.gateway.dev/share_od_trajectory'
+VIEWER_PAGE = "https://www.all-hands.dev/share"
+
+
+def store_feedback(feedback: FeedbackDataModel) -> dict[str, str]:
+    # Start logging
+    display_feedback = feedback.model_dump()
+    if 'trajectory' in display_feedback:
+        display_feedback['trajectory'] = (
+            f"elided [length: {len(display_feedback['trajectory'])}"
+        )
+    if 'token' in display_feedback:
+        display_feedback['token'] = 'elided'
+    logger.info(f'Got feedback: {display_feedback}')
+
+    # Start actual request
+    response = requests.post(
+        FEEDBACK_URL,
+        headers={'Content-Type': 'application/json'},
+        json=feedback.model_dump(),
+    )
+    if response.status_code != 200:
+        raise ValueError(f'Failed to store feedback: {response.text}')
+    response_data = json.loads(response.text)
+    logger.info(f'Stored feedback: {response.text}')
+    return response_data
+
+
+def get_trajectory_url(feedback_id: str) -> str:
+    """Get the URL to view the trajectory."""
+    return f"{VIEWER_PAGE}?share_id={feedback_id}"
+
+
+def submit_resolver_output(output: 'ResolverOutput', token: str) -> str:
+    """Submit a ResolverOutput to Share-OpenHands and return the trajectory URL."""
+    feedback = FeedbackDataModel(
+        version='1.0',
+        email='[email protected]',
+        token=token,
+        feedback='positive' if output.success else 'negative',
+        permissions='private',
+        trajectory=output.history
+    )
+
+    response_data = store_feedback(feedback)
+    trajectory_url = get_trajectory_url(response_data['feedback_id'])
+    return trajectory_url

openhands_resolver/resolve_issues.py

@@ -22,6 +22,7 @@
     PRHandler
 )
 from openhands_resolver.resolver_output import ResolverOutput
+from openhands_resolver.feedback import submit_resolver_output
 import openhands
 from openhands.core.main import create_runtime, run_controller
 from openhands.controller.state.state import State
@@ -298,6 +299,16 @@ async def process_issue(
         success_explanation=success_explanation,
         error=state.last_error if state and state.last_error else None,
     )
+
+    # Submit feedback and get trajectory URL
+    try:
+        trajectory_url = submit_resolver_output(output, token)
+        output.trajectory_url = trajectory_url
+        logger.info(f'Successfully submitted feedback, trajectory URL: {trajectory_url}')
+    except Exception as e:
+        logger.error(f'Failed to submit feedback: {e}')
+        output.trajectory_url = None
+
     return output
 
 # This function tracks the progress AND write the output to a JSONL file
@@ -669,4 +680,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

openhands_resolver/resolver_output.py

@@ -16,3 +16,4 @@ class ResolverOutput(BaseModel):
     comment_success: list[bool] | None
     success_explanation: str
     error: str | None
+    trajectory_url: str | None = None

pyproject.toml

@@ -20,6 +20,7 @@ python = "^3.12"
 openhands-ai = "^0.12.0"
 pandas = "^2.2.3"
 pytest = "^8.3.3"
+requests = "^2.32.3"
 
 [tool.poetry.group.dev.dependencies]
 mypy = "*"

tests/test_feedback.py

@@ -0,0 +1,115 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from openhands_resolver.feedback import (
+    FeedbackDataModel,
+    store_feedback,
+    get_trajectory_url,
+    submit_resolver_output
+)
+from openhands_resolver.resolver_output import ResolverOutput
+from openhands_resolver.github_issue import GithubIssue
+
+
+@pytest.fixture
+def mock_resolver_output():
+    return ResolverOutput(
+        issue=GithubIssue(
+            owner="test-owner",
+            repo="test-repo",
+            number=123,
+            title="Test Issue",
+            body="Test Body",
+        ),
+        issue_type="issue",
+        instruction="Fix the bug",
+        base_commit="abc123",
+        git_patch="test patch",
+        history=[{"type": "test", "data": "test"}],
+        metrics={"test": "test"},
+        success=True,
+        comment_success=[True],
+        success_explanation="Fixed successfully",
+        error=None,
+        trajectory_url=None,
+    )
+
+
+def test_feedback_data_model():
+    feedback = FeedbackDataModel(
+        version="1.0",
+        email="[email protected]",
+        token="test-token",
+        feedback="positive",
+        permissions="private",
+        trajectory=[{"type": "test", "data": "test"}]
+    )
+    assert feedback.version == "1.0"
+    assert feedback.email == "[email protected]"
+    assert feedback.token == "test-token"
+    assert feedback.feedback == "positive"
+    assert feedback.permissions == "private"
+    assert feedback.trajectory == [{"type": "test", "data": "test"}]
+
+
+def test_get_trajectory_url():
+    url = get_trajectory_url("test-id")
+    assert url == "https://www.all-hands.dev/share?share_id=test-id"
+
+
+@patch('requests.post')
+def test_store_feedback_success(mock_post):
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.text = '{"feedback_id": "test-id"}'
+    mock_post.return_value = mock_response
+
+    feedback = FeedbackDataModel(
+        version="1.0",
+        email="[email protected]",
+        token="test-token",
+        feedback="positive",
+        permissions="private",
+        trajectory=[{"type": "test", "data": "test"}]
+    )
+
+    response = store_feedback(feedback)
+    assert response == {"feedback_id": "test-id"}
+    mock_post.assert_called_once()
+
+
+@patch('requests.post')
+def test_store_feedback_failure(mock_post):
+    mock_response = MagicMock()
+    mock_response.status_code = 400
+    mock_response.text = 'Error'
+    mock_post.return_value = mock_response
+
+    feedback = FeedbackDataModel(
+        version="1.0",
+        email="[email protected]",
+        token="test-token",
+        feedback="positive",
+        permissions="private",
+        trajectory=[{"type": "test", "data": "test"}]
+    )
+
+    with pytest.raises(ValueError, match="Failed to store feedback: Error"):
+        store_feedback(feedback)
+
+
+@patch('openhands_resolver.feedback.store_feedback')
+def test_submit_resolver_output_success(mock_store_feedback, mock_resolver_output):
+    mock_store_feedback.return_value = {"feedback_id": "test-id"}
+
+    url = submit_resolver_output(mock_resolver_output, "test-token")
+    assert url == "https://www.all-hands.dev/share?share_id=test-id"
+
+    mock_store_feedback.assert_called_once()
+    feedback = mock_store_feedback.call_args[0][0]
+    assert isinstance(feedback, FeedbackDataModel)
+    assert feedback.version == "1.0"
+    assert feedback.email == "[email protected]"
+    assert feedback.token == "test-token"
+    assert feedback.feedback == "positive"
+    assert feedback.permissions == "private"
+    assert feedback.trajectory == mock_resolver_output.history