diff --git a/openhands_resolver/feedback.py b/openhands_resolver/feedback.py new file mode 100644 index 0000000..d87105d --- /dev/null +++ b/openhands_resolver/feedback.py @@ -0,0 +1,64 @@ +from typing import Any, Literal +import json +import logging +import requests +from litellm import BaseModel + +logger = logging.getLogger(__name__) + +class FeedbackDataModel(BaseModel): + version: str + email: str + token: str + feedback: Literal['positive', 'negative'] + permissions: Literal['public', 'private'] + trajectory: list[dict[str, Any]] + + +FEEDBACK_URL = 'https://share-od-trajectory-3u9bw9tx.uc.gateway.dev/share_od_trajectory' +VIEWER_PAGE = "https://www.all-hands.dev/share" + + +def store_feedback(feedback: FeedbackDataModel) -> dict[str, str]: + # Start logging + display_feedback = feedback.model_dump() + if 'trajectory' in display_feedback: + display_feedback['trajectory'] = ( + f"elided [length: {len(display_feedback['trajectory'])}" + ) + if 'token' in display_feedback: + display_feedback['token'] = 'elided' + logger.info(f'Got feedback: {display_feedback}') + + # Start actual request + response = requests.post( + FEEDBACK_URL, + headers={'Content-Type': 'application/json'}, + json=feedback.model_dump(), + ) + if response.status_code != 200: + raise ValueError(f'Failed to store feedback: {response.text}') + response_data = json.loads(response.text) + logger.info(f'Stored feedback: {response.text}') + return response_data + + +def get_trajectory_url(feedback_id: str) -> str: + """Get the URL to view the trajectory.""" + return f"{VIEWER_PAGE}?share_id={feedback_id}" + + +def submit_resolver_output(output: 'ResolverOutput', token: str) -> str: + """Submit a ResolverOutput to Share-OpenHands and return the trajectory URL.""" + feedback = FeedbackDataModel( + version='1.0', + email='openhands@all-hands.dev', + token=token, + feedback='positive' if output.success else 'negative', + permissions='private', + trajectory=output.history + ) + + response_data = store_feedback(feedback) + trajectory_url = get_trajectory_url(response_data['feedback_id']) + return trajectory_url diff --git a/openhands_resolver/resolve_issues.py b/openhands_resolver/resolve_issues.py index 89cfb89..41342e1 100644 --- a/openhands_resolver/resolve_issues.py +++ b/openhands_resolver/resolve_issues.py @@ -22,6 +22,7 @@ PRHandler ) from openhands_resolver.resolver_output import ResolverOutput +from openhands_resolver.feedback import submit_resolver_output import openhands from openhands.core.main import create_runtime, run_controller from openhands.controller.state.state import State @@ -298,6 +299,16 @@ async def process_issue( success_explanation=success_explanation, error=state.last_error if state and state.last_error else None, ) + + # Submit feedback and get trajectory URL + try: + trajectory_url = submit_resolver_output(output, token) + output.trajectory_url = trajectory_url + logger.info(f'Successfully submitted feedback, trajectory URL: {trajectory_url}') + except Exception as e: + logger.error(f'Failed to submit feedback: {e}') + output.trajectory_url = None + return output # This function tracks the progress AND write the output to a JSONL file @@ -669,4 +680,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/openhands_resolver/resolver_output.py b/openhands_resolver/resolver_output.py index 4b539b4..145e332 100644 --- a/openhands_resolver/resolver_output.py +++ b/openhands_resolver/resolver_output.py @@ -16,3 +16,4 @@ class ResolverOutput(BaseModel): comment_success: list[bool] | None success_explanation: str error: str | None + trajectory_url: str | None = None diff --git a/poetry.lock b/poetry.lock index c5dfd1c..bcecc5a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6696,4 +6696,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "41ff3359f67e4f51da7cc4b275f8b52e2a72bc491387cd781fa40e85e3e6dac7" +content-hash = "80c558b5c1989ed71e927d2b12849fe3f343055d6b91114894ee9b6ccdf18535" diff --git a/pyproject.toml b/pyproject.toml index 34a2ec0..b28fd02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ python = "^3.12" openhands-ai = "^0.12.0" pandas = "^2.2.3" pytest = "^8.3.3" +requests = "^2.32.3" [tool.poetry.group.dev.dependencies] mypy = "*" diff --git a/tests/test_feedback.py b/tests/test_feedback.py new file mode 100644 index 0000000..e45d9a7 --- /dev/null +++ b/tests/test_feedback.py @@ -0,0 +1,115 @@ +import pytest +from unittest.mock import patch, MagicMock +from openhands_resolver.feedback import ( + FeedbackDataModel, + store_feedback, + get_trajectory_url, + submit_resolver_output +) +from openhands_resolver.resolver_output import ResolverOutput +from openhands_resolver.github_issue import GithubIssue + + +@pytest.fixture +def mock_resolver_output(): + return ResolverOutput( + issue=GithubIssue( + owner="test-owner", + repo="test-repo", + number=123, + title="Test Issue", + body="Test Body", + ), + issue_type="issue", + instruction="Fix the bug", + base_commit="abc123", + git_patch="test patch", + history=[{"type": "test", "data": "test"}], + metrics={"test": "test"}, + success=True, + comment_success=[True], + success_explanation="Fixed successfully", + error=None, + trajectory_url=None, + ) + + +def test_feedback_data_model(): + feedback = FeedbackDataModel( + version="1.0", + email="test@example.com", + token="test-token", + feedback="positive", + permissions="private", + trajectory=[{"type": "test", "data": "test"}] + ) + assert feedback.version == "1.0" + assert feedback.email == "test@example.com" + assert feedback.token == "test-token" + assert feedback.feedback == "positive" + assert feedback.permissions == "private" + assert feedback.trajectory == [{"type": "test", "data": "test"}] + + +def test_get_trajectory_url(): + url = get_trajectory_url("test-id") + assert url == "https://www.all-hands.dev/share?share_id=test-id" + + +@patch('requests.post') +def test_store_feedback_success(mock_post): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = '{"feedback_id": "test-id"}' + mock_post.return_value = mock_response + + feedback = FeedbackDataModel( + version="1.0", + email="test@example.com", + token="test-token", + feedback="positive", + permissions="private", + trajectory=[{"type": "test", "data": "test"}] + ) + + response = store_feedback(feedback) + assert response == {"feedback_id": "test-id"} + mock_post.assert_called_once() + + +@patch('requests.post') +def test_store_feedback_failure(mock_post): + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.text = 'Error' + mock_post.return_value = mock_response + + feedback = FeedbackDataModel( + version="1.0", + email="test@example.com", + token="test-token", + feedback="positive", + permissions="private", + trajectory=[{"type": "test", "data": "test"}] + ) + + with pytest.raises(ValueError, match="Failed to store feedback: Error"): + store_feedback(feedback) + + +@patch('openhands_resolver.feedback.store_feedback') +def test_submit_resolver_output_success(mock_store_feedback, mock_resolver_output): + mock_store_feedback.return_value = {"feedback_id": "test-id"} + + url = submit_resolver_output(mock_resolver_output, "test-token") + assert url == "https://www.all-hands.dev/share?share_id=test-id" + + mock_store_feedback.assert_called_once() + feedback = mock_store_feedback.call_args[0][0] + assert isinstance(feedback, FeedbackDataModel) + assert feedback.version == "1.0" + assert feedback.email == "openhands@all-hands.dev" + assert feedback.token == "test-token" + assert feedback.feedback == "positive" + assert feedback.permissions == "private" + assert feedback.trajectory == mock_resolver_output.history