chore: Add support for parsing key-value pairs in URL source response (…

…#97)
locaal-ai · May 17, 2024 · a665e26 · a665e26
1 parent 802803c
commit a665e26
Show file tree

Hide file tree

Showing 9 changed files with 119 additions and 40 deletions.
diff --git a/buildspec.json b/buildspec.json
@@ -45,7 +45,7 @@
         }
     },
     "name": "obs-urlsource",
-    "version": "0.3.0",
+    "version": "0.3.1",
     "author": "Roy Shilkrot",
     "website": "https://github.com/occ-ai/obs-urlsource",
     "email": "[email protected]",

diff --git a/src/parsers/CMakeLists.txt b/src/parsers/CMakeLists.txt
@@ -1,12 +1,13 @@
 target_sources(
   ${CMAKE_PROJECT_NAME}
-  PRIVATE jsonpointer.cpp
-          jsonpath.cpp
-          regex.cpp
-          xml.cpp
+  PRIVATE binary-data.cpp
           errors.cpp
           html.cpp
-          binary-data.cpp)
+          jsonpath.cpp
+          jsonpointer.cpp
+          key-value.cpp
+          regex.cpp
+          xml.cpp)
 
 # on linux, disable conversion errors
 if(UNIX AND NOT APPLE)

diff --git a/src/parsers/key-value.cpp b/src/parsers/key-value.cpp
@@ -0,0 +1,37 @@
+#include "errors.h"
+#include "request-data.h"
+
+#include <obs-module.h>
+
+#include <sstream>
+#include <string>
+
+struct request_data_handler_response parse_key_value(struct request_data_handler_response response,
+						     const url_source_request_data *request_data)
+{
+
+	UNUSED_PARAMETER(request_data);
+
+	// Create a string stream from the body
+	std::istringstream body_stream(response.body);
+	std::string line;
+
+	// Iterate through each line of the body
+	while (std::getline(body_stream, line)) {
+		// Skip empty lines
+		if (line.empty())
+			continue;
+
+		// Split the line by the delimiter
+		// look for the first occurrence of the delimiter from the beginning of the line
+		size_t delimiter_pos = line.find(request_data->kv_delimiter);
+		if (delimiter_pos != std::string::npos) {
+			std::string key = line.substr(0, delimiter_pos);
+			std::string value = line.substr(delimiter_pos + 1);
+			response.key_value_pairs[key] = value;
+			response.body_parts_parsed.push_back(line);
+		}
+	}
+
+	return response;
+}
diff --git a/src/parsers/parsers.h b/src/parsers/parsers.h
@@ -32,4 +32,7 @@ struct request_data_handler_response
 parse_xml_by_xquery(struct request_data_handler_response response,
 		    const url_source_request_data *request_data);
 
+struct request_data_handler_response parse_key_value(struct request_data_handler_response response,
+						     const url_source_request_data *request_data);
+
 #endif // PARSERS_H
diff --git a/src/request-data.cpp b/src/request-data.cpp
@@ -464,6 +464,8 @@ struct request_data_handler_response request_data_handler(url_source_request_dat
 			// attempt to parse as json and return the whole object
 			response = parse_json(response, request_data);
 		}
+	} else if (request_data->output_type == "Key-Value") {
+		response = parse_key_value(response, request_data);
 	} else if (request_data->output_type == "XML (XPath)") {
 		response = parse_xml(response, request_data);
 	} else if (request_data->output_type == "XML (XQuery)") {
@@ -545,6 +547,7 @@ std::string serialize_request_data(url_source_request_data *request_data)
 	json["output_regex_flags"] = request_data->output_regex_flags;
 	json["output_regex_group"] = request_data->output_regex_group;
 	json["output_cssselector"] = request_data->output_cssselector;
+	json["kv_delimiter"] = request_data->kv_delimiter;
 	// postprocess options
 	json["post_process_regex"] = request_data->post_process_regex;
 	json["post_process_regex_is_replace"] = request_data->post_process_regex_is_replace;
@@ -622,6 +625,7 @@ url_source_request_data unserialize_request_data(std::string serialized_request_
 		request_data.output_regex_flags = json["output_regex_flags"].get<std::string>();
 		request_data.output_regex_group = json["output_regex_group"].get<std::string>();
 		request_data.output_cssselector = json.value("output_cssselector", "");
+		request_data.kv_delimiter = json.value("kv_delimiter", "=");
 
 		// postprocess options
 		request_data.post_process_regex = json.value("post_process_regex", "");

diff --git a/src/request-data.h b/src/request-data.h
@@ -119,6 +119,7 @@ struct url_source_request_data {
 	std::string post_process_regex;
 	bool post_process_regex_is_replace;
 	std::string post_process_regex_replace;
+	std::string kv_delimiter;
 
 	// default constructor
 	url_source_request_data()
@@ -149,6 +150,10 @@ struct url_source_request_data {
 		output_regex_flags = std::string("");
 		output_regex_group = std::string("0");
 		output_cssselector = std::string("");
+		post_process_regex = std::string("");
+		post_process_regex_is_replace = false;
+		post_process_regex_replace = std::string("");
+		kv_delimiter = std::string("=");
 	}
 };
 
@@ -158,6 +163,7 @@ struct request_data_handler_response {
 	nlohmann::json body_json;
 	std::string content_type;
 	std::vector<std::string> body_parts_parsed;
+	std::map<std::string, std::string> key_value_pairs;
 	std::map<std::string, std::string> headers;
 	int status_code = URL_SOURCE_REQUEST_SUCCESS;
 	long http_status_code;

diff --git a/src/ui/RequestBuilder.cpp b/src/ui/RequestBuilder.cpp
@@ -434,18 +434,23 @@ RequestBuilder::RequestBuilder(url_source_request_data *request_data,
 	ui->outputRegexGroupLineEdit->setText(
 		QString::fromStdString(request_data->output_regex_group));
 	ui->cssSelectorLineEdit->setText(QString::fromStdString(request_data->output_cssselector));
+	ui->lineEdit_delimiter->setText(QString::fromStdString(request_data->kv_delimiter));
 	auto setVisibilityOfOutputParsingOptions = [=]() {
 		// Hide all output parsing options
 		for (const auto &widget :
 		     {ui->outputJSONPathLineEdit, ui->outputXPathLineEdit, ui->outputXQueryLineEdit,
 		      ui->outputRegexLineEdit, ui->outputRegexFlagsLineEdit,
 		      ui->outputRegexGroupLineEdit, ui->outputJSONPointerLineEdit,
-		      ui->cssSelectorLineEdit, ui->postProcessRegexLineEdit}) {
+		      ui->cssSelectorLineEdit, ui->postProcessRegexLineEdit,
+		      ui->lineEdit_delimiter}) {
 			set_form_row_visibility(ui->formOutputParsing, widget, false);
 		}
 
 		// Show the output parsing options for the selected output type
-		if (ui->outputTypeComboBox->currentText() == "JSON") {
+		if (ui->outputTypeComboBox->currentText() == "Key-Value") {
+			set_form_row_visibility(ui->formOutputParsing, ui->lineEdit_delimiter,
+						true);
+		} else if (ui->outputTypeComboBox->currentText() == "JSON") {
 			set_form_row_visibility(ui->formOutputParsing, ui->outputJSONPathLineEdit,
 						true);
 			set_form_row_visibility(ui->formOutputParsing,
@@ -577,6 +582,8 @@ RequestBuilder::RequestBuilder(url_source_request_data *request_data,
 			ui->outputRegexGroupLineEdit->text().toStdString();
 		request_data_for_saving->output_cssselector =
 			ui->cssSelectorLineEdit->text().toStdString();
+		request_data_for_saving->kv_delimiter =
+			ui->lineEdit_delimiter->text().toStdString();
 
 		// Save the postprocess regex options
 		request_data_for_saving->post_process_regex =

diff --git a/src/ui/requestbuilder.ui b/src/ui/requestbuilder.ui
@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>493</width>
-    <height>809</height>
+    <height>931</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -747,7 +747,7 @@
       <item row="0" column="0">
        <widget class="QLabel" name="label_7">
         <property name="text">
-         <string>Content-Type</string>
+         <string>Parsing Type</string>
         </property>
        </widget>
       </item>
@@ -758,6 +758,11 @@
           <string>Text</string>
          </property>
         </item>
+        <item>
+         <property name="text">
+          <string>Key-Value</string>
+         </property>
+        </item>
         <item>
          <property name="text">
           <string>Image (data)</string>
@@ -790,102 +795,102 @@
         </item>
        </widget>
       </item>
-      <item row="1" column="0">
+      <item row="2" column="0">
        <widget class="QLabel" name="label_8">
         <property name="text">
          <string>JSON Pointer</string>
         </property>
        </widget>
       </item>
-      <item row="1" column="1">
+      <item row="2" column="1">
        <widget class="QLineEdit" name="outputJSONPointerLineEdit">
         <property name="placeholderText">
          <string/>
         </property>
        </widget>
       </item>
-      <item row="2" column="0">
+      <item row="3" column="0">
        <widget class="QLabel" name="label_9">
         <property name="text">
          <string>JSON Path</string>
         </property>
        </widget>
       </item>
-      <item row="2" column="1">
+      <item row="3" column="1">
        <widget class="QLineEdit" name="outputJSONPathLineEdit"/>
       </item>
-      <item row="3" column="0">
+      <item row="4" column="0">
        <widget class="QLabel" name="label_10">
         <property name="text">
          <string>XPath</string>
         </property>
        </widget>
       </item>
-      <item row="3" column="1">
+      <item row="4" column="1">
        <widget class="QLineEdit" name="outputXPathLineEdit"/>
       </item>
-      <item row="4" column="0">
+      <item row="5" column="0">
        <widget class="QLabel" name="label_11">
         <property name="text">
          <string>XQuery</string>
         </property>
        </widget>
       </item>
-      <item row="4" column="1">
+      <item row="5" column="1">
        <widget class="QLineEdit" name="outputXQueryLineEdit"/>
       </item>
-      <item row="5" column="0">
+      <item row="6" column="0">
        <widget class="QLabel" name="label_12">
         <property name="text">
          <string>Regex</string>
         </property>
        </widget>
       </item>
-      <item row="5" column="1">
+      <item row="6" column="1">
        <widget class="QLineEdit" name="outputRegexLineEdit"/>
       </item>
-      <item row="6" column="0">
+      <item row="7" column="0">
        <widget class="QLabel" name="label_13">
         <property name="text">
          <string>Regex Flags</string>
         </property>
        </widget>
       </item>
-      <item row="6" column="1">
+      <item row="7" column="1">
        <widget class="QLineEdit" name="outputRegexFlagsLineEdit"/>
       </item>
-      <item row="7" column="0">
+      <item row="8" column="0">
        <widget class="QLabel" name="label_14">
         <property name="text">
          <string>Regex Group</string>
         </property>
        </widget>
       </item>
-      <item row="7" column="1">
+      <item row="8" column="1">
        <widget class="QLineEdit" name="outputRegexGroupLineEdit">
         <property name="text">
          <string>0</string>
         </property>
        </widget>
       </item>
-      <item row="8" column="0">
+      <item row="9" column="0">
        <widget class="QLabel" name="label_17">
         <property name="text">
          <string>CSS Selector</string>
         </property>
        </widget>
       </item>
-      <item row="8" column="1">
+      <item row="9" column="1">
        <widget class="QLineEdit" name="cssSelectorLineEdit"/>
       </item>
-      <item row="9" column="0">
+      <item row="10" column="0">
        <widget class="QLabel" name="label_15">
         <property name="text">
          <string>Postprocess Regex</string>
         </property>
        </widget>
       </item>
-      <item row="9" column="1">
+      <item row="10" column="1">
        <widget class="QWidget" name="widget_9" native="true">
         <layout class="QHBoxLayout" name="horizontalLayout_7">
          <property name="leftMargin">
@@ -916,6 +921,20 @@
         </layout>
        </widget>
       </item>
+      <item row="1" column="1">
+       <widget class="QLineEdit" name="lineEdit_delimiter">
+        <property name="text">
+         <string>=</string>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0">
+       <widget class="QLabel" name="label_18">
+        <property name="text">
+         <string>Delimiter</string>
+        </property>
+       </widget>
+      </item>
      </layout>
     </widget>
    </item>

diff --git a/src/url-source-callbacks.cpp b/src/url-source-callbacks.cpp
@@ -100,21 +100,25 @@ void setAudioCallback(const std::string &str, const output_mapping &mapping)
 };
 
 std::string renderOutputTemplate(inja::Environment &env, const std::string &input,
-				 const std::vector<std::string> &outputs,
-				 const nlohmann::json &body)
+				 const request_data_handler_response &response)
 try {
 	// Use Inja to render the template
 	nlohmann::json data;
-	if (outputs.size() > 1) {
-		for (size_t i = 0; i < outputs.size(); i++) {
-			data["output" + std::to_string(i)] = outputs[i];
+	if (response.body_parts_parsed.size() > 1) {
+		for (size_t i = 0; i < response.body_parts_parsed.size(); i++) {
+			data["output" + std::to_string(i)] = response.body_parts_parsed[i];
 		}
 		// in "output" add an array of all the outputs
-		data["output"] = outputs;
+		data["output"] = response.body_parts_parsed;
 	} else {
-		data["output"] = outputs[0];
+		data["output"] = response.body_parts_parsed[0];
 	}
-	data["body"] = body;
+	if (response.key_value_pairs.size() > 0) {
+		for (const auto &pair : response.key_value_pairs) {
+			data[pair.first] = pair.second;
+		}
+	}
+	data["body"] = response.body_json;
 	return env.render(input, data);
 } catch (std::exception &e) {
 	obs_log(LOG_ERROR, "Failed to parse template: %s", e.what());
@@ -172,8 +176,7 @@ std::string prepare_text_from_template(const output_mapping &mapping,
 				mime_type = response.headers.at("content-type");
 			}
 		} else {
-			text = renderOutputTemplate(env, text, response.body_parts_parsed,
-						    response.body_json);
+			text = renderOutputTemplate(env, text, response);
 			// use fetch_image to get the image
 			image_data = fetch_image(text.c_str(), mime_type);
 		}
@@ -182,8 +185,7 @@ std::string prepare_text_from_template(const output_mapping &mapping,
 		// build an image tag with the base64 image
 		text = "<img src=\"data:" + mime_type + ";base64," + base64_image + "\" />";
 	} else {
-		text = renderOutputTemplate(env, text, response.body_parts_parsed,
-					    response.body_json);
+		text = renderOutputTemplate(env, text, response);
 	}
 	return text;
 }