diff --git a/buildspec.json b/buildspec.json index a4ae984..e456035 100644 --- a/buildspec.json +++ b/buildspec.json @@ -45,7 +45,7 @@ } }, "name": "obs-urlsource", - "version": "0.3.0", + "version": "0.3.1", "author": "Roy Shilkrot", "website": "https://github.com/occ-ai/obs-urlsource", "email": "roy.shil@gmail.com", diff --git a/src/parsers/CMakeLists.txt b/src/parsers/CMakeLists.txt index 49a78ae..886318a 100644 --- a/src/parsers/CMakeLists.txt +++ b/src/parsers/CMakeLists.txt @@ -1,12 +1,13 @@ target_sources( ${CMAKE_PROJECT_NAME} - PRIVATE jsonpointer.cpp - jsonpath.cpp - regex.cpp - xml.cpp + PRIVATE binary-data.cpp errors.cpp html.cpp - binary-data.cpp) + jsonpath.cpp + jsonpointer.cpp + key-value.cpp + regex.cpp + xml.cpp) # on linux, disable conversion errors if(UNIX AND NOT APPLE) diff --git a/src/parsers/key-value.cpp b/src/parsers/key-value.cpp new file mode 100644 index 0000000..5397a2d --- /dev/null +++ b/src/parsers/key-value.cpp @@ -0,0 +1,37 @@ +#include "errors.h" +#include "request-data.h" + +#include + +#include +#include + +struct request_data_handler_response parse_key_value(struct request_data_handler_response response, + const url_source_request_data *request_data) +{ + + UNUSED_PARAMETER(request_data); + + // Create a string stream from the body + std::istringstream body_stream(response.body); + std::string line; + + // Iterate through each line of the body + while (std::getline(body_stream, line)) { + // Skip empty lines + if (line.empty()) + continue; + + // Split the line by the delimiter + // look for the first occurrence of the delimiter from the beginning of the line + size_t delimiter_pos = line.find(request_data->kv_delimiter); + if (delimiter_pos != std::string::npos) { + std::string key = line.substr(0, delimiter_pos); + std::string value = line.substr(delimiter_pos + 1); + response.key_value_pairs[key] = value; + response.body_parts_parsed.push_back(line); + } + } + + return response; +} diff --git a/src/parsers/parsers.h b/src/parsers/parsers.h index 74e7148..80284f9 100644 --- a/src/parsers/parsers.h +++ b/src/parsers/parsers.h @@ -32,4 +32,7 @@ struct request_data_handler_response parse_xml_by_xquery(struct request_data_handler_response response, const url_source_request_data *request_data); +struct request_data_handler_response parse_key_value(struct request_data_handler_response response, + const url_source_request_data *request_data); + #endif // PARSERS_H diff --git a/src/request-data.cpp b/src/request-data.cpp index 7b52b97..60fa89e 100644 --- a/src/request-data.cpp +++ b/src/request-data.cpp @@ -464,6 +464,8 @@ struct request_data_handler_response request_data_handler(url_source_request_dat // attempt to parse as json and return the whole object response = parse_json(response, request_data); } + } else if (request_data->output_type == "Key-Value") { + response = parse_key_value(response, request_data); } else if (request_data->output_type == "XML (XPath)") { response = parse_xml(response, request_data); } else if (request_data->output_type == "XML (XQuery)") { @@ -545,6 +547,7 @@ std::string serialize_request_data(url_source_request_data *request_data) json["output_regex_flags"] = request_data->output_regex_flags; json["output_regex_group"] = request_data->output_regex_group; json["output_cssselector"] = request_data->output_cssselector; + json["kv_delimiter"] = request_data->kv_delimiter; // postprocess options json["post_process_regex"] = request_data->post_process_regex; json["post_process_regex_is_replace"] = request_data->post_process_regex_is_replace; @@ -622,6 +625,7 @@ url_source_request_data unserialize_request_data(std::string serialized_request_ request_data.output_regex_flags = json["output_regex_flags"].get(); request_data.output_regex_group = json["output_regex_group"].get(); request_data.output_cssselector = json.value("output_cssselector", ""); + request_data.kv_delimiter = json.value("kv_delimiter", "="); // postprocess options request_data.post_process_regex = json.value("post_process_regex", ""); diff --git a/src/request-data.h b/src/request-data.h index a9f4ef1..9d3eb63 100644 --- a/src/request-data.h +++ b/src/request-data.h @@ -119,6 +119,7 @@ struct url_source_request_data { std::string post_process_regex; bool post_process_regex_is_replace; std::string post_process_regex_replace; + std::string kv_delimiter; // default constructor url_source_request_data() @@ -149,6 +150,10 @@ struct url_source_request_data { output_regex_flags = std::string(""); output_regex_group = std::string("0"); output_cssselector = std::string(""); + post_process_regex = std::string(""); + post_process_regex_is_replace = false; + post_process_regex_replace = std::string(""); + kv_delimiter = std::string("="); } }; @@ -158,6 +163,7 @@ struct request_data_handler_response { nlohmann::json body_json; std::string content_type; std::vector body_parts_parsed; + std::map key_value_pairs; std::map headers; int status_code = URL_SOURCE_REQUEST_SUCCESS; long http_status_code; diff --git a/src/ui/RequestBuilder.cpp b/src/ui/RequestBuilder.cpp index 7b91f4f..66a03b1 100644 --- a/src/ui/RequestBuilder.cpp +++ b/src/ui/RequestBuilder.cpp @@ -434,18 +434,23 @@ RequestBuilder::RequestBuilder(url_source_request_data *request_data, ui->outputRegexGroupLineEdit->setText( QString::fromStdString(request_data->output_regex_group)); ui->cssSelectorLineEdit->setText(QString::fromStdString(request_data->output_cssselector)); + ui->lineEdit_delimiter->setText(QString::fromStdString(request_data->kv_delimiter)); auto setVisibilityOfOutputParsingOptions = [=]() { // Hide all output parsing options for (const auto &widget : {ui->outputJSONPathLineEdit, ui->outputXPathLineEdit, ui->outputXQueryLineEdit, ui->outputRegexLineEdit, ui->outputRegexFlagsLineEdit, ui->outputRegexGroupLineEdit, ui->outputJSONPointerLineEdit, - ui->cssSelectorLineEdit, ui->postProcessRegexLineEdit}) { + ui->cssSelectorLineEdit, ui->postProcessRegexLineEdit, + ui->lineEdit_delimiter}) { set_form_row_visibility(ui->formOutputParsing, widget, false); } // Show the output parsing options for the selected output type - if (ui->outputTypeComboBox->currentText() == "JSON") { + if (ui->outputTypeComboBox->currentText() == "Key-Value") { + set_form_row_visibility(ui->formOutputParsing, ui->lineEdit_delimiter, + true); + } else if (ui->outputTypeComboBox->currentText() == "JSON") { set_form_row_visibility(ui->formOutputParsing, ui->outputJSONPathLineEdit, true); set_form_row_visibility(ui->formOutputParsing, @@ -577,6 +582,8 @@ RequestBuilder::RequestBuilder(url_source_request_data *request_data, ui->outputRegexGroupLineEdit->text().toStdString(); request_data_for_saving->output_cssselector = ui->cssSelectorLineEdit->text().toStdString(); + request_data_for_saving->kv_delimiter = + ui->lineEdit_delimiter->text().toStdString(); // Save the postprocess regex options request_data_for_saving->post_process_regex = diff --git a/src/ui/requestbuilder.ui b/src/ui/requestbuilder.ui index bdb475d..61e6e73 100644 --- a/src/ui/requestbuilder.ui +++ b/src/ui/requestbuilder.ui @@ -7,7 +7,7 @@ 0 0 493 - 809 + 931 @@ -747,7 +747,7 @@ - Content-Type + Parsing Type @@ -758,6 +758,11 @@ Text + + + Key-Value + + Image (data) @@ -790,102 +795,102 @@ - + JSON Pointer - + - + JSON Path - + - + XPath - + - + XQuery - + - + Regex - + - + Regex Flags - + - + Regex Group - + 0 - + CSS Selector - + - + Postprocess Regex - + @@ -916,6 +921,20 @@ + + + + = + + + + + + + Delimiter + + + diff --git a/src/url-source-callbacks.cpp b/src/url-source-callbacks.cpp index eda3216..82f79ce 100644 --- a/src/url-source-callbacks.cpp +++ b/src/url-source-callbacks.cpp @@ -100,21 +100,25 @@ void setAudioCallback(const std::string &str, const output_mapping &mapping) }; std::string renderOutputTemplate(inja::Environment &env, const std::string &input, - const std::vector &outputs, - const nlohmann::json &body) + const request_data_handler_response &response) try { // Use Inja to render the template nlohmann::json data; - if (outputs.size() > 1) { - for (size_t i = 0; i < outputs.size(); i++) { - data["output" + std::to_string(i)] = outputs[i]; + if (response.body_parts_parsed.size() > 1) { + for (size_t i = 0; i < response.body_parts_parsed.size(); i++) { + data["output" + std::to_string(i)] = response.body_parts_parsed[i]; } // in "output" add an array of all the outputs - data["output"] = outputs; + data["output"] = response.body_parts_parsed; } else { - data["output"] = outputs[0]; + data["output"] = response.body_parts_parsed[0]; } - data["body"] = body; + if (response.key_value_pairs.size() > 0) { + for (const auto &pair : response.key_value_pairs) { + data[pair.first] = pair.second; + } + } + data["body"] = response.body_json; return env.render(input, data); } catch (std::exception &e) { obs_log(LOG_ERROR, "Failed to parse template: %s", e.what()); @@ -172,8 +176,7 @@ std::string prepare_text_from_template(const output_mapping &mapping, mime_type = response.headers.at("content-type"); } } else { - text = renderOutputTemplate(env, text, response.body_parts_parsed, - response.body_json); + text = renderOutputTemplate(env, text, response); // use fetch_image to get the image image_data = fetch_image(text.c_str(), mime_type); } @@ -182,8 +185,7 @@ std::string prepare_text_from_template(const output_mapping &mapping, // build an image tag with the base64 image text = ""; } else { - text = renderOutputTemplate(env, text, response.body_parts_parsed, - response.body_json); + text = renderOutputTemplate(env, text, response); } return text; }