From e22d7ea496cb272178582f027006cf8780929230 Mon Sep 17 00:00:00 2001 From: hahwul Date: Mon, 20 Nov 2023 23:46:43 +0900 Subject: [PATCH 1/3] Unified endpoint headers by removing header class variable and using param_type in param variable (#152) --- src/models/endpoint.cr | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/models/endpoint.cr b/src/models/endpoint.cr index bb9d8e0e..3a790938 100644 --- a/src/models/endpoint.cr +++ b/src/models/endpoint.cr @@ -4,20 +4,14 @@ require "yaml" struct Endpoint include JSON::Serializable include YAML::Serializable - property url, method, params, headers, protocol + property url, method, params, protocol def initialize(@url : String, @method : String) @params = [] of Param - @headers = [] of Header @protocol = "http" end def initialize(@url : String, @method : String, @params : Array(Param)) - @headers = [] of Header - @protocol = "http" - end - - def initialize(@url : String, @method : String, @params : Array(Param), @headers : Array(Header)) @protocol = "http" end @@ -48,16 +42,9 @@ struct Param include YAML::Serializable property name, value, param_type - def initialize(@name : String, @value : String, @param_type : String) - end -end - -struct Header - include JSON::Serializable - include YAML::Serializable - property name, value + # param_type can be "query", "json", "form", "header", "cookie" - def initialize(@name : String, @value : String) + def initialize(@name : String, @value : String, @param_type : String) end end From bfc5a39e5a1101ff447bf6d79de0fc0c7283470c Mon Sep 17 00:00:00 2001 From: hahwul Date: Tue, 21 Nov 2023 00:25:54 +0900 Subject: [PATCH 2/3] Add cookie type of param, Improve logger & baker (#152) --- .../fixtures/crystal_kemal/src/testapp.cr | 2 ++ .../testers/crystal_kemal_spec.cr | 7 +++++-- .../unit_test/analyzer/analyzer_kemal_spec.cr | 2 +- .../analyzers/analyzer_crystal_kemal.cr | 9 ++++++-- src/models/output_builder.cr | 6 ++++++ src/output_builder/common.cr | 21 ++++++++++++++----- src/output_builder/curl.cr | 4 ++++ src/output_builder/httpie.cr.cr | 6 +++++- 8 files changed, 46 insertions(+), 11 deletions(-) diff --git a/spec/functional_test/fixtures/crystal_kemal/src/testapp.cr b/spec/functional_test/fixtures/crystal_kemal/src/testapp.cr index 607b1842..787f08ac 100644 --- a/spec/functional_test/fixtures/crystal_kemal/src/testapp.cr +++ b/spec/functional_test/fixtures/crystal_kemal/src/testapp.cr @@ -1,10 +1,12 @@ require "kemal" get "/" do + env.request.headers["x-api-key"].as(String) "Hello World!" end post "/query" do + env.request.cookies["my_auth"].as(String) env.params.body["query"].as(String) end diff --git a/spec/functional_test/testers/crystal_kemal_spec.cr b/spec/functional_test/testers/crystal_kemal_spec.cr index 8e5cc7e7..5359e6fe 100644 --- a/spec/functional_test/testers/crystal_kemal_spec.cr +++ b/spec/functional_test/testers/crystal_kemal_spec.cr @@ -1,9 +1,12 @@ require "../func_spec.cr" extected_endpoints = [ - Endpoint.new("/", "GET"), + Endpoint.new("/", "GET", [Param.new("x-api-key", "", "header")]), Endpoint.new("/socket", "GET"), - Endpoint.new("/query", "POST", [Param.new("query", "", "form")]), + Endpoint.new("/query", "POST", [ + Param.new("query", "", "form"), + Param.new("my_auth", "", "cookie"), + ]), ] FunctionalTester.new("fixtures/crystal_kemal/", { diff --git a/spec/unit_test/analyzer/analyzer_kemal_spec.cr b/spec/unit_test/analyzer/analyzer_kemal_spec.cr index e58b23d0..f5f0e2a2 100644 --- a/spec/unit_test/analyzer/analyzer_kemal_spec.cr +++ b/spec/unit_test/analyzer/analyzer_kemal_spec.cr @@ -21,7 +21,7 @@ describe "mapping_to_path" do end it "line_to_param - env.response.headers[]" do - line = "env.response.headers[\"x-token\"]" + line = "env.request.headers[\"x-token\"]" instance.line_to_param(line).name.should eq("x-token") end end diff --git a/src/analyzer/analyzers/analyzer_crystal_kemal.cr b/src/analyzer/analyzers/analyzer_crystal_kemal.cr index f01a2042..f6b5301f 100644 --- a/src/analyzer/analyzers/analyzer_crystal_kemal.cr +++ b/src/analyzer/analyzers/analyzer_crystal_kemal.cr @@ -49,11 +49,16 @@ class AnalyzerCrystalKemal < Analyzer return Param.new(param, "", "form") end - if content.includes? "env.response.headers[" - param = content.split("env.response.headers[")[1].split("]")[0].gsub("\"", "").gsub("'", "") + if content.includes? "env.request.headers[" + param = content.split("env.request.headers[")[1].split("]")[0].gsub("\"", "").gsub("'", "") return Param.new(param, "", "header") end + if content.includes? "env.request.cookies[" + param = content.split("env.request.cookies[")[1].split("]")[0].gsub("\"", "").gsub("'", "") + return Param.new(param, "", "cookie") + end + Param.new("", "", "") end diff --git a/src/models/output_builder.cr b/src/models/output_builder.cr index 4d9bfcee..7c92500d 100644 --- a/src/models/output_builder.cr +++ b/src/models/output_builder.cr @@ -41,6 +41,7 @@ class OutputBuilder final_url = url final_body = "" final_headers = [] of String + final_cookies = [] of String is_json = false first_query = true first_form = true @@ -69,6 +70,10 @@ class OutputBuilder final_headers << "#{param.name}: #{param.value}" end + if param.param_type == "cookie" + final_cookies << "#{param.name}=#{param.value}" + end + if param.param_type == "json" is_json = true end @@ -95,6 +100,7 @@ class OutputBuilder url: final_url, body: final_body, header: final_headers, + cookie: final_cookies, body_type: is_json ? "json" : "form", } end diff --git a/src/output_builder/common.cr b/src/output_builder/common.cr index 4690e886..36abc30a 100644 --- a/src/output_builder/common.cr +++ b/src/output_builder/common.cr @@ -9,18 +9,29 @@ class OutputBuilderCommon < OutputBuilder r_method = endpoint.method.colorize(:light_blue).toggle(@is_color) r_url = baked[:url].colorize(:light_yellow).toggle(@is_color) r_headers = baked[:header].join(" ").colorize(:light_green).toggle(@is_color) - + r_cookies = baked[:cookie].join(";").colorize(:light_green).toggle(@is_color) r_ws = "" + r_buffer = "#{r_method} #{r_url}" + if endpoint.protocol == "ws" - r_ws = "[WEBSOCKET]".colorize(:light_red).toggle(@is_color) + r_ws = "[websocket]".colorize(:light_red).toggle(@is_color) + r_buffer += " #{r_ws}" + end + + if baked[:header].size > 0 + r_buffer += "\n ○ headers: #{r_headers}" + end + + if baked[:cookie].size > 0 + r_buffer += "\n ○ cookies: #{r_cookies}" end if baked[:body] != "" r_body = baked[:body].colorize(:cyan).toggle(@is_color) - ob_puts "#{r_method} #{r_url} #{r_body} #{r_headers} #{r_ws}" - else - ob_puts "#{r_method} #{r_url} #{r_headers} #{r_ws}" + r_buffer += "\n ○ body: #{r_body}" end + + ob_puts r_buffer end end end diff --git a/src/output_builder/curl.cr b/src/output_builder/curl.cr index 48854754..7fad930e 100644 --- a/src/output_builder/curl.cr +++ b/src/output_builder/curl.cr @@ -16,6 +16,10 @@ class OutputBuilderCurl < OutputBuilder baked[:header].each do |header| cmd += " -H \"#{header}\"" end + + baked[:cookie].each do |cookie| + cmd += " --cookie \"#{cookie}\"" + end end ob_puts cmd diff --git a/src/output_builder/httpie.cr.cr b/src/output_builder/httpie.cr.cr index e5bfc502..b440af72 100644 --- a/src/output_builder/httpie.cr.cr +++ b/src/output_builder/httpie.cr.cr @@ -8,13 +8,17 @@ class OutputBuilderHttpie < OutputBuilder cmd = "http #{endpoint.method} #{baked[:url]}" if baked[:body] != "" - cmd += " #{baked[:body]}" + cmd += " \"#{baked[:body]}\"" if baked[:body_type] == "json" cmd += " \"Content-Type:application/json\"" end baked[:header].each do |header| cmd += " \"#{header}\"" end + + baked[:cookie].each do |cookie| + cmd += " \"Cookie: #{cookie}\"" + end end ob_puts cmd From 15dfb97a01e60a06673024d02f8c5f5235da22aa Mon Sep 17 00:00:00 2001 From: ksg97031 Date: Tue, 21 Nov 2023 02:50:52 +0900 Subject: [PATCH 3/3] Corrected cookie parameters properly in Python web analyzers Signed-off-by: ksg97031 --- spec/functional_test/fixtures/python_flask/app.py | 11 +++++++++-- spec/functional_test/testers/python_django_spec.cr | 2 +- spec/functional_test/testers/python_fastapi_spec.cr | 2 +- spec/functional_test/testers/python_flask_spec.cr | 3 ++- src/analyzer/analyzers/analyzer_django.cr | 6 +----- src/analyzer/analyzers/analyzer_fastapi.cr | 4 ---- src/analyzer/analyzers/analyzer_flask.cr | 7 +------ 7 files changed, 15 insertions(+), 20 deletions(-) diff --git a/spec/functional_test/fixtures/python_flask/app.py b/spec/functional_test/fixtures/python_flask/app.py index b1735e92..f6c90f90 100644 --- a/spec/functional_test/fixtures/python_flask/app.py +++ b/spec/functional_test/fixtures/python_flask/app.py @@ -11,7 +11,7 @@ @app.teardown_appcontext def shutdown_session(exception=None): - db_session.remove() + db_session.remove() @app.route('/sign', methods=['GET', 'POST']) def sign_sample(): @@ -28,6 +28,13 @@ def sign_sample(): return render_template('sign.html') +@app.route('/cookie', methods=['GET']) +def cookie_test(): + if request.cookies.get('test') == "y": + return "exist cookie" + + return "no cookie" + @app.route('/login', methods=['POST']) def login_sample(): if request.method == 'POST': @@ -40,7 +47,7 @@ def login_sample(): else: return "Fail" - return render_template('login.html') + return render_template('login.html') @app.route('/create_record', methods=['PUT']) def create_record(): diff --git a/spec/functional_test/testers/python_django_spec.cr b/spec/functional_test/testers/python_django_spec.cr index 63e73e1e..16490d86 100644 --- a/spec/functional_test/testers/python_django_spec.cr +++ b/spec/functional_test/testers/python_django_spec.cr @@ -14,7 +14,7 @@ extected_endpoints = [ Endpoint.new("/links.html", "GET"), Endpoint.new("/upload", "GET", [Param.new("sign", "", "query"), Param.new("sign", "", "query"), Param.new("X_FORWARDED_FOR", "", "header"), Param.new("X_REAL_IP", "", "header")]), Endpoint.new("/upload", "POST", [Param.new("sign", "", "query"), Param.new("X_FORWARDED_FOR", "", "header"), Param.new("X_REAL_IP", "", "header")]), - Endpoint.new("/not_found", "GET", [Param.new("Cookie['app_type']", "", "header")]), + Endpoint.new("/not_found", "GET", [Param.new("app_type", "", "cookie")]), Endpoint.new("/test", "GET", [Param.new("test_param", "", "form")]), Endpoint.new("/test", "POST", [Param.new("test_param", "", "form")]), Endpoint.new("/test", "PUT", [Param.new("test_param", "", "form")]), diff --git a/spec/functional_test/testers/python_fastapi_spec.cr b/spec/functional_test/testers/python_fastapi_spec.cr index b9ac8add..c03a28dd 100644 --- a/spec/functional_test/testers/python_fastapi_spec.cr +++ b/spec/functional_test/testers/python_fastapi_spec.cr @@ -4,7 +4,7 @@ extected_endpoints = [ Endpoint.new("/query/param-required/int", "GET", [Param.new("query", "", "query")]), Endpoint.new("/items/{item_id}", "PUT", [Param.new("name", "", "form"), Param.new("size", "", "form")]), Endpoint.new("/hidden_header", "GET", [Param.new("hidden_header", "", "header")]), - Endpoint.new("/cookie_examples/", "GET", [Param.new("Cookie['data']", "", "header")]), + Endpoint.new("/cookie_examples/", "GET", [Param.new("data", "", "cookie")]), Endpoint.new("/dummypath", "POST", [Param.new("dummy", "", "json")]), Endpoint.new("/main", "GET"), ] diff --git a/spec/functional_test/testers/python_flask_spec.cr b/spec/functional_test/testers/python_flask_spec.cr index 9684c386..7aa9e2bb 100644 --- a/spec/functional_test/testers/python_flask_spec.cr +++ b/spec/functional_test/testers/python_flask_spec.cr @@ -3,6 +3,7 @@ require "../func_spec.cr" extected_endpoints = [ Endpoint.new("/sign", "GET"), Endpoint.new("/sign", "POST", [Param.new("username", "", "form"), Param.new("password", "", "form")]), + Endpoint.new("/cookie", "GET", [Param.new("test", "", "cookie")]), Endpoint.new("/login", "POST", [Param.new("username", "", "form"), Param.new("password", "", "form")]), Endpoint.new("/create_record", "PUT"), Endpoint.new("/delete_record", "DELETE", [Param.new("name", "", "json")]), @@ -12,5 +13,5 @@ extected_endpoints = [ FunctionalTester.new("fixtures/python_flask/", { :techs => 1, - :endpoints => 7, + :endpoints => 8, }, extected_endpoints).test_all diff --git a/src/analyzer/analyzers/analyzer_django.cr b/src/analyzer/analyzers/analyzer_django.cr index 798bb37e..e6ecc4b7 100644 --- a/src/analyzer/analyzers/analyzer_django.cr +++ b/src/analyzer/analyzers/analyzer_django.cr @@ -10,7 +10,7 @@ class AnalyzerDjango < AnalyzerPython REQUEST_PARAM_FIELD_MAP = { "GET" => {["GET"], "query"}, "POST" => {["POST"], "form"}, - "COOKIES" => {nil, "header"}, + "COOKIES" => {nil, "cookie"}, "META" => {nil, "header"}, "data" => {["POST", "PUT", "PATCH"], "form"}, } @@ -293,10 +293,6 @@ class AnalyzerDjango < AnalyzerPython if param_name.starts_with? "HTTP_" param_name = param_name[5..] end - elsif noir_param_type == "header" - if field_name == "COOKIES" - param_name = "Cookie['#{param_name}']" - end end # If it receives a specific parameter, it is considered to allow the method. diff --git a/src/analyzer/analyzers/analyzer_fastapi.cr b/src/analyzer/analyzers/analyzer_fastapi.cr index 76c615d5..f978a884 100644 --- a/src/analyzer/analyzers/analyzer_fastapi.cr +++ b/src/analyzer/analyzers/analyzer_fastapi.cr @@ -100,10 +100,6 @@ class AnalyzerFastAPI < AnalyzerPython # Get param type by default value first if param.default.size != 0 param_type = infer_parameter_type(param.default) - if param_type == "cookie" - param_type = "header" - param.name = "Cookie['#{param.name}']" - end end # Get param type by type if not found diff --git a/src/analyzer/analyzers/analyzer_flask.cr b/src/analyzer/analyzers/analyzer_flask.cr index 27ab75ae..5fd99667 100644 --- a/src/analyzer/analyzers/analyzer_flask.cr +++ b/src/analyzer/analyzers/analyzer_flask.cr @@ -11,7 +11,7 @@ class AnalyzerFlask < AnalyzerPython "files" => {["POST", "PUT", "PATCH", "DELETE"], "form"}, "values" => {["GET", "POST", "PUT", "PATCH", "DELETE"], "query"}, "json" => {["POST", "PUT", "PATCH", "DELETE"], "json"}, - "cookie" => {nil, "header"}, + "cookie" => {nil, "cookie"}, "headers" => {nil, "header"}, } @@ -200,11 +200,6 @@ class AnalyzerFlask < AnalyzerPython matches.each do |parameter_match| next if parameter_match.size != 2 param_name = parameter_match[1] - if noir_param_type == "header" - if field_name == "cookie" - param_name = "Cookie['#{param_name}']" - end - end suspicious_params << Param.new(param_name, "", noir_param_type) end