refactor: Refactor analyzers and add path details to endpoints (#192)

- Modified various analyzer classes to include a `Details` object in each `Endpoint` for better endpoint details tracking. - Updated the output builder to properly display multiple file paths and line numbers in the output. Signed-off-by: ksg97031 <[email protected]>
owasp-noir · Dec 28, 2023 · 28e6eee · 28e6eee
1 parent 745652b
commit 28e6eee
Show file tree

Hide file tree

Showing 27 changed files with 161 additions and 90 deletions.
diff --git a/src/analyzer/analyzers/analyzer_armeria.cr b/src/analyzer/analyzers/analyzer_armeria.cr
@@ -12,6 +12,8 @@ class AnalyzerArmeria < Analyzer
         next if File.directory?(path)
 
         if File.exists?(path) && (path.ends_with?(".java") || path.ends_with?(".kt"))
+          details = Details.new(EndpointPath.new(path))
+
           content = File.read(path, encoding: "utf-8", invalid: :skip)
           content.scan(REGEX_SERVER_CODE_BLOCK) do |server_codeblcok_match|
             server_codeblock = server_codeblcok_match[0]
@@ -30,7 +32,7 @@ class AnalyzerArmeria < Analyzer
               endpoint = split_params[endpoint_param_index].strip
 
               endpoint = endpoint[1..-2]
-              @result << Endpoint.new("#{endpoint}", "GET")
+              @result << Endpoint.new("#{endpoint}", "GET", details)
             end
 
             server_codeblock.scan(REGEX_ROUTE_CODE) do |route_code_match|
@@ -47,7 +49,7 @@ class AnalyzerArmeria < Analyzer
               next if endpoint[0] != '"'
 
               endpoint = endpoint[1..-2]
-              @result << Endpoint.new("#{endpoint}", method)
+              @result << Endpoint.new("#{endpoint}", method, details)
             end
           end
         end

diff --git a/src/analyzer/analyzers/analyzer_crystal_kemal.cr b/src/analyzer/analyzers/analyzer_crystal_kemal.cr
@@ -9,9 +9,11 @@ class AnalyzerCrystalKemal < Analyzer
         if File.exists?(path) && File.extname(path) == ".cr" && !path.includes?("lib")
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
               endpoint = line_to_endpoint(line)
               if endpoint.method != ""
+                details = Details.new(EndpointPath.new(path, index + 1))
+                endpoint.set_details(details)
                 result << endpoint
                 last_endpoint = endpoint
               end

diff --git a/src/analyzer/analyzers/analyzer_crystal_lucky.cr b/src/analyzer/analyzers/analyzer_crystal_lucky.cr
@@ -21,9 +21,11 @@ class AnalyzerCrystalLucky < Analyzer
         if File.exists?(path) && File.extname(path) == ".cr" && !path.includes?("lib")
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
               endpoint = line_to_endpoint(line)
               if endpoint.method != ""
+                details = Details.new(EndpointPath.new(path, index + 1))
+                endpoint.set_details(details)
                 result << endpoint
                 last_endpoint = endpoint
               end

diff --git a/src/analyzer/analyzers/analyzer_cs_aspnet_mvc.cr b/src/analyzer/analyzers/analyzer_cs_aspnet_mvc.cr
@@ -11,7 +11,7 @@ class AnalyzerCsAspNetMvc < Analyzer
         maproute_check = false
         maproute_buffer = ""
 
-        file.each_line do |line|
+        file.each_line.with_index do |line, index|
           if line.includes? ".MapRoute("
             maproute_check = true
             maproute_buffer = line
@@ -25,7 +25,8 @@ class AnalyzerCsAspNetMvc < Analyzer
               buffer.split(",").each do |item|
                 if item.includes? "url:"
                   url = item.gsub(/url:/, "").gsub(/"/, "")
-                  @result << Endpoint.new("/#{url}", "GET")
+                  details = Details.new(EndpointPath.new(route_config_file, index + 1))
+                  @result << Endpoint.new("/#{url}", "GET", details)
                 end
               end
 

diff --git a/src/analyzer/analyzers/analyzer_django.cr b/src/analyzer/analyzers/analyzer_django.cr
@@ -112,7 +112,6 @@ class AnalyzerDjango < AnalyzerPython
       route = route.gsub(/^\^/, "").gsub(/\$$/, "")
       view = route_match[2].split(",")[0]
       url = "/#{django_urls.prefix}/#{route}".gsub(/\/+/, "/")
-
       new_django_urls = nil
       view.scan(REGEX_INCLUDE_URLS) do |include_pattern_match|
         # Detect new url configs
@@ -121,15 +120,18 @@ class AnalyzerDjango < AnalyzerPython
 
         if File.exists?(new_route_path)
           new_django_urls = DjangoUrls.new("#{django_urls.prefix}#{route}", new_route_path, django_urls.basepath)
+          details = Details.new(EndpointPath.new(new_route_path))
           get_endpoints(new_django_urls).each do |endpoint|
+            endpoint.set_details(details)
             endpoints << endpoint
           end
         end
       end
       next if new_django_urls != nil
 
+      details = Details.new(EndpointPath.new(django_urls.filepath))
       if view == ""
-        endpoints << Endpoint.new(url, "GET")
+        endpoints << Endpoint.new(url, "GET", details)
       else
         dotted_as_names_split = view.split(".")
 
@@ -149,12 +151,13 @@ class AnalyzerDjango < AnalyzerPython
 
         if filepath != ""
           get_endpoint_from_files(url, filepath, function_or_class_name).each do |endpoint|
+            endpoint.set_details(details)
             endpoints << endpoint
           end
         else
           # By default, Django allows requests with methods other than GET as well
           # Prevent this flow, we need to improve trace code of 'get_endpoint_from_files()
-          endpoints << Endpoint.new(url, "GET")
+          endpoints << Endpoint.new(url, "GET", details)
         end
       end
     end

diff --git a/src/analyzer/analyzers/analyzer_elixir_phoenix.cr b/src/analyzer/analyzers/analyzer_elixir_phoenix.cr
@@ -8,13 +8,14 @@ class AnalyzerElixirPhoenix < Analyzer
         next if File.directory?(path)
         if File.exists?(path) && File.extname(path) == ".ex"
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
-            last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
-              endpoint = line_to_endpoint(line)
-              if endpoint.method != ""
-                @result << endpoint
-                last_endpoint = endpoint
-                _ = last_endpoint
+            file.each_line.with_index do |line, index|
+              endpoints = line_to_endpoint(line)
+              endpoints.each do |endpoint|
+                if endpoint.method != ""
+                  details = Details.new(EndpointPath.new(path, index + 1))
+                  endpoint.set_details(details)
+                  @result << endpoint
+                end
               end
             end
           end
@@ -27,34 +28,36 @@ class AnalyzerElixirPhoenix < Analyzer
     @result
   end
 
-  def line_to_endpoint(line : String) : Endpoint
+  def line_to_endpoint(line : String) : Array(Endpoint)
+    endpoints = Array(Endpoint).new
+
     line.scan(/get\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
-      @result << Endpoint.new("#{match[1]}", "GET")
+      endpoints << Endpoint.new("#{match[1]}", "GET")
     end
 
     line.scan(/post\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
-      @result << Endpoint.new("#{match[1]}", "POST")
+      endpoints << Endpoint.new("#{match[1]}", "POST")
     end
 
     line.scan(/patch\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
-      @result << Endpoint.new("#{match[1]}", "PATCH")
+      endpoints << Endpoint.new("#{match[1]}", "PATCH")
     end
 
     line.scan(/put\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
-      @result << Endpoint.new("#{match[1]}", "PUT")
+      endpoints << Endpoint.new("#{match[1]}", "PUT")
     end
 
     line.scan(/delete\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
-      @result << Endpoint.new("#{match[1]}", "DELETE")
+      endpoints << Endpoint.new("#{match[1]}", "DELETE")
     end
 
     line.scan(/socket\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
       tmp = Endpoint.new("#{match[1]}", "GET")
       tmp.set_protocol("ws")
-      @result << tmp
+      endpoints << tmp
     end
 
-    Endpoint.new("", "")
+    endpoints
   end
 end
 

diff --git a/src/analyzer/analyzers/analyzer_express.cr b/src/analyzer/analyzers/analyzer_express.cr
@@ -9,9 +9,11 @@ class AnalyzerExpress < Analyzer
         if File.exists?(path)
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
               endpoint = line_to_endpoint(line)
               if endpoint.method != ""
+                details = Details.new(EndpointPath.new(path, index + 1))
+                endpoint.set_details(details)
                 result << endpoint
                 last_endpoint = endpoint
               end

diff --git a/src/analyzer/analyzers/analyzer_fastapi.cr b/src/analyzer/analyzers/analyzer_fastapi.cr
@@ -167,7 +167,8 @@ class AnalyzerFastAPI < AnalyzerPython
                   end
                 end
 
-                result << Endpoint.new(router_class.join(http_route_path), http_method_name, params)
+                details = Details.new(EndpointPath.new(path, index + 1))
+                result << Endpoint.new(router_class.join(http_route_path), http_method_name, params, details)
               end
             end
           end

diff --git a/src/analyzer/analyzers/analyzer_flask.cr b/src/analyzer/analyzers/analyzer_flask.cr
@@ -30,7 +30,7 @@ class AnalyzerFlask < AnalyzerPython
       Dir.glob("#{base_path}/**/*.py") do |path|
         next if File.directory?(path)
         File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
-          file.each_line do |line|
+          file.each_line.with_index do |line, index|
             # [TODO] We should be cautious about instance replace with other variable
             match = line.match /(#{REGEX_PYTHON_VARIABLE_NAME})\s*=\s*Flask\s*\(/
             if !match.nil?
@@ -77,7 +77,8 @@ class AnalyzerFlask < AnalyzerPython
                     if !route_url.starts_with? "/"
                       route_url = "/#{route_url}"
                     end
-                    result << Endpoint.new(route_url, "GET")
+                    details = Details.new(EndpointPath.new(path, index + 1))
+                    result << Endpoint.new(route_url, "GET", details)
                   end
                 end
               end
@@ -123,6 +124,8 @@ class AnalyzerFlask < AnalyzerPython
                 codeblock_lines = codeblock_lines[1..]
 
                 get_endpoints(route_path, extra_params, codeblock_lines, prefix).each do |endpoint|
+                  details = Details.new(EndpointPath.new(path, line_index + 1))
+                  endpoint.set_details(details)
                   result << endpoint
                 end
               end

diff --git a/src/analyzer/analyzers/analyzer_go_echo.cr b/src/analyzer/analyzers/analyzer_go_echo.cr
@@ -10,11 +10,12 @@ class AnalyzerGoEcho < Analyzer
         if File.exists?(path) && File.extname(path) == ".go"
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
+              details = Details.new(EndpointPath.new(path, index + 1))
               if line.includes?(".GET(") || line.includes?(".POST(") || line.includes?(".PUT(") || line.includes?(".DELETE(")
                 get_route_path(line).tap do |route_path|
                   if route_path.size > 0
-                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0])
+                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0], details)
                     result << new_endpoint
                     last_endpoint = new_endpoint
                   end
@@ -69,7 +70,8 @@ class AnalyzerGoEcho < Analyzer
             p_dir["static_path"] = p_dir["static_path"][0..-2]
           end
 
-          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET")
+          details = Details.new(EndpointPath.new(path))
+          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET", details)
         end
       end
     end

diff --git a/src/analyzer/analyzers/analyzer_go_fiber.cr b/src/analyzer/analyzers/analyzer_go_fiber.cr
@@ -10,11 +10,12 @@ class AnalyzerGoFiber < Analyzer
         if File.exists?(path) && File.extname(path) == ".go"
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
+              details = Details.new(EndpointPath.new(path, index + 1))
               if line.includes?(".Get(") || line.includes?(".Post(") || line.includes?(".Put(") || line.includes?(".Delete(")
                 get_route_path(line).tap do |route_path|
                   if route_path.size > 0
-                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0].upcase)
+                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0].upcase, details)
                     if line.includes?("websocket.New(")
                       new_endpoint.set_protocol("ws")
                     end
@@ -80,7 +81,8 @@ class AnalyzerGoFiber < Analyzer
             p_dir["static_path"] = p_dir["static_path"][0..-2]
           end
 
-          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET")
+          details = Details.new(EndpointPath.new(path))
+          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET", details)
         end
       end
     end

diff --git a/src/analyzer/analyzers/analyzer_go_gin.cr b/src/analyzer/analyzers/analyzer_go_gin.cr
@@ -10,11 +10,12 @@ class AnalyzerGoGin < Analyzer
         if File.exists?(path) && File.extname(path) == ".go"
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
+              details = Details.new(EndpointPath.new(path, index + 1))
               if line.includes?(".GET(") || line.includes?(".POST(") || line.includes?(".PUT(") || line.includes?(".DELETE(")
                 get_route_path(line).tap do |route_path|
                   if route_path.size > 0
-                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0])
+                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0], details)
                     result << new_endpoint
                     last_endpoint = new_endpoint
                   end
@@ -63,7 +64,8 @@ class AnalyzerGoGin < Analyzer
             p_dir["static_path"] = p_dir["static_path"][0..-2]
           end
 
-          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET")
+          details = Details.new(EndpointPath.new(path))
+          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET", details)
         end
       end
     end

diff --git a/src/analyzer/analyzers/analyzer_jsp.cr b/src/analyzer/analyzers/analyzer_jsp.cr
@@ -33,7 +33,8 @@ class AnalyzerJsp < Analyzer
             rescue
               next
             end
-            result << Endpoint.new("/#{relative_path}", "GET", params_query)
+            details = Details.new(EndpointPath.new(path))
+            result << Endpoint.new("/#{relative_path}", "GET", params_query, details)
           end
         end
       end

diff --git a/src/analyzer/analyzers/analyzer_oas2.cr b/src/analyzer/analyzers/analyzer_oas2.cr
@@ -9,6 +9,7 @@ class AnalyzerOAS2 < Analyzer
     if swagger_jsons.is_a?(Array(String))
       swagger_jsons.each do |swagger_json|
         if File.exists?(swagger_json)
+          details = Details.new(EndpointPath.new(swagger_json))
           content = File.read(swagger_json, encoding: "utf-8", invalid: :skip)
           json_obj = JSON.parse(content)
           base_path = ""
@@ -44,9 +45,9 @@ class AnalyzerOAS2 < Analyzer
                       params << param
                     end
                   end
-                  @result << Endpoint.new(base_path + path, method.upcase, params)
+                  @result << Endpoint.new(base_path + path, method.upcase, params, details)
                 else
-                  @result << Endpoint.new(base_path + path, method.upcase)
+                  @result << Endpoint.new(base_path + path, method.upcase, details)
                 end
               rescue e
                 @logger.debug "Exception of #{swagger_json}/paths/path/method"
@@ -67,6 +68,7 @@ class AnalyzerOAS2 < Analyzer
     if swagger_yamls.is_a?(Array(String))
       swagger_yamls.each do |swagger_yaml|
         if File.exists?(swagger_yaml)
+          details = Details.new(EndpointPath.new(swagger_yaml))
           content = File.read(swagger_yaml, encoding: "utf-8", invalid: :skip)
           yaml_obj = YAML.parse(content)
           base_path = ""
@@ -102,9 +104,9 @@ class AnalyzerOAS2 < Analyzer
                       params << param
                     end
                   end
-                  @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, params)
+                  @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, params, details)
                 else
-                  @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase)
+                  @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, details)
                 end
               rescue e
                 @logger.debug "Exception of #{swagger_yaml}/paths/path/method"
-Original file line number
+Diff line change
@@ Expand Up / @@ -167,7 +167,8 @@ class AnalyzerFastAPI < AnalyzerPython @@
                       end
                     end
-                    result << Endpoint.new(router_class.join(http_route_path), http_method_name, params)
+                    details = Details.new(EndpointPath.new(path, index + 1))
+                    result << Endpoint.new(router_class.join(http_route_path), http_method_name, params, details)
                   end
                 end
               end
@@ Expand Down @@