From 166b296124ec69215c51852b6112d71556dac099 Mon Sep 17 00:00:00 2001 From: Shashwat Jaiswal Date: Wed, 24 Aug 2022 11:49:30 +0530 Subject: [PATCH] nginx support flow layout files added Signed-off-by: Shashwat Jaiswal --- .../src/main/conf/xml/FlowLayoutNginx.xml | 181 ++++++++++++++++++ .../src/main/conf/xml/FlowLayoutNginx.xsd | 175 +++++++++++++++++ .../ext/os/parser/TestNginxLogLineParser.java | 97 +++++----- .../ext/os/parser/TestNginxLogParserBase.java | 1 - 4 files changed, 409 insertions(+), 45 deletions(-) create mode 100644 ade-assembly/src/main/conf/xml/FlowLayoutNginx.xml create mode 100644 ade-assembly/src/main/conf/xml/FlowLayoutNginx.xsd diff --git a/ade-assembly/src/main/conf/xml/FlowLayoutNginx.xml b/ade-assembly/src/main/conf/xml/FlowLayoutNginx.xml new file mode 100644 index 0000000..24c314f --- /dev/null +++ b/ade-assembly/src/main/conf/xml/FlowLayoutNginx.xml @@ -0,0 +1,181 @@ + + + + + + ConsecutiveTimeFramer + + + + + ConsecutiveTimeFramer + + + ContinuousTimeFramer + + + + + ConsecutiveTimeFramer + + + ContinuousTimeFramer + + + + + ConsecutiveTimeFramer + + + ContinuousTimeFramer + + + + + ConsecutiveTimeFramer + + + ContinuousTimeFramer + + + + + ConsecutiveTimeFramer + + + ContinuousTimeFramer + + + + + + + ConsecutiveTimeFramer + + + ContinuousTimeFramer + + + + + ConsecutiveTimeFramer + + + ContinuousTimeFramer + + + + + + + + oneMinuteTrain + 1 + oneMinuteTrain + + CriticalWordCountReporter + + + ClusteringContextScore + + + + + + + + + + + + + + + + FullBernoulliClusterAwareScore + + + NGINX01 + + + LastSeenLoggingScorerContinuous + + + LastSeenScorer + NGINX11 + + + BestOfTwoScorer + + + NGINX02 + NGINX12 + + + SeverityScore + + + LogNormalScore + + + + + AdeWeightedMessageAnomalyScorerLogNormal + + + + + NGINX01 + NGINX31 + NGINX21 + NGINX41 + + + AdeAnomalyIntervalScorer + NGINX51 + + + org.openmainframe.ade.ext.output.ExtendedAnalyzedIntervalDbStorer + + + org.openmainframe.ade.ext.output.ExtJaxbAnalyzedPeriodV2XmlStorer + + + + + + org.openmainframe.ade.ext.output.ExtAnalyzedIntervalV2FullXmlStorer + + + + + org.openmainframe.ade.ext.output.ExtJaxbAnalyzedIntervalV2XmlStorer + + + + NGINX51 + NGINX52 + + \ No newline at end of file diff --git a/ade-assembly/src/main/conf/xml/FlowLayoutNginx.xsd b/ade-assembly/src/main/conf/xml/FlowLayoutNginx.xsd new file mode 100644 index 0000000..aac5564 --- /dev/null +++ b/ade-assembly/src/main/conf/xml/FlowLayoutNginx.xsd @@ -0,0 +1,175 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ade-ext/src/test/java/org/openmainframe/ade/ext/os/parser/TestNginxLogLineParser.java b/ade-ext/src/test/java/org/openmainframe/ade/ext/os/parser/TestNginxLogLineParser.java index 887440e..528e4c0 100644 --- a/ade-ext/src/test/java/org/openmainframe/ade/ext/os/parser/TestNginxLogLineParser.java +++ b/ade-ext/src/test/java/org/openmainframe/ade/ext/os/parser/TestNginxLogLineParser.java @@ -31,6 +31,7 @@ This file is part of Anomaly Detection Engine for Linux Logs (ADE). import java.util.regex.Pattern; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.openmainframe.ade.ext.os.parser.NginxLogParserBase.NGINX_LOG; public class TestNginxLogLineParser { @@ -55,50 +56,58 @@ public void testWithRealLog() { final Pattern pattern = Pattern.compile(NGINX_LOG); final String line = "93.180.71.3 - - [17/May/2015:08:05:32 +0000] \"GET /downloads/product_1 HTTP/1.1\" 304 0 \"-\" \"Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)\""; assertEquals(true, slp.parseLine(pattern, 1,2,3,4,5,6,line)); - System.out.println(slp); } -// @Test -// public void testParseLineWithMatchingPattern() { -// Pattern pattern = Pattern.compile("^([^:]+):.*COMMAND=(.*)$"); -// assertEquals("Pattern matches for all parameters ",true, slp.parseLine(pattern,1,2,2,2,2,2,"(username):.COMMAND=nub")); -// } - -// @Test -// public void testParseLineWith255CharacterHostname() { -// Pattern pattern = Pattern.compile("^([^:]+):.*COMMAND=(.*)$"); -// assertEquals("Pattern matches but hostname has over 255 chars ",true, slp.parseLine(pattern,1,1,1,1,1,1,longString + ":.COMMAND=nub")); -// } - -// @Test -// public void testParseLineWith255CharacterHostnameSecondTime() { -// Pattern pattern = Pattern.compile("^([^:]+):.*COMMAND=(.*)$"); -// slp.parseLine(pattern,1,1,1,1,1,1,longString + ":.COMMAND=nub"); -// -// assertEquals("Hostname over 255 characters but we go through parseLine twice to skip the logging " -// ,true,slp.parseLine(pattern,1,1,1,1,1,1,longString + ":.COMMAND=nub")); -// } - -// @Test -// public void testGettersGetCorrectInfoAfterRunningParseLine() { -// Pattern pattern = Pattern.compile("^([^:]+):.*COMMAND=(.*)$"); -// slp.parseLine(pattern,0,1,0,1,0,1,"(username):.COMMAND=nub"); -// -// assertEquals("The message time is thee",null,slp.getMsgTime()); -// assertEquals("The source is in the first matched group third param ","(username)",slp.getRemoteAddress()); -// -// slp.parseLine(pattern,0,0,2,2,2,2,"(PID!):.COMMAND=msgBody"); -// assertEquals("The messsage body is in second group and 6th param","msgBody",slp.getRequest()); -// } - -// @Test -// public void testToString() { -// Pattern pattern = Pattern.compile("^([^:]+):.*COMMAND=(.*)$"); -// slp.parseLine(pattern,2,2,2,2,2,2,"(username):.COMMAND=nub"); -// assertEquals("Testing to String works correctly " -// , "timestamp=(null) " -// + "hostname=(nub) " -// + "msg=(nub)" -// ,slp.toString()); -// } + @Test + public void testParseLineWithMatchingPattern() { + final Pattern pattern = Pattern.compile(NGINX_LOG); + final String line = "address - - [17/May/2015:08:05:32 +0000] \"GET\" 0 0 \"-\" \"-\""; + assertEquals("Pattern matches for all parameters ",true, slp.parseLine(pattern,1,2,3,4,5,6,line)); + } + + @Test + public void testParseLineWith255CharacterHostname() { + final Pattern pattern = Pattern.compile(NGINX_LOG); + final String line = "address - - [17/May/2015:08:05:32 +0000] \"GET\" 0 0 \"-\" \"-\""; + assertTrue("Pattern matches but hostname has over 255 chars ", slp.parseLine(pattern, 1, 2, 3, 4, 5, 6, longString + line)); + } + + @Test + public void testParseLineWith255CharacterHostnameSecondTime() { + final Pattern pattern = Pattern.compile(NGINX_LOG); + final String line = "address - - [17/May/2015:08:05:32 +0000] \"GET\" 0 0 \"-\" \"-\""; + slp.parseLine(pattern,1,2,3,4,5,6,longString + line); + + assertEquals("Hostname over 255 characters but we go through parseLine twice to skip the logging " + ,true,slp.parseLine(pattern,1,2,3,4,5,6,longString + line)); + } + + @Test + public void testGettersGetCorrectInfoAfterRunningParseLine() { + final Pattern pattern = Pattern.compile(NGINX_LOG); + final String line = "address - - [17/May/2015:08:05:32 +0000] \"GET\" 0 0 \"-\" \"-\""; + slp.parseLine(pattern,1,2,3,4,5,6,line); + + assertEquals(null, slp.getMsgTime()); + assertEquals("address",slp.getRemoteAddress()); + assertEquals("GET",slp.getRequest()); + assertEquals("-", slp.getRemoteUser()); + assertEquals(0, slp.getBytes()); + assertEquals(0, slp.getStatus()); + } + + @Test + public void testToString() { + final Pattern pattern = Pattern.compile(NGINX_LOG); + final String line = "nub - nub [17/May/2015:08:05:32 +0000] \"nub\" 0 0 \"-\" \"nub\""; + slp.parseLine(pattern,1,2,3,4,5,6,line); + assertEquals("Testing to String works correctly " + , "timestamp=(null) " + + "remote_address=(nub) " + + "remote_user=(nub) " + + "request=(nub) " + + "status=(0) " + + "bytes=(0)" + ,slp.toString()); + } } diff --git a/ade-ext/src/test/java/org/openmainframe/ade/ext/os/parser/TestNginxLogParserBase.java b/ade-ext/src/test/java/org/openmainframe/ade/ext/os/parser/TestNginxLogParserBase.java index 8735fef..43a76b4 100644 --- a/ade-ext/src/test/java/org/openmainframe/ade/ext/os/parser/TestNginxLogParserBase.java +++ b/ade-ext/src/test/java/org/openmainframe/ade/ext/os/parser/TestNginxLogParserBase.java @@ -46,7 +46,6 @@ This file is part of Anomaly Detection Engine for Linux Logs (ADE). public class TestNginxLogParserBase { Ade ade; -// @Before public void setup() throws AdeException{ ade = mock(Ade.class, RETURNS_DEEP_STUBS); when(ade.getConfigProperties().database().getDatabaseDriver()).thenReturn("derby");