diff --git a/src/annotate.test.ts b/src/annotate.test.ts
index bf8bb2e..17cb6d6 100644
--- a/src/annotate.test.ts
+++ b/src/annotate.test.ts
@@ -91,3 +91,40 @@ test("should get correct level from annotations", () => {
     ] as any)
   ).toBe(Level.FAILURE);
 });
+
+test("should annotate with correct level", () => {
+  const files = parse(fs.readFileSync("./fixtures/pull.failing.diff", "utf8"));
+  const annotations = annotate(
+    {
+      ...DEFAULT_CONFIGURATION,
+      rules: {
+        whitelist: {
+          regex: DEFAULT_CONFIGURATION.rules.whitelist.regex,
+          level: Level.FAILURE,
+        },
+        master: {
+          regex: DEFAULT_CONFIGURATION.rules.master.regex,
+          level: Level.OFF,
+        },
+      },
+    },
+    files
+  );
+  expect(annotations).toMatchInlineSnapshot(`
+    Array [
+      Object {
+        "annotation_level": "failure",
+        "end_column": 8,
+        "end_line": 2,
+        "message": "
+    Please consider an alternative to \`whitelist\`. 
+    ",
+        "path": "README.md",
+        "raw_details": "/white[_-]*list/gi",
+        "start_column": 0,
+        "start_line": 2,
+        "title": "Match Found",
+      },
+    ]
+  `);
+});
diff --git a/src/annotate.ts b/src/annotate.ts
index 2b6107a..fa020da 100644
--- a/src/annotate.ts
+++ b/src/annotate.ts
@@ -1,3 +1,4 @@
+import { Context, Logger } from "probot";
 /**
  * Copyright 2020 Google LLC
  *
@@ -53,6 +54,10 @@ export const annotate = (
         if (change.isInsert) {
           for (const k in config.rules) {
             for (const pattern of config.rules[k].regex) {
+              if (config.rules[k].level == "off") {
+                continue;
+              }
+
               // @ts-ignore matchAll may not be available
               for (const match of change.content.matchAll(pattern)) {
                 const context: MessageContext = {
@@ -62,8 +67,12 @@ export const annotate = (
                   content: change.content,
                   alternatives: config.rules[k].alternatives || [],
                 };
-                annotations.push({
-                  annotation_level: "warning",
+
+                const annotation = {
+                  annotation_level: config.rules[k].level as
+                    | "notice"
+                    | "warning"
+                    | "failure",
                   end_column: match.index + match[0].length - 1,
                   end_line: (change.lineNumber ||
                     change.newLineNumber) as number,
@@ -76,7 +85,9 @@ export const annotate = (
                   start_line: (change.lineNumber ||
                     change.newLineNumber) as number,
                   title: "Match Found",
-                });
+                };
+
+                annotations.push(annotation);
               }
             }
           }