Merge pull request #14 from RasaHQ/facebook-1faab00

Sync with facebook/duckling 1faab00
RasaHQ · Apr 21, 2022 · 5c2c606 · 5c2c606
2 parents 970e0f7 + b431ece
commit 5c2c606
Show file tree

Hide file tree

Showing 97 changed files with 8,181 additions and 6,563 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macOS-latest, windows-latest]
+        os: [ubuntu-latest]
         stack: ["2.5.1"]
         ghc: ["8.6.5", "8.8.4", "8.10.3", "9.0.1"]
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,7 @@
 FROM haskell:8-buster AS builder
 
 RUN apt-get update -qq && \
-  apt-get install -qq -y libssl-dev libpcre3 libpcre3-dev build-essential --fix-missing --no-install-recommends && \
+  apt-get install -qq -y libssl-dev libpcre3 libpcre3-dev build-essential pkg-config --fix-missing --no-install-recommends && \
   apt-get clean && \
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 

diff --git a/Duckling/AmountOfMoney/ZH/Rules.hs b/Duckling/AmountOfMoney/ZH/Rules.hs
@@ -226,7 +226,7 @@ ruleOneDollarAnd = Rule
   { name = "one dollar and x dimes (short form)"
   , pattern =
     [ regex "個"
-    , numberBetween 1 10
+    , Predicate $ numberBetween 1 10
     ]
   , prod = \case
       (_:Token Numeral NumeralData{TNumeral.value = v}:_) ->

diff --git a/Duckling/Duration/DE/Rules.hs b/Duckling/Duration/DE/Rules.hs
@@ -19,7 +19,7 @@ import Duckling.Regex.Types
 import Control.Monad (join)
 import qualified Data.Text as Text
 import Prelude
-import Duckling.Numeral.Helpers (parseInteger)
+import Duckling.Numeral.Helpers (numberBetween, parseInteger)
 import Duckling.Duration.Types (DurationData(..))
 import qualified Duckling.Duration.Types as TDuration
 import Data.String
@@ -216,7 +216,7 @@ ruleHoursAndMinutes = Rule
   , pattern =
     [ Predicate isNatural
     , regex "(ein(en?) )?stunden?( und)?"
-    , Predicate isNatural
+    , Predicate $ and . sequence [isNatural, numberBetween 1 60]
     , Predicate $ isGrain TG.Minute
     ]
   , prod = \case

diff --git a/Duckling/Duration/EN/Rules.hs b/Duckling/Duration/EN/Rules.hs
@@ -20,7 +20,7 @@ import qualified Data.Text as Text
 import Duckling.Dimensions.Types
 import Duckling.Duration.Helpers
 import Duckling.Duration.Types (DurationData(..))
-import Duckling.Numeral.Helpers (parseInteger)
+import Duckling.Numeral.Helpers (numberBetween, parseInteger)
 import Duckling.Numeral.Types (NumeralData(..))
 import Duckling.Regex.Types
 import Duckling.Types
@@ -180,7 +180,7 @@ ruleDurationHoursAndMinutes = Rule
   , pattern =
     [ Predicate isNatural
     , regex "hours?( and)?"
-    , Predicate isNatural
+    , Predicate $ and . sequence [isNatural, numberBetween 1 60]
     ]
   , prod = \case
       (Token Numeral h:

diff --git a/Duckling/Numeral/AF/Rules.hs b/Duckling/Numeral/AF/Rules.hs
@@ -124,7 +124,7 @@ ruleCompositeTens :: Rule
 ruleCompositeTens = Rule
   { name = "integer ([3-9][1-9])"
   , pattern =
-    [ numberBetween 1 10
+    [ Predicate $ numberBetween 1 10
     , regex "en"
     , oneOf [20, 30..90]
     ]

diff --git a/Duckling/Numeral/AR/Rules.hs b/Duckling/Numeral/AR/Rules.hs
@@ -42,7 +42,7 @@ ruleInteger23 = Rule
   , pattern =
     [ oneOf [100, 200 .. 900]
     , regex "و"
-    , numberBetween 1 100
+    , Predicate $ numberBetween 1 100
     ]
   , prod = \tokens -> case tokens of
       (Token Numeral NumeralData{TNumeral.value = v1}:
@@ -86,7 +86,7 @@ ruleInteger22 :: Rule
 ruleInteger22 = Rule
   { name = "integer 21..99"
   , pattern =
-    [ numberBetween 1 10
+    [ Predicate $ numberBetween 1 10
     , regex "و"
     , oneOf [20, 30 .. 90]
     ]
@@ -102,7 +102,7 @@ ruleInteger21 :: Rule
 ruleInteger21 = Rule
   { name = "integer (13..19)"
   , pattern =
-    [ numberBetween 3 10
+    [ Predicate $ numberBetween 3 10
     , numberWith TNumeral.value (== 10)
     ]
   , prod = \tokens -> case tokens of

diff --git a/Duckling/Numeral/BG/Rules.hs b/Duckling/Numeral/BG/Rules.hs
@@ -111,7 +111,7 @@ ruleCompositeTens = Rule
   , pattern =
     [ oneOf [20, 30..90]
     , regex "и"
-    , numberBetween 1 10
+    , Predicate $ numberBetween 1 10
     ]
   , prod = \tokens -> case tokens of
       (Token Numeral NumeralData{TNumeral.value = tens}:
@@ -147,7 +147,7 @@ ruleCompositeHundreds = Rule
   { name = "integer 101..999"
   , pattern =
     [ oneOf [200, 300..900]
-    , numberBetween 1 100
+    , Predicate $ numberBetween 1 100
     ]
   , prod = \tokens -> case tokens of
       (Token Numeral NumeralData{TNumeral.value = hundreds}:

diff --git a/Duckling/Numeral/CA/Rules.hs b/Duckling/Numeral/CA/Rules.hs
@@ -140,7 +140,10 @@ ruleHigherTensWithOnes :: Rule
 ruleHigherTensWithOnes = Rule
   { name = "number (31..39 41..49 51..59 61..69 71..79 81..89 91..99)"
   , pattern =
-      [oneOf [30, 40, 50, 60, 70, 80, 90], regex "-", numberBetween 1 9]
+      [ oneOf [30, 40, 50, 60, 70, 80, 90]
+      , regex "-"
+      , Predicate $ numberBetween 1 9
+      ]
   , prod = \case
       (Token Numeral NumeralData{TNumeral.value = v1}:
        _:
@@ -185,7 +188,10 @@ ruleTwenties :: Rule
 ruleTwenties = Rule
   { name = "number (21..29)"
   , pattern =
-      [oneOf [20], regex "(-i-| i )", numberBetween 1 10]
+      [ oneOf [20]
+      , regex "(-i-| i )"
+      , Predicate $ numberBetween 1 10
+      ]
   , prod = \case
       (Token Numeral NumeralData{TNumeral.value = v1}:
        _:
@@ -212,10 +218,10 @@ ruleNumerals :: Rule
 ruleNumerals = Rule
   { name = "numbers 200..999"
   , pattern =
-      [ numberBetween 2 10
+      [ Predicate $ numberBetween 2 10
       , regex "-"
       , numberWith TNumeral.value (== 100)
-      , numberBetween 0 100
+      , Predicate $ numberBetween 0 100
       ]
   , prod = \case
       (Token Numeral NumeralData{TNumeral.value = v1}:
@@ -242,7 +248,7 @@ ruleBelowTenWithTwoDigits = Rule
   { name = "integer (0-9) with two digits"
   , pattern =
       [ regex "zero|0"
-      , numberBetween 1 10
+      , Predicate $ numberBetween 1 10
       ]
   , prod = \case
       (_:Token Numeral NumeralData{TNumeral.value = v}:_) -> double v

diff --git a/Duckling/Numeral/DA/Rules.hs b/Duckling/Numeral/DA/Rules.hs
@@ -87,7 +87,7 @@ ruleInteger3 = Rule
   { name = "integer 21..99"
   , pattern =
     [ oneOf [20, 30 .. 90]
-    , numberBetween 1 10
+    , Predicate $ numberBetween 1 10
     ]
   , prod = \tokens -> case tokens of
       (Token Numeral NumeralData{TNumeral.value = v1}:
@@ -170,7 +170,7 @@ ruleNumeralsOg :: Rule
 ruleNumeralsOg = Rule
   { name = "numbers og"
   , pattern =
-    [ numberBetween 1 10
+    [ Predicate $ numberBetween 1 10
     , regex "og"
     , oneOf [70, 20, 60, 50, 40, 90, 30, 80]
     ]

diff --git a/Duckling/Numeral/DE/Corpus.hs b/Duckling/Numeral/DE/Corpus.hs
@@ -140,6 +140,10 @@ allExamples = concat
              [ "1.416,15"
              ]
   , examples (NumeralValue 1000000.0)
-             [ "1.000.000,00"
+             [ "1.000.000,00",
+               "eine million"
+             ]
+  , examples (NumeralValue 2771090092000000.0)
+             [ "zwei billiarden sieben hundert ein und siebzig billionen neunzig milliarden zwei und neunzig millionen"
              ]
   ]
diff --git a/Duckling/Numeral/DE/Rules.hs b/Duckling/Numeral/DE/Rules.hs
@@ -123,7 +123,7 @@ ruleNumeralsUnd :: Rule
 ruleNumeralsUnd = Rule
   { name = "numbers und"
   , pattern =
-    [ numberBetween 1 10
+    [ Predicate $ numberBetween 1 10
     , regex "und"
     , oneOf [20, 30 .. 90]
     ]
@@ -201,17 +201,23 @@ rulePowersOfTen :: Rule
 rulePowersOfTen = Rule
   { name = "powers of tens"
   , pattern =
-    [ regex "(hunderte?|tausende?|million(en)?)"
+    [ regex "(hunderte?|tausende?|million(en)?|milliarde(n)?|billion(en)?|billiarde(n)?)"
     ]
   , prod = \tokens -> case tokens of
       (Token RegexMatch (GroupMatch (match:_)):_) -> case Text.toLower match of
-        "hundert"   -> double 1e2 >>= withGrain 2 >>= withMultipliable
-        "hunderte"  -> double 1e2 >>= withGrain 2 >>= withMultipliable
-        "tausend"   -> double 1e3 >>= withGrain 3 >>= withMultipliable
-        "tausende"  -> double 1e3 >>= withGrain 3 >>= withMultipliable
-        "million"   -> double 1e6 >>= withGrain 6 >>= withMultipliable
-        "millionen" -> double 1e6 >>= withGrain 6 >>= withMultipliable
-        _           -> Nothing
+        "hundert"    -> double 1e2  >>= withGrain 2  >>= withMultipliable
+        "hunderte"   -> double 1e2  >>= withGrain 2  >>= withMultipliable
+        "tausend"    -> double 1e3  >>= withGrain 3  >>= withMultipliable
+        "tausende"   -> double 1e3  >>= withGrain 3  >>= withMultipliable
+        "million"    -> double 1e6  >>= withGrain 6  >>= withMultipliable
+        "millionen"  -> double 1e6  >>= withGrain 6  >>= withMultipliable
+        "milliarde"  -> double 1e9  >>= withGrain 9  >>= withMultipliable
+        "milliarden" -> double 1e9  >>= withGrain 9  >>= withMultipliable
+        "billion"    -> double 1e12 >>= withGrain 12 >>= withMultipliable
+        "billionen"  -> double 1e12 >>= withGrain 12 >>= withMultipliable
+        "billiarde"  -> double 1e15 >>= withGrain 15 >>= withMultipliable
+        "billiarden" -> double 1e15 >>= withGrain 15 >>= withMultipliable
+        _            -> Nothing
       _ -> Nothing
   }
 
@@ -255,7 +261,7 @@ ruleZeroToNineteen :: Rule
 ruleZeroToNineteen = Rule
   { name = "integer (0..19)"
   , pattern =
-    [ regex "(keine[rn]|keine?s?|null|nichts|eins?(er)?|zwei|dreizehn|drei|vierzehn|vier|fünfzehn|fünf|sechzehn|sechs|siebzehn|sieben|achtzehn|acht|neunzehn|neun|elf|zwölf)"
+    [ regex "(keine[rn]|keine?s?|null|nichts|eins?(er?)?|zwei|dreizehn|drei|vierzehn|vier|fünfzehn|fünf|sechzehn|sechs|siebzehn|sieben|achtzehn|acht|neunzehn|neun|elf|zwölf)"
     ]
   , prod = \tokens -> case tokens of
       (Token RegexMatch (GroupMatch (match:_)):_) ->

diff --git a/Duckling/Numeral/EL/Rules.hs b/Duckling/Numeral/EL/Rules.hs
@@ -123,7 +123,7 @@ ruleCompositeTens = Rule
   { name = "integer 21..99"
   , pattern =
       [ oneOf [20,30..90]
-      , numberBetween 1 10
+      , Predicate $ numberBetween 1 10
       ]
   , prod = \tokens -> case tokens of
       (Token Numeral NumeralData{TNumeral.value = tens} :

diff --git a/Duckling/Numeral/EN/Rules.hs b/Duckling/Numeral/EN/Rules.hs
@@ -161,7 +161,7 @@ ruleCompositeTens = Rule
   , pattern =
     [ oneOf [20,30..90]
     , regex "[\\s\\-]+"
-    , numberBetween 1 10
+    , Predicate $ numberBetween 1 10
     ]
   , prod = \tokens -> case tokens of
       (Token Numeral NumeralData{TNumeral.value = tens}:

diff --git a/Duckling/Numeral/ES/Rules.hs b/Duckling/Numeral/ES/Rules.hs
@@ -69,9 +69,8 @@ ruleBelowTenWithTwoDigits = Rule
   {
     name = "integer (0-9) with two digits"
   , pattern =
-      [
-        regex "((c|z)ero)|0"
-      , numberBetween 1 10
+      [ regex "((c|z)ero)|0"
+      , Predicate $ numberBetween 1 10
       ]
   , prod = \case
       (
@@ -124,12 +123,17 @@ ruleNumeralSixteenToTwentyNine = Rule
 ruleNumeralSixteenToNineteenWithDiez :: Rule
 ruleNumeralSixteenToNineteenWithDiez = Rule
   { name = "number (16..19, two words)"
-  , pattern = [numberWith TNumeral.value (== 10), regex "y", numberBetween 6 10]
+  , pattern =
+      [ numberWith TNumeral.value (== 10)
+      , regex "y"
+      , Predicate $ numberBetween 6 10
+      ]
   , prod = \case
       (_ : _ : Token Numeral NumeralData { TNumeral.value = v } : _) ->
         double $ 10 + v
       _ -> Nothing
   }
+
 byTensMap :: HashMap.HashMap Text.Text Integer
 byTensMap =
   HashMap.fromList
@@ -161,7 +165,10 @@ ruleNumeralTwentyOneToNinetyNine :: Rule
 ruleNumeralTwentyOneToNinetyNine = Rule
   { name = "number (21..29 31..39 41..49 51..59 61..69 71..79 81..89 91..99)"
   , pattern =
-      [oneOf [20, 30 .. 90], regex "y", numberBetween 1 10]
+      [ oneOf [20, 30 .. 90]
+      , regex "y"
+      , Predicate $ numberBetween 1 10
+      ]
   , prod = \case
       (Token Numeral NumeralData { TNumeral.value = v1 } : _ : Token Numeral NumeralData { TNumeral.value = v2 } : _) ->
         double $ v1 + v2
@@ -221,7 +228,7 @@ ruleTwoPartHundreds :: Rule
 ruleTwoPartHundreds = Rule
  { name = "2..9 cientos"
  , pattern =
-     [ numberBetween 2 10
+     [ Predicate $ numberBetween 2 10
      , regex "cientos"
      ]
   , prod = \case
@@ -235,7 +242,7 @@ ruleNumeralHundredsAndSmaller = Rule
   { name = "<hundreds> 0..99"
   , pattern =
       [ numberWith TNumeral.value (TNumeral.isMultiple 100)
-      , numberBetween 0 100
+      , Predicate $ numberBetween 0 100
       ]
   , prod = \case
       (Token Numeral NumeralData { TNumeral.value = v1 } : Token Numeral NumeralData { TNumeral.value = v2 } : _)
@@ -247,7 +254,7 @@ ruleNumeralMultiply :: Rule
 ruleNumeralMultiply = Rule
   { name = "2..999 <multipliable>"
   , pattern =
-      [ numberBetween 2 1000
+      [ Predicate $ numberBetween 2 1000
       , Predicate isMultipliable
       ]
   , prod = \case
@@ -261,7 +268,7 @@ ruleNumeralThousandsAnd = Rule
   { name = "<thousands> 0..999"
   , pattern =
       [ numberWith TNumeral.value (TNumeral.isMultiple 1000)
-      , numberBetween 0 999
+      , Predicate $ numberBetween 0 999
       ]
   , prod = \case
       (Token Numeral NumeralData { TNumeral.value = v1 } : Token Numeral NumeralData { TNumeral.value = v2 } : _)
@@ -274,7 +281,7 @@ ruleNumeralMillionsAnd = Rule
   { name = "<millions> 0..999999"
   , pattern =
       [ numberWith TNumeral.value (TNumeral.isMultiple 1000000)
-      , numberBetween 0 999999
+      , Predicate $ numberBetween 0 999999
       ]
   , prod = \case
       (Token Numeral NumeralData { TNumeral.value = v1 } : Token Numeral NumeralData { TNumeral.value = v2 } : _)

diff --git a/Duckling/Numeral/ET/Rules.hs b/Duckling/Numeral/ET/Rules.hs
@@ -87,7 +87,7 @@ ruleInteger3 = Rule
   { name = "integer 21..99"
   , pattern =
     [ oneOf [70, 20, 60, 50, 40, 90, 30, 80]
-    , numberBetween 1 10
+    , Predicate $ numberBetween 1 10
     ]
   , prod = \tokens -> case tokens of
       (Token Numeral NumeralData{TNumeral.value = v1}:

diff --git a/Duckling/Numeral/FA/Rules.hs b/Duckling/Numeral/FA/Rules.hs
@@ -135,7 +135,7 @@ ruleCompositeTens = Rule
   , pattern =
     [ oneOf [20,30..90]
     , regex "و"
-    , numberBetween 1 10
+    , Predicate $ numberBetween 1 10
     ]
   , prod = \tokens -> case tokens of
       (Token Numeral NumeralData{TNumeral.value = tens}:
@@ -151,7 +151,7 @@ ruleCompositeHundred = Rule
   , pattern =
     [ oneOf [100,200..900]
     , regex "و"
-    , numberBetween 1 100
+    , Predicate $ numberBetween 1 100
     ]
   , prod = \tokens -> case tokens of
       (Token Numeral NumeralData{TNumeral.value = tens}: