From 7bf58ff47b750b58eab16c65f65995ecb0e7ac8c Mon Sep 17 00:00:00 2001
From: SystemsPurge <naktiyoussef@proton.me>
Date: Thu, 14 Nov 2024 15:59:45 +0100
Subject: [PATCH 01/12] Added query language test for arbitrary q expressions

Signed-off-by: SystemsPurge <naktiyoussef@proton.me>
---
 tests/clients/test_ngsi_ld_query.py | 291 ++++++++++++++++++++++++++++
 1 file changed, 291 insertions(+)
 create mode 100644 tests/clients/test_ngsi_ld_query.py

diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py
new file mode 100644
index 00000000..dffa550e
--- /dev/null
+++ b/tests/clients/test_ngsi_ld_query.py
@@ -0,0 +1,291 @@
+"""
+Tests for filip.cb.client
+"""
+import unittest
+import logging
+from collections.abc import Iterable
+from requests import RequestException
+from filip.clients.ngsi_ld.cb import ContextBrokerLDClient
+from filip.models.base import FiwareLDHeader
+from filip.models.ngsi_ld.context import ActionTypeLD, ContextLDEntity, ContextProperty, \
+    NamedContextProperty, NamedContextRelationship
+from tests.config import settings
+import re
+import math
+from random import Random
+
+
+# Setting up logging
+logging.basicConfig(
+    level='ERROR',
+    format='%(asctime)s %(name)s %(levelname)s: %(message)s')
+
+
+class TestLDQueryLanguage(unittest.TestCase):
+    """
+    Test class for ContextBrokerClient
+    """
+    def setUp(self) -> None:
+        """
+        Setup test data
+        Returns:
+            None
+        """
+        #Extra size parameters for modular testing
+        self.cars_nb = 10
+        self.period = 3
+        
+        #client parameters
+        self.fiware_header = FiwareLDHeader(ngsild_tenant=settings.FIWARE_SERVICE)
+        self.cb = ContextBrokerLDClient(fiware_header=self.fiware_header,
+                                        url=settings.LD_CB_URL)
+        #base id
+        self.base='urn:ngsi-ld:'
+        
+        #Some entities for relationships
+        self.garage = ContextLDEntity(id=f"{self.base}garage0",type=f"{self.base}gar")
+        self.cam = ContextLDEntity(id=f"{self.base}cam0",type=f"{self.base}cam")
+        
+        #Entities to post/test on
+        self.cars = [ContextLDEntity(id=f"{self.base}car0{i}",type=f"{self.base}car") for i in range(0,self.cars_nb-1)]
+        
+        #Some dictionaries for randomizing properties
+        self.brands = ["Batmobile","DeLorean","Knight 2000"]
+        self.addresses = [
+            {
+                "country": "Germany",
+                "street-address": {
+                    "street":"Mathieustr.",
+                    "number":10},
+                "postal-code": 52072
+            },
+            {
+                "country": "USA",
+                "street-address": {
+                    "street":"Goosetown Drive",
+                    "number":810},
+                "postal-code": 27320
+            },
+            {
+                "country": "Nigeria",
+                "street-address": {
+                    "street":"Mustapha Street",
+                    "number":46},
+                "postal-code": 65931
+            },
+        ]
+
+        #base properties/relationships
+        self.humidity = NamedContextProperty(name="humidity",value=1)
+        self.temperature = NamedContextProperty(name="temperature",value=0);
+        self.isParked = NamedContextRelationship(name="isParked",object="placeholder")
+        self.isMonitoredBy = NamedContextRelationship(name="isMonitoredBy",object="placeholder")
+        
+        #q Expressions to test
+        #Mixing single checks with op (e.g : isMonitoredBy ; temperature<30)
+        #is not implemented
+        self.qs = [
+            'temperature > 0',
+            'brand != "Batmobile"',
+            '(isParked | isMonitoredBy); address[stree-address.number]'
+            'isParked == "urn:ngsi-ld:garage0"',
+            'temperature < 60; isParked == "urn:ngsi-ld:garage"',
+            '(temperature >= 59 | humidity < 3); brand == "DeLorean"',
+            '(temperature > 30; temperature < 90)| humidity <= 5',
+            'temperature.observedAt >= "2020-12-24T12:00:00Z"',
+            'address[country] == "Germany"',
+            'address[street-address.number] == 810'
+        ]
+        self.post()
+        
+
+    def tearDown(self) -> None:
+        """
+        Cleanup test server
+        """
+        try:
+            entity_list = True
+            while entity_list:
+                entity_list = self.cb.get_entity_list(limit=1000)
+                self.cb.entity_batch_operation(action_type=ActionTypeLD.DELETE,
+                                                   entities=entity_list)
+        except RequestException:
+            pass
+        self.cb.close()
+    
+    def test_ld_query_language(self):
+        #Itertools product actually interferes with test results here
+        for q in self.qs:
+            entities = self.cb.get_entity_list(q=q)
+            tokenized,keys_dict = self.extract_keys(q)
+            f = self.expr_eval_func
+            
+            #This means that q expression contains no comparaison operators
+            #And should be dealt with as such
+            if re.fullmatch('[$\d();|][^<>=!]',tokenized) is not None:
+                f = self.single_eval_func
+            
+            for e in entities:
+                bool = f(tokenized,keys_dict,e)
+                self.assertTrue(bool)
+                
+    def extract_keys(self,q:str):
+        '''
+        Extract substring from string expression that is likely to be the name of a 
+        property/relationship of a given entity
+        Returns:
+            str,dict
+        '''
+        #First, trim empty spaces
+        n=q.replace(" ","")
+        
+        #Find all literals that are not logical operators or parentheses -> keys/values
+        res = re.findall('[^<>=)()|;!]*', n)
+        keys = {}
+        i=0
+        for r in res:
+            #Remove empty string from the regex search result
+            if len(r) == 0:
+                continue
+            
+            #Remove anything purely numeric -> Definitely a value
+            if r.isnumeric():
+                continue
+            
+            #Remove anything with a double quote -> Definitely a string value
+            if '"' in r:
+                continue
+            
+            #Replace the key name with a custom token in the string
+            token=f'${i}'
+            n= n.replace(r,token)
+            i+=1
+            
+            #Flatten composite keys by chaining them together
+            l = []
+            #Composite of the form x[...]
+            if '[' in r:
+                idx_st = r.index('[')
+                idx_e = r.index(']')
+                outer_key = r[:idx_st]
+                l.append(outer_key)
+                inner_key = r[idx_st+1:idx_e]
+                
+                #Composite of the form x[y.z...]
+                if '.' in inner_key:
+                    rest = inner_key.split('.')
+                #Composite of the form x[y]
+                else :
+                    rest = [inner_key]
+                l+=rest
+            #Composite of the form x.y...
+            elif '.' in r:
+                l+=r.split('.')
+            #Simple key
+            else:
+                l=[r]
+            
+            #Associate each chain of nested keys with the token it was replaced with
+            keys[token] = l
+        
+        return n,keys
+    
+    def sub_key_with_val(self,q:str,entity:ContextLDEntity,keylist:list[str],token:str):
+        '''
+        Substitute key names in q expression with corresponding entity property/
+        relationship values. All while accounting for access of nested properties
+        Returns:
+            str
+        '''
+        obj = entity.model_dump()
+        for key in keylist:
+            if 'value' in obj:
+                obj = obj['value']
+            obj = obj[key]
+            
+        if isinstance(obj,Iterable):
+            if 'value' in obj:
+                obj=obj['value']
+            elif 'object' in obj:
+                obj=obj['object']
+        
+        #Enclose value in double quotes if it's a string ( contains at least one letter)
+        if re.compile('[a-zA-Z]+').match(str(obj)):
+            obj = f'"{str(obj)}"'
+        
+        #replace key names with entity values
+        n = q.replace(token,str(obj))
+        
+        #replace logical operators with python ones
+        n = n.replace("|"," or ")
+        n = n.replace(";"," and ")
+        
+        return n
+    
+    def expr_eval_func(self,tokenized,keys_dict,e):
+        '''
+        Check function for the case of q expression containing comparaison operators
+        Have to replace the keys with values then call Eval
+        '''
+        for token,keylist in keys_dict.items():
+            tokenized = self.sub_key_with_val(tokenized,e,keylist,token)
+        return eval(tokenized)
+    
+    def single_eval_func(self,tokenized,keys_dict,e):
+        '''
+        Check function for the case of q expression containing NO comparaison operators
+        Only have to check if entity has the key
+        '''
+        for token,keylist in keys_dict.items():
+            level = e.model_dump()
+            for key in keylist:
+                if 'value' in level:
+                    level = level['value']
+                if key not in level:
+                    return False
+                level = level[key]
+        
+        return True
+    
+    def post(self):
+        '''
+        Somewhat randomized generation of data. Can be made further random by 
+        Choosing a bigger number of cars, and a more irregular number for remainder
+        Calculations (self.cars_nb & self.period)
+        Returns:
+            None
+        '''
+        for i in range(len(self.cars)):
+            r = i%self.period
+            a=r*30
+            b=a+30
+
+            #Every car will have temperature, humidity, brand and address
+            t = self.temperature.model_copy()
+            t.value = Random().randint(a,b)
+            
+            h = self.humidity.model_copy()
+            h.value = Random().randint(math.trunc(a/10),math.trunc(b/10))
+            
+            self.cars[i].add_properties([t,h,NamedContextProperty(name="brand",value=self.brands[r]),
+                                         NamedContextProperty(name="address",value=self.addresses[r])])
+
+            p = self.isParked.model_copy()
+            p.object = self.garage.id
+
+            m = self.isMonitoredBy.model_copy()
+            m.object = self.cam.id
+
+            #Every car is endowed with a set of relationships , periodically
+            match (i % self.period):
+                case 0:
+                    self.cars[i].add_relationships([p])
+                case 1:
+                    self.cars[i].add_relationships([m])
+                case 2:
+                    self.cars[i].add_relationships([p,m])
+                case _:
+                    pass
+        #Post everything
+        for car in self.cars:
+            self.cb.post_entity(entity=car)

From 13eba86940bde0f0c3f231109085b47d44c648b9 Mon Sep 17 00:00:00 2001
From: SystemsPurge <naktiyoussef@proton.me>
Date: Thu, 14 Nov 2024 16:14:56 +0100
Subject: [PATCH 02/12] Changed python version incompatible code

Signed-off-by: SystemsPurge <naktiyoussef@proton.me>
---
 tests/clients/test_ngsi_ld_query.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py
index dffa550e..6f718ec5 100644
--- a/tests/clients/test_ngsi_ld_query.py
+++ b/tests/clients/test_ngsi_ld_query.py
@@ -277,15 +277,14 @@ def post(self):
             m.object = self.cam.id
 
             #Every car is endowed with a set of relationships , periodically
-            match (i % self.period):
-                case 0:
-                    self.cars[i].add_relationships([p])
-                case 1:
-                    self.cars[i].add_relationships([m])
-                case 2:
-                    self.cars[i].add_relationships([p,m])
-                case _:
-                    pass
+            r = i % self.period
+            if r==0:
+                self.cars[i].add_relationships([p])
+            elif r==1:
+                self.cars[i].add_relationships([m])
+            elif r==2:
+                self.cars[i].add_relationships([p,m])
+    
         #Post everything
         for car in self.cars:
             self.cb.post_entity(entity=car)

From 44de78b8693a21235cdfa4a9c4a74fae91dbd2ff Mon Sep 17 00:00:00 2001
From: SystemsPurge <naktiyoussef@proton.me>
Date: Thu, 14 Nov 2024 16:16:55 +0100
Subject: [PATCH 03/12] Removed python version incompatible code

Signed-off-by: SystemsPurge <naktiyoussef@proton.me>
---
 tests/clients/test_ngsi_ld_query.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py
index 6f718ec5..7c1b9b89 100644
--- a/tests/clients/test_ngsi_ld_query.py
+++ b/tests/clients/test_ngsi_ld_query.py
@@ -190,7 +190,7 @@ def extract_keys(self,q:str):
         
         return n,keys
     
-    def sub_key_with_val(self,q:str,entity:ContextLDEntity,keylist:list[str],token:str):
+    def sub_key_with_val(self,q:str,entity:ContextLDEntity,keylist,token:str):
         '''
         Substitute key names in q expression with corresponding entity property/
         relationship values. All while accounting for access of nested properties

From 4e07235878d12206c6a53308472e4a2f90b24bff Mon Sep 17 00:00:00 2001
From: SystemsPurge <naktiyoussef@proton.me>
Date: Thu, 14 Nov 2024 16:25:29 +0100
Subject: [PATCH 04/12] Added initial teardown to prep db

Signed-off-by: SystemsPurge <naktiyoussef@proton.me>
---
 tests/clients/test_ngsi_ld_query.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py
index 7c1b9b89..16a572db 100644
--- a/tests/clients/test_ngsi_ld_query.py
+++ b/tests/clients/test_ngsi_ld_query.py
@@ -39,6 +39,17 @@ def setUp(self) -> None:
         self.fiware_header = FiwareLDHeader(ngsild_tenant=settings.FIWARE_SERVICE)
         self.cb = ContextBrokerLDClient(fiware_header=self.fiware_header,
                                         url=settings.LD_CB_URL)
+
+        #Prep db
+        try:
+            entity_list = True
+            while entity_list:
+                entity_list = self.cb.get_entity_list(limit=1000)
+                self.cb.entity_batch_operation(action_type=ActionTypeLD.DELETE,
+                                                   entities=entity_list)
+        except RequestException:
+            pass
+        
         #base id
         self.base='urn:ngsi-ld:'
         

From 7bf7b9830d31f21430ce8e920493ecb279b481a7 Mon Sep 17 00:00:00 2001
From: SystemsPurge <naktiyoussef@proton.me>
Date: Fri, 15 Nov 2024 10:58:34 +0100
Subject: [PATCH 05/12] Unified parsing cases of q expressions

Signed-off-by: SystemsPurge <naktiyoussef@proton.me>
---
 tests/clients/test_ngsi_ld_query.py | 114 +++++++++++++++-------------
 1 file changed, 62 insertions(+), 52 deletions(-)

diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py
index 16a572db..0ca02fb9 100644
--- a/tests/clients/test_ngsi_ld_query.py
+++ b/tests/clients/test_ngsi_ld_query.py
@@ -32,8 +32,8 @@ def setUp(self) -> None:
             None
         """
         #Extra size parameters for modular testing
-        self.cars_nb = 10
-        self.period = 3
+        self.cars_nb = 500
+        self.span = 3
         
         #client parameters
         self.fiware_header = FiwareLDHeader(ngsild_tenant=settings.FIWARE_SERVICE)
@@ -93,20 +93,21 @@ def setUp(self) -> None:
         self.isMonitoredBy = NamedContextRelationship(name="isMonitoredBy",object="placeholder")
         
         #q Expressions to test
-        #Mixing single checks with op (e.g : isMonitoredBy ; temperature<30)
-        #is not implemented
         self.qs = [
             'temperature > 0',
             'brand != "Batmobile"',
-            '(isParked | isMonitoredBy); address[stree-address.number]'
+            'isParked | isMonitoredBy',
             'isParked == "urn:ngsi-ld:garage0"',
-            'temperature < 60; isParked == "urn:ngsi-ld:garage"',
+            'temperature < 60; isParked == "urn:ngsi-ld:garage0"',
             '(temperature >= 59 | humidity < 3); brand == "DeLorean"',
+            '(isMonitoredBy; temperature<30) | isParked',
             '(temperature > 30; temperature < 90)| humidity <= 5',
             'temperature.observedAt >= "2020-12-24T12:00:00Z"',
             'address[country] == "Germany"',
-            'address[street-address.number] == 810'
+            'address[street-address.number] == 810',
+            'address[street-address.number]'
         ]
+        
         self.post()
         
 
@@ -127,18 +128,22 @@ def tearDown(self) -> None:
     def test_ld_query_language(self):
         #Itertools product actually interferes with test results here
         for q in self.qs:
-            entities = self.cb.get_entity_list(q=q)
+            entities = self.cb.get_entity_list(q=q,limit=1000)
             tokenized,keys_dict = self.extract_keys(q)
-            f = self.expr_eval_func
-            
-            #This means that q expression contains no comparaison operators
-            #And should be dealt with as such
-            if re.fullmatch('[$\d();|][^<>=!]',tokenized) is not None:
-                f = self.single_eval_func
             
+            #Replace logical ops with python ones
+            tokenized = tokenized.replace("|"," or ")
+            tokenized = tokenized.replace(";"," and ")
+            size = len([x for x in self.cars if self.search_predicate(x,tokenized,keys_dict)])
+            #Check we get the same number of entities
+            self.assertEqual(size,len(entities))
             for e in entities:
-                bool = f(tokenized,keys_dict,e)
-                self.assertTrue(bool)
+                copy = tokenized
+                for token,keylist in keys_dict.items():
+                    copy = self.sub_key_with_val(copy,e,keylist,token)
+                
+                #Check each obtained entity obeys the q expression
+                self.assertTrue(eval(copy))
                 
     def extract_keys(self,q:str):
         '''
@@ -147,7 +152,7 @@ def extract_keys(self,q:str):
         Returns:
             str,dict
         '''
-        #First, trim empty spaces
+        #Trim empty spaces
         n=q.replace(" ","")
         
         #Find all literals that are not logical operators or parentheses -> keys/values
@@ -155,18 +160,22 @@ def extract_keys(self,q:str):
         keys = {}
         i=0
         for r in res:
-            #Remove empty string from the regex search result
+            #Skip empty string from the regex search result
             if len(r) == 0:
                 continue
             
-            #Remove anything purely numeric -> Definitely a value
+            #Skip anything purely numeric -> Definitely a value
             if r.isnumeric():
                 continue
             
-            #Remove anything with a double quote -> Definitely a string value
+            #Skip anything with a double quote -> Definitely a string value
             if '"' in r:
                 continue
             
+            #Skip keys we already encountered
+            if [r] in keys.values():
+                continue
+            
             #Replace the key name with a custom token in the string
             token=f'${i}'
             n= n.replace(r,token)
@@ -196,9 +205,14 @@ def extract_keys(self,q:str):
             else:
                 l=[r]
             
+            #Finalize incomplete key presence check 
+            idx_next = n.index(token)+len(token)
+            if idx_next>=len(n) or n[idx_next] not in ['>','<','=','!']:
+                n = n.replace(token,f'{token} != None')
+            
             #Associate each chain of nested keys with the token it was replaced with
             keys[token] = l
-        
+            
         return n,keys
     
     def sub_key_with_val(self,q:str,entity:ContextLDEntity,keylist,token:str):
@@ -212,7 +226,11 @@ def sub_key_with_val(self,q:str,entity:ContextLDEntity,keylist,token:str):
         for key in keylist:
             if 'value' in obj:
                 obj = obj['value']
-            obj = obj[key]
+            try:   
+                obj = obj[key]
+            except:
+                obj = None
+                break
             
         if isinstance(obj,Iterable):
             if 'value' in obj:
@@ -221,54 +239,47 @@ def sub_key_with_val(self,q:str,entity:ContextLDEntity,keylist,token:str):
                 obj=obj['object']
         
         #Enclose value in double quotes if it's a string ( contains at least one letter)
-        if re.compile('[a-zA-Z]+').match(str(obj)):
+        if obj is not None and re.compile('[a-zA-Z]+').match(str(obj)):
             obj = f'"{str(obj)}"'
         
         #replace key names with entity values
         n = q.replace(token,str(obj))
         
-        #replace logical operators with python ones
-        n = n.replace("|"," or ")
-        n = n.replace(";"," and ")
-        
         return n
     
-    def expr_eval_func(self,tokenized,keys_dict,e):
+    def search_predicate(self,e,tokenized,keys_dict):
         '''
-        Check function for the case of q expression containing comparaison operators
-        Have to replace the keys with values then call Eval
+        Search function to search our posted data for checks
+        This function is needed because , whereas the context broker will not return
+        an entity with no nested key if that key is given as a filter, our eval attempts
+        to compare None values using logical operators
         '''
+        copy = tokenized
         for token,keylist in keys_dict.items():
-            tokenized = self.sub_key_with_val(tokenized,e,keylist,token)
-        return eval(tokenized)
-    
-    def single_eval_func(self,tokenized,keys_dict,e):
-        '''
-        Check function for the case of q expression containing NO comparaison operators
-        Only have to check if entity has the key
-        '''
-        for token,keylist in keys_dict.items():
-            level = e.model_dump()
-            for key in keylist:
-                if 'value' in level:
-                    level = level['value']
-                if key not in level:
-                    return False
-                level = level[key]
+            copy = self.sub_key_with_val(copy,e,keylist,token)
         
-        return True
+        try:
+            return eval(copy)
+        except:
+            return False
+    
     
     def post(self):
         '''
         Somewhat randomized generation of data. Can be made further random by 
         Choosing a bigger number of cars, and a more irregular number for remainder
-        Calculations (self.cars_nb & self.period)
+        Calculations (self.cars_nb & self.span)
         Returns:
             None
         '''
         for i in range(len(self.cars)):
-            r = i%self.period
-            a=r*30
+            #Big number rnd generator
+            r = Random().randint(1,self.span)
+            tri_rnd = Random().randint(0,(10*self.span)**2)
+            r = math.trunc(tri_rnd/r) % self.span
+            r_2 = Random().randint(0,r)
+            
+            a=r_2*30
             b=a+30
 
             #Every car will have temperature, humidity, brand and address
@@ -288,14 +299,13 @@ def post(self):
             m.object = self.cam.id
 
             #Every car is endowed with a set of relationships , periodically
-            r = i % self.period
             if r==0:
                 self.cars[i].add_relationships([p])
             elif r==1:
                 self.cars[i].add_relationships([m])
             elif r==2:
                 self.cars[i].add_relationships([p,m])
-    
+
         #Post everything
         for car in self.cars:
             self.cb.post_entity(entity=car)

From da7722c053f50400b629ef8dea0e91a173b6a00d Mon Sep 17 00:00:00 2001
From: JunsongDu <junsong.du@eonerc.rwth-aachen.de>
Date: Tue, 19 Nov 2024 11:49:53 +0100
Subject: [PATCH 06/12] fix: add exclude none in batch operation

---
 filip/clients/ngsi_ld/cb.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/filip/clients/ngsi_ld/cb.py b/filip/clients/ngsi_ld/cb.py
index 573a060b..7dca70df 100644
--- a/filip/clients/ngsi_ld/cb.py
+++ b/filip/clients/ngsi_ld/cb.py
@@ -766,6 +766,7 @@ def entity_batch_operation(self,
                     params=params,
                     data=json.dumps(update.model_dump(by_alias=True,
                                                       exclude_unset=True,
+                                                      exclude_none=True,
                                                       ).get('entities'))
                 )
             self.handle_multi_status_response(res)

From 40886bb08cd9fff80b3783aa6ff23b8e9fb82231 Mon Sep 17 00:00:00 2001
From: JunsongDu <junsong.du@eonerc.rwth-aachen.de>
Date: Tue, 19 Nov 2024 11:50:37 +0100
Subject: [PATCH 07/12] chore: delete unused parameter in subscription

---
 filip/models/ngsi_ld/subscriptions.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/filip/models/ngsi_ld/subscriptions.py b/filip/models/ngsi_ld/subscriptions.py
index 1bbf66d1..0c72b859 100644
--- a/filip/models/ngsi_ld/subscriptions.py
+++ b/filip/models/ngsi_ld/subscriptions.py
@@ -277,10 +277,6 @@ def check_q(cls, v: str):
         default=None,
         description="Temporal Query"
     )
-    scopeQ: Optional[str] = Field(
-        default=None,
-        description="Scope query"
-    )
     lang: Optional[str] = Field(
         default=None,
         description="Language filter applied to the query"

From 1caed4f75e272dcaef5ed95279b6a2c841a196c2 Mon Sep 17 00:00:00 2001
From: JunsongDu <junsong.du@eonerc.rwth-aachen.de>
Date: Tue, 19 Nov 2024 11:53:04 +0100
Subject: [PATCH 08/12] chore: use clean up function and some minor changes

---
 tests/clients/test_ngsi_ld_query.py | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py
index 0ca02fb9..ef8dd067 100644
--- a/tests/clients/test_ngsi_ld_query.py
+++ b/tests/clients/test_ngsi_ld_query.py
@@ -13,6 +13,7 @@
 import re
 import math
 from random import Random
+from filip.utils.cleanup import clear_context_broker_ld
 
 
 # Setting up logging
@@ -41,14 +42,7 @@ def setUp(self) -> None:
                                         url=settings.LD_CB_URL)
 
         #Prep db
-        try:
-            entity_list = True
-            while entity_list:
-                entity_list = self.cb.get_entity_list(limit=1000)
-                self.cb.entity_batch_operation(action_type=ActionTypeLD.DELETE,
-                                                   entities=entity_list)
-        except RequestException:
-            pass
+        clear_context_broker_ld(cb_ld_client=self.cb)
         
         #base id
         self.base='urn:ngsi-ld:'
@@ -56,6 +50,8 @@ def setUp(self) -> None:
         #Some entities for relationships
         self.garage = ContextLDEntity(id=f"{self.base}garage0",type=f"{self.base}gar")
         self.cam = ContextLDEntity(id=f"{self.base}cam0",type=f"{self.base}cam")
+        self.cb.post_entity(entity=self.garage)
+        self.cb.post_entity(entity=self.cam)
         
         #Entities to post/test on
         self.cars = [ContextLDEntity(id=f"{self.base}car0{i}",type=f"{self.base}car") for i in range(0,self.cars_nb-1)]
@@ -88,7 +84,7 @@ def setUp(self) -> None:
 
         #base properties/relationships
         self.humidity = NamedContextProperty(name="humidity",value=1)
-        self.temperature = NamedContextProperty(name="temperature",value=0);
+        self.temperature = NamedContextProperty(name="temperature",value=0)
         self.isParked = NamedContextRelationship(name="isParked",object="placeholder")
         self.isMonitoredBy = NamedContextRelationship(name="isMonitoredBy",object="placeholder")
         
@@ -115,14 +111,7 @@ def tearDown(self) -> None:
         """
         Cleanup test server
         """
-        try:
-            entity_list = True
-            while entity_list:
-                entity_list = self.cb.get_entity_list(limit=1000)
-                self.cb.entity_batch_operation(action_type=ActionTypeLD.DELETE,
-                                                   entities=entity_list)
-        except RequestException:
-            pass
+        clear_context_broker_ld(cb_ld_client=self.cb)
         self.cb.close()
     
     def test_ld_query_language(self):

From a2f8eabcd312c8dc0989d241de0a2a17592ed201 Mon Sep 17 00:00:00 2001
From: JunsongDu <junsong.du@eonerc.rwth-aachen.de>
Date: Tue, 19 Nov 2024 11:55:10 +0100
Subject: [PATCH 09/12] chore: use batch operation to create entities

---
 tests/clients/test_ngsi_ld_query.py |  7 ++---
 tests/models/test_ngsi_ld_query.py  | 46 -----------------------------
 2 files changed, 3 insertions(+), 50 deletions(-)
 delete mode 100644 tests/models/test_ngsi_ld_query.py

diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py
index ef8dd067..e37ac9a4 100644
--- a/tests/clients/test_ngsi_ld_query.py
+++ b/tests/clients/test_ngsi_ld_query.py
@@ -251,8 +251,7 @@ def search_predicate(self,e,tokenized,keys_dict):
             return eval(copy)
         except:
             return False
-    
-    
+
     def post(self):
         '''
         Somewhat randomized generation of data. Can be made further random by 
@@ -296,5 +295,5 @@ def post(self):
                 self.cars[i].add_relationships([p,m])
 
         #Post everything
-        for car in self.cars:
-            self.cb.post_entity(entity=car)
+        self.cb.entity_batch_operation(action_type=ActionTypeLD.CREATE,
+                                       entities=self.cars)
diff --git a/tests/models/test_ngsi_ld_query.py b/tests/models/test_ngsi_ld_query.py
deleted file mode 100644
index f9c9d086..00000000
--- a/tests/models/test_ngsi_ld_query.py
+++ /dev/null
@@ -1,46 +0,0 @@
-"""
-Test module for NGSI-LD query language based on NGSI-LD Spec section 4.9
-"""
-import json
-import unittest
-
-from pydantic import ValidationError
-from filip.clients.ngsi_v2 import ContextBrokerClient
-from filip.models.ngsi_v2.subscriptions import \
-    Http, \
-    HttpCustom, \
-    Mqtt, \
-    MqttCustom, \
-    Notification, \
-    Subscription
-from filip.models.base import FiwareHeader
-from filip.utils.cleanup import clear_all, clean_test
-from tests.config import settings
-
-
-class TestLDQuery(unittest.TestCase):
-    """
-    Test class for context broker models
-    """
-    # TODO the specs have to be read carefully
-
-    def setUp(self) -> None:
-        """
-        Setup test data
-        Returns:
-            None
-        """
-        self.fiware_header = FiwareHeader(
-            service=settings.FIWARE_SERVICE,
-            service_path=settings.FIWARE_SERVICEPATH)
-        # self.http_url = "https://test.de:80"
-        # self.mqtt_url = "mqtt://test.de:1883"
-        # self.mqtt_topic = '/filip/testing'
-
-
-    def tearDown(self) -> None:
-        """
-        Cleanup test server
-        """
-        clear_all(fiware_header=self.fiware_header,
-                  cb_url=settings.CB_URL)
\ No newline at end of file

From 5f39488f8ac502b7e488fd7f2e7504180b97ada7 Mon Sep 17 00:00:00 2001
From: JunsongDu <junsong.du@eonerc.rwth-aachen.de>
Date: Tue, 19 Nov 2024 12:08:40 +0100
Subject: [PATCH 10/12] chore: add not existed attr as condition

---
 tests/clients/test_ngsi_ld_query.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py
index e37ac9a4..ff256a44 100644
--- a/tests/clients/test_ngsi_ld_query.py
+++ b/tests/clients/test_ngsi_ld_query.py
@@ -101,7 +101,8 @@ def setUp(self) -> None:
             'temperature.observedAt >= "2020-12-24T12:00:00Z"',
             'address[country] == "Germany"',
             'address[street-address.number] == 810',
-            'address[street-address.number]'
+            'address[street-address.number]',
+            'address[street-address.extra]',
         ]
         
         self.post()

From e90dd89479bb0f5bfc36c3b6af520a202634e4ab Mon Sep 17 00:00:00 2001
From: SystemsPurge <naktiyoussef@proton.me>
Date: Wed, 20 Nov 2024 13:45:29 +0100
Subject: [PATCH 11/12] Added proper parsing and comparaison of date fields

Signed-off-by: SystemsPurge <naktiyoussef@proton.me>
---
 tests/clients/test_ngsi_ld_query.py | 51 ++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 19 deletions(-)

diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py
index ff256a44..00edf696 100644
--- a/tests/clients/test_ngsi_ld_query.py
+++ b/tests/clients/test_ngsi_ld_query.py
@@ -3,6 +3,10 @@
 """
 import unittest
 import logging
+import re
+import math
+import time
+from dateutil.parser import parse
 from collections.abc import Iterable
 from requests import RequestException
 from filip.clients.ngsi_ld.cb import ContextBrokerLDClient
@@ -10,8 +14,6 @@
 from filip.models.ngsi_ld.context import ActionTypeLD, ContextLDEntity, ContextProperty, \
     NamedContextProperty, NamedContextRelationship
 from tests.config import settings
-import re
-import math
 from random import Random
 from filip.utils.cleanup import clear_context_broker_ld
 
@@ -48,8 +50,8 @@ def setUp(self) -> None:
         self.base='urn:ngsi-ld:'
         
         #Some entities for relationships
-        self.garage = ContextLDEntity(id=f"{self.base}garage0",type=f"{self.base}gar")
-        self.cam = ContextLDEntity(id=f"{self.base}cam0",type=f"{self.base}cam")
+        self.garage = ContextLDEntity(id=f"{self.base}garage0",type=f"garage")
+        self.cam = ContextLDEntity(id=f"{self.base}cam0",type=f"camera")
         self.cb.post_entity(entity=self.garage)
         self.cb.post_entity(entity=self.cam)
         
@@ -58,6 +60,7 @@ def setUp(self) -> None:
         
         #Some dictionaries for randomizing properties
         self.brands = ["Batmobile","DeLorean","Knight 2000"]
+        self.timestamps = ["2020-12-24T11:00:00Z","2020-12-24T12:00:00Z","2020-12-24T13:00:00Z"]
         self.addresses = [
             {
                 "country": "Germany",
@@ -126,14 +129,14 @@ def test_ld_query_language(self):
             tokenized = tokenized.replace(";"," and ")
             size = len([x for x in self.cars if self.search_predicate(x,tokenized,keys_dict)])
             #Check we get the same number of entities
-            self.assertEqual(size,len(entities))
+            self.assertEqual(size,len(entities),q)
             for e in entities:
                 copy = tokenized
                 for token,keylist in keys_dict.items():
                     copy = self.sub_key_with_val(copy,e,keylist,token)
                 
                 #Check each obtained entity obeys the q expression
-                self.assertTrue(eval(copy))
+                self.assertTrue(eval(copy),q)
                 
     def extract_keys(self,q:str):
         '''
@@ -157,11 +160,18 @@ def extract_keys(self,q:str):
             #Skip anything purely numeric -> Definitely a value
             if r.isnumeric():
                 continue
-            
-            #Skip anything with a double quote -> Definitely a string value
+            #Skip anything with a double quote -> string or date
             if '"' in r:
+                try:
+                    #replace date with unix ts
+                    timestamp = r.replace("\"","")
+                    date = parse(timestamp)
+                    timestamp = str(time.mktime(date.timetuple()))
+                    n = n.replace(r,timestamp)
+                except Exception as e:
+                    r=f'\"{r}\"'
                 continue
-            
+             
             #Skip keys we already encountered
             if [r] in keys.values():
                 continue
@@ -202,7 +212,6 @@ def extract_keys(self,q:str):
             
             #Associate each chain of nested keys with the token it was replaced with
             keys[token] = l
-            
         return n,keys
     
     def sub_key_with_val(self,q:str,entity:ContextLDEntity,keylist,token:str):
@@ -214,11 +223,11 @@ def sub_key_with_val(self,q:str,entity:ContextLDEntity,keylist,token:str):
         '''
         obj = entity.model_dump()
         for key in keylist:
-            if 'value' in obj:
-                obj = obj['value']
-            try:   
+            if key in obj:
                 obj = obj[key]
-            except:
+            elif 'value' in obj and key in obj['value']:
+                obj = obj['value'][key]
+            else:
                 obj = None
                 break
             
@@ -228,13 +237,16 @@ def sub_key_with_val(self,q:str,entity:ContextLDEntity,keylist,token:str):
             elif 'object' in obj:
                 obj=obj['object']
         
-        #Enclose value in double quotes if it's a string ( contains at least one letter)
-        if obj is not None and re.compile('[a-zA-Z]+').match(str(obj)):
-            obj = f'"{str(obj)}"'
+        if obj is not None and re.compile('[a-zA-Z]+').search(str(obj)) is not None:
+            try:
+                date = parse(obj)
+                obj = str(time.mktime(date.timetuple())) #convert to unix ts
+            except Exception as e:
+                obj = f'"{str(obj)}"'
+            
         
         #replace key names with entity values
         n = q.replace(token,str(obj))
-        
         return n
     
     def search_predicate(self,e,tokenized,keys_dict):
@@ -274,6 +286,7 @@ def post(self):
             #Every car will have temperature, humidity, brand and address
             t = self.temperature.model_copy()
             t.value = Random().randint(a,b)
+            t.observedAt = self.timestamps[r]
             
             h = self.humidity.model_copy()
             h.value = Random().randint(math.trunc(a/10),math.trunc(b/10))
@@ -287,7 +300,7 @@ def post(self):
             m = self.isMonitoredBy.model_copy()
             m.object = self.cam.id
 
-            #Every car is endowed with a set of relationships , periodically
+            #Every car is endowed with a set of relationships/nested key
             if r==0:
                 self.cars[i].add_relationships([p])
             elif r==1:

From 20b3912800e068361bee77a23890f5c802015876 Mon Sep 17 00:00:00 2001
From: JunsongDu <junsong.du@eonerc.rwth-aachen.de>
Date: Wed, 20 Nov 2024 14:35:05 +0100
Subject: [PATCH 12/12] chore: minor change for readability

---
 tests/clients/test_ngsi_ld_query.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py
index 00edf696..c8050485 100644
--- a/tests/clients/test_ngsi_ld_query.py
+++ b/tests/clients/test_ngsi_ld_query.py
@@ -129,14 +129,14 @@ def test_ld_query_language(self):
             tokenized = tokenized.replace(";"," and ")
             size = len([x for x in self.cars if self.search_predicate(x,tokenized,keys_dict)])
             #Check we get the same number of entities
-            self.assertEqual(size,len(entities),q)
+            self.assertEqual(size,len(entities),msg=q)
             for e in entities:
                 copy = tokenized
                 for token,keylist in keys_dict.items():
                     copy = self.sub_key_with_val(copy,e,keylist,token)
                 
                 #Check each obtained entity obeys the q expression
-                self.assertTrue(eval(copy),q)
+                self.assertTrue(eval(copy),msg=q)
                 
     def extract_keys(self,q:str):
         '''