diff --git a/etk/core.py b/etk/core.py index c6f25ad0..5b4ae8d2 100644 --- a/etk/core.py +++ b/etk/core.py @@ -45,6 +45,7 @@ import traceback import logging import logstash +import signal _KNOWLEDGE_GRAPH = "knowledge_graph" _EXTRACTION_POLICY = 'extraction_policy' @@ -107,6 +108,7 @@ _POPULATED_CITIES = "populated_cities" _CASE_SENSITIVE = 'case_sensitive' +_EXTRACT_AS_IS = "extract_as_is" _EXTRACT_USING_DICTIONARY = "extract_using_dictionary" _EXTRACT_USING_REGEX = "extract_using_regex" _EXTRACT_FROM_LANDMARK = "extract_from_landmark" @@ -120,6 +122,7 @@ _CONFIG = "config" _DICTIONARIES = "dictionaries" +_STOP_WORD_DICTIONARIES = "stop_word_dictionaries" _INFERLINK = "inferlink" _HTML = "html" @@ -153,6 +156,27 @@ _ETK_VERSION = "etk_version" _CONVERT_TO_KG = "convert_to_kg" _PREFER_INFERLINK_DESCRIPTION = "prefer_inferlink_description" +_TIMEOUT = "timeout" +_JSON_CONTENT = 'json_content' + +ten = '\n \n \n \n \n \n \n \n \n \n' +nine = '\n \n \n \n \n \n \n \n \n' +eight = '\n \n \n \n \n \n \n \n' +seven = '\n \n \n \n \n \n \n' +six = '\n \n \n \n \n \n' +five = '\n \n \n \n \n ' +four = '\n \n \n \n' +three = '\n \n \n' +two = '\n \n' +one = '\n' +remove_break_html_2 = re.compile("[\r\n][\s]*[\r\n]") +remove_break_html_1 = re.compile("[\r\n][\s]*") + +ns = [ten, nine, eight, seven, six, five, four, three, two, one] + + +class TimeoutException(Exception): # Custom exception class + pass class Core(object): @@ -161,6 +185,7 @@ def __init__(self, extraction_config=None, debug=False, load_spacy=False): self.debug = debug self.html_title_regex = r'(.*?)' self.tries = dict() + self.stop_word_dicts = dict() self.pickles = dict() self.jobjs = dict() self.global_extraction_policy = None @@ -169,6 +194,7 @@ def __init__(self, extraction_config=None, debug=False, load_spacy=False): self.content_extraction_path = None self.data_extraction_path = dict() self.kgc_paths = dict() + self.json_content_paths = dict() if load_spacy: self.prep_spacy() else: @@ -182,6 +208,7 @@ def __init__(self, extraction_config=None, debug=False, load_spacy=False): self.logstash_logger = None self.etk_version = "1" self.prefer_inferlink_description = False + self.readability_timeout = 3 if self.extraction_config: if _PREFER_INFERLINK_DESCRIPTION in self.extraction_config: self.prefer_inferlink_description = self.extraction_config[_PREFER_INFERLINK_DESCRIPTION] @@ -223,8 +250,12 @@ def log(self, message, level, doc_id=None, url=None, extra=None): """ Define all API methods """ - def process(self, doc, create_knowledge_graph=False): - start_time = time.time() + @staticmethod + def timeout_handler(signum, frame): # Custom signal handler + raise TimeoutException + + def process(self, doc, create_knowledge_graph=False, html_description=False): + start_time_process = time.time() try: if self.extraction_config: doc_id = None @@ -278,61 +309,87 @@ def process(self, doc, create_knowledge_graph=False): if _CONTENT_EXTRACTION not in doc: doc[_CONTENT_EXTRACTION] = dict() ce_config = self.extraction_config[_CONTENT_EXTRACTION] + + # JSON CONTENT: create content for data extraction from json paths + if _JSON_CONTENT in ce_config: + jc_extractors = ce_config[_JSON_CONTENT] + if isinstance(jc_extractors, dict): + jc_extractors = [jc_extractors] + for jc_extractor in jc_extractors: + doc = self.convert_json_content(doc, jc_extractor) + html_path = ce_config[_INPUT_PATH] if _INPUT_PATH in ce_config else None - if not html_path: + if not html_path and _EXTRACTORS in ce_config: raise KeyError('{} not found in extraction_config'.format(_INPUT_PATH)) - - if not self.content_extraction_path: + if html_path and _EXTRACTORS in ce_config: + if not self.content_extraction_path: + start_time = time.time() + self.content_extraction_path = parse(html_path) + time_taken = time.time() - start_time + # print 'LOG: {},{},{},{}'.format(doc_id, 'Json path parser', 'parse', time_taken) + if self.debug: + self.log('time taken to process parse %s' % time_taken, _DEBUG, doc_id=doc[_DOCUMENT_ID], + url=doc[_URL]) start_time = time.time() - self.content_extraction_path = parse(html_path) + matches = self.content_extraction_path.find(doc) time_taken = time.time() - start_time + # print 'LOG: {},{},{},{}'.format(doc_id, 'Json path parser', 'find', time_taken) if self.debug: - self.log('time taken to process parse %s' % time_taken, _DEBUG, doc_id=doc[_DOCUMENT_ID], + self.log('time taken to process matches %s' % time_taken, _DEBUG, doc_id=doc[_DOCUMENT_ID], url=doc[_URL]) - start_time = time.time() - matches = self.content_extraction_path.find(doc) - time_taken = time.time() - start_time - if self.debug: - self.log('time taken to process matches %s' % time_taken, _DEBUG, doc_id=doc[_DOCUMENT_ID], - url=doc[_URL]) - extractors = ce_config[_EXTRACTORS] - run_readability = True - for index in range(len(matches)): - for extractor in extractors.keys(): - if extractor == _LANDMARK: - doc[_CONTENT_EXTRACTION] = self.run_landmark(doc[_CONTENT_EXTRACTION], - matches[index].value, - extractors[extractor], doc[_URL]) - landmark_config = extractors[extractor] - landmark_field_name = landmark_config[_FIELD_NAME] if _FIELD_NAME in landmark_config \ - else _INFERLINK_EXTRACTIONS - if self.prefer_inferlink_description: - if landmark_field_name in doc[_CONTENT_EXTRACTION]: - if _INFERLINK_DESCRIPTION in doc[_CONTENT_EXTRACTION][landmark_field_name]: - inferlink_desc = doc[_CONTENT_EXTRACTION][landmark_field_name][ - _INFERLINK_DESCRIPTION] - if _TEXT in inferlink_desc and inferlink_desc[_TEXT] and inferlink_desc[ - _TEXT].strip() != '': - run_readability = False - - elif extractor == _READABILITY: - if run_readability: - re_extractors = extractors[extractor] - if isinstance(re_extractors, dict): - re_extractors = [re_extractors] - for re_extractor in re_extractors: - doc[_CONTENT_EXTRACTION] = self.run_readability(doc[_CONTENT_EXTRACTION], + extractors = ce_config[_EXTRACTORS] + run_readability = True + for index in range(len(matches)): + for extractor in extractors.keys(): + if extractor == _LANDMARK: + s = time.time() + doc[_CONTENT_EXTRACTION] = self.run_landmark(doc[_CONTENT_EXTRACTION], + matches[index].value, + extractors[extractor], doc[_URL]) + e = time.time()-s + + landmark_config = extractors[extractor] + landmark_field_name = landmark_config[_FIELD_NAME] if _FIELD_NAME in landmark_config \ + else _INFERLINK_EXTRACTIONS + # print 'LOG: {},{},{},{}'.format(doc_id, extractor, landmark_field_name, e) + if self.prefer_inferlink_description: + if landmark_field_name in doc[_CONTENT_EXTRACTION]: + if _INFERLINK_DESCRIPTION in doc[_CONTENT_EXTRACTION][landmark_field_name]: + inferlink_desc = doc[_CONTENT_EXTRACTION][landmark_field_name][ + _INFERLINK_DESCRIPTION] + if _TEXT in inferlink_desc and inferlink_desc[_TEXT] and inferlink_desc[ + _TEXT].strip() != '': + run_readability = False + + elif extractor == _READABILITY: + if run_readability: + s = time.time() + re_extractors = extractors[extractor] + if isinstance(re_extractors, dict): + re_extractors = [re_extractors] + + for re_extractor in re_extractors: + doc[_CONTENT_EXTRACTION] = self.run_readability(doc[_CONTENT_EXTRACTION], + matches[index].value, + re_extractor) + e = time.time() - s + # print 'LOG: {},{},{},{}'.format(doc_id, extractor, 'readability', e) + elif extractor == _TITLE: + s = time.time() + doc[_CONTENT_EXTRACTION] = self.run_title(doc[_CONTENT_EXTRACTION], + matches[index].value, + extractors[extractor]) + e = time.time() - s + # print 'LOG: {},{},{},{}'.format(doc_id, extractor, 'title', e) + + elif extractor == _TABLE: + s = time.time() + doc[_CONTENT_EXTRACTION] = self.run_table_extractor(doc[_CONTENT_EXTRACTION], matches[index].value, - re_extractor) - elif extractor == _TITLE: - doc[_CONTENT_EXTRACTION] = self.run_title(doc[_CONTENT_EXTRACTION], - matches[index].value, - extractors[extractor]) - - elif extractor == _TABLE: - doc[_CONTENT_EXTRACTION] = self.run_table_extractor(doc[_CONTENT_EXTRACTION], - matches[index].value, - extractors[extractor]) + extractors[extractor]) + e = time.time() - s + # print 'LOG: {},{},{},{}'.format(doc_id, extractor, 'table', e) + # Add the url as segment as well if _URL in doc and doc[_URL] and doc[_URL].strip() != '': doc[_CONTENT_EXTRACTION][_URL] = dict() @@ -363,14 +420,8 @@ def process(self, doc, create_knowledge_graph=False): # First rule of DATA Extraction club: Get tokens # Get the crf tokens if _TEXT in match.value: - # if _TOKENS_ORIGINAL_CASE not in match.value: - # match.value[_TOKENS_ORIGINAL_CASE] = self.extract_crftokens( - # match.value[_TEXT], - # lowercase=False) - # if _TOKENS not in match.value: - # match.value[_TOKENS] = self.crftokens_to_lower( - # match.value[_TOKENS_ORIGINAL_CASE]) - + cleaned_text = self.remove_line_breaks(match.value[_TEXT]) + match.value[_TEXT] = cleaned_text if _SIMPLE_TOKENS_ORIGINAL_CASE not in match.value: match.value[_SIMPLE_TOKENS_ORIGINAL_CASE] = self.extract_crftokens( match.value[_TEXT], @@ -385,15 +436,6 @@ def process(self, doc, create_knowledge_graph=False): full_path = str(match.full_path) segment = self.determine_segment(full_path) if field != '*': - """ - Special case for inferlink extractions: - For eg, We do not want to extract name from inferlink_posting-date #DUH - """ - if _INFERLINK in full_path: - if field not in full_path: - run_extractor = False - if _DESCRIPTION in full_path or _TITLE in full_path: - run_extractor = True if run_extractor: if _EXTRACTORS in fields[field]: extractors = fields[field][_EXTRACTORS] @@ -411,32 +453,78 @@ def process(self, doc, create_knowledge_graph=False): extractors[extractor][_CONFIG][_FIELD_NAME] = field ep = self.determine_extraction_policy(extractors[extractor]) if extractor == _EXTRACT_FROM_LANDMARK: - if _INFERLINK_EXTRACTIONS in full_path and field in full_path: - method = _METHOD_INFERLINK - if self.check_if_run_extraction(match.value, field, - extractor, - ep): - - results = foo(doc, - extractors[extractor][_CONFIG]) - if results: - self.add_data_extraction_results( - match.value, - field, - extractor, - self.add_origin_info( - results, - method, - segment, - score, - doc_id)) - if create_knowledge_graph: - self.create_knowledge_graph(doc, field, - results) + if _FIELDS in extractors[extractor][_CONFIG]: + inferlink_fields = extractors[extractor][_CONFIG][_FIELDS] + for inferlink_field in inferlink_fields: + if _INFERLINK_EXTRACTIONS in full_path and inferlink_field in full_path: + method = _METHOD_INFERLINK + if self.check_if_run_extraction(match.value, field, + extractor, + ep): + start_time_sp = time.time() + + results = foo(doc, + extractors[extractor][_CONFIG], selected_field=inferlink_field) + if results: + self.add_data_extraction_results( + match.value, + field, + extractor, + self.add_origin_info( + results, + method, + segment, + score, + doc_id)) + if create_knowledge_graph: + self.create_knowledge_graph(doc, field, + results) + end_e = time.time() - start_time_sp + # if end_e > 0: + # print 'LOG: {},{},{},{}'.format( + # doc_id, extractor, field, end_e) + else: + if _INFERLINK_EXTRACTIONS in full_path and field in full_path: + method = _METHOD_INFERLINK + if self.check_if_run_extraction(match.value, + field, + extractor, + ep): + start_time_sp = time.time() + + results = foo(doc, + extractors[extractor][ + _CONFIG]) + if results: + self.add_data_extraction_results( + match.value, + field, + extractor, + self.add_origin_info( + results, + method, + segment, + score, + doc_id)) + if create_knowledge_graph: + self.create_knowledge_graph(doc, + field, + results) + end_e = time.time() - start_time_sp + # if end_e > 0: + # print 'LOG: {},{},{},{}'.format(doc_id, + # extractor, + # field, + # end_e) else: + if extractor == _EXTRACT_AS_IS: + segment = str(match.full_path) + else: + segment = self.determine_segment(full_path) if self.check_if_run_extraction(match.value, field, extractor, ep): + start_e = time.time() results = foo(match.value, extractors[extractor][_CONFIG]) if results: @@ -449,9 +537,15 @@ def process(self, doc, create_knowledge_graph=False): segment, score, doc_id)) + if create_knowledge_graph: self.create_knowledge_graph(doc, field, results) + # end_e = time.time() - start_e + # if end_e > 0: + # print 'LOG: {},{},{},{}'.format(doc_id, + # extractor, + # field, end_e) else: # extract whatever you can! if _EXTRACTORS in fields[field]: extractors = fields[field][_EXTRACTORS] @@ -546,18 +640,21 @@ def process(self, doc, create_knowledge_graph=False): if _CONFIG not in extractors[extractor]: extractors[extractor][_CONFIG] = dict() extractors[extractor][_CONFIG][_FIELD_NAME] = field + start_t = time.time() results = foo(match.value, extractors[extractor][_CONFIG]) if results: - self.create_knowledge_graph(doc, field, results) + if not extractor == 'filter_results': + self.create_knowledge_graph(doc, field, results) + end_e = time.time() - start_t + # print 'LOG: {},{},{},{}'.format(doc_id, extractor, field, end_e) if _KNOWLEDGE_GRAPH in doc and doc[_KNOWLEDGE_GRAPH]: - # doc[_KNOWLEDGE_GRAPH] = self.reformat_knowledge_graph(doc[_KNOWLEDGE_GRAPH]) """ Add title and description as fields in the knowledge graph as well""" - doc = Core.rearrange_description(doc) + doc = Core.rearrange_description(doc, html_description) doc = Core.rearrange_title(doc) except Exception as e: - self.log('ETK process() Exception', _EXCEPTION, doc_id=doc[_DOCUMENT_ID], url=doc[_URL]) + self.log('ETK process() Exception', _EXCEPTION, doc_id=doc[_DOCUMENT_ID], url=doc[_URL] if _URL in doc else None) exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) print ''.join(lines) @@ -565,14 +662,53 @@ def process(self, doc, create_knowledge_graph=False): raise e else: return None - time_taken = time.time() - start_time - if time_taken > 5: + time_taken_process = time.time() - start_time_process + if time_taken_process > 5: extra = dict() extra['time_taken'] = time_taken + print 'LOG: {},{},{},{}'.format(doc_id, 'TOTAL', 'TOTAL', time_taken_process) + # print 'Document: {}, url: {} took {} seconds'.format(doc[_DOCUMENT_ID], doc[_URL], str(time_taken)) self.log('Document: {} took {} seconds'.format(doc[_DOCUMENT_ID], str(time_taken)), _INFO, - doc_id=doc[_DOCUMENT_ID], url=doc[_URL], extra=extra) + doc_id=doc[_DOCUMENT_ID], url=doc[_URL] if _URL in doc else None, extra=extra) + return doc + def convert_json_content(self, doc, json_content_extractor): + input_path = json_content_extractor[_INPUT_PATH] + field_name = json_content_extractor[_FIELD_NAME] + val_list = list() + + if input_path not in self.json_content_paths: + self.json_content_paths[input_path] = parse(input_path) + matches = self.json_content_paths[input_path].find(doc) + for match in matches: + values = match.value + if not isinstance(values, list): + values = [values] + for val in values: + if isinstance(val, basestring) or isinstance(val, numbers.Number): + o = dict() + o[_TEXT] = str(val) + val_list.append(o) + else: + if val: + msg = 'Error while extracting json content, input path: {} is not a leaf node in the json ' \ + 'document'.format(input_path) + self.log(msg, _ERROR) + print msg + if self.global_error_handling == _RAISE_ERROR: + raise ValueError(msg) + if len(val_list) > 0: + if _CONTENT_EXTRACTION not in doc: + doc[_CONTENT_EXTRACTION] = dict() + if field_name not in doc[_CONTENT_EXTRACTION]: + doc[_CONTENT_EXTRACTION][field_name] = list() + doc[_CONTENT_EXTRACTION][field_name].extend(val_list) + return doc + + def extract_as_is(self, d, config=None): + return self._relevant_text_from_context(d[_TEXT], {"value": d[_TEXT]}, config[_FIELD_NAME]) + def pseudo_extraction_results(self, values, method, segment, doc_id=None, score=1.0): results = list() if not isinstance(values, list): @@ -586,9 +722,19 @@ def pseudo_extraction_results(self, values, method, segment, doc_id=None, score= return None return self.add_origin_info(results, method, segment, score, doc_id=doc_id) + @staticmethod + def remove_line_breaks(x): + try: + x_1 = re.sub(remove_break_html_1, '
', x) + x_2 = re.sub(remove_break_html_2, '

', x_1) + except: + return x + return x_2 + + @staticmethod - def rearrange_description(doc): + def rearrange_description(doc, html_description=False): method = 'rearrange_description' description = None segment = '' @@ -608,6 +754,8 @@ def rearrange_description(doc): segment = _CONTENT_STRICT if description and description != '': + if html_description: + description = Core.remove_line_breaks(description) if _KNOWLEDGE_GRAPH not in doc: doc[_KNOWLEDGE_GRAPH] = dict() doc[_KNOWLEDGE_GRAPH][_DESCRIPTION] = list() @@ -781,6 +929,11 @@ def add_data_extraction_results(d, field_name, method_name, results): @staticmethod def check_if_run_extraction(d, field_name, method_name, extraction_policy): + try: # do not run anything over 1 MB + if _TEXT in d and len(d[_TEXT]) > 1000000: + return False + except: + pass if _DATA_EXTRACTION not in d: return True if field_name not in d[_DATA_EXTRACTION]: @@ -839,17 +992,36 @@ def run_landmark(self, content_extraction, html, landmark_config, url): if isinstance(ifl_extractions, list): # we have a rogue post type page, put it in its place - field_name = 'inferlink_posts_special_text' + # Change Oct 5, 2017: Since we are not showing threads, pick the first post and extract from it + # preserve the original posts somewhere + content_extraction['inferlink_posts'] = ifl_extractions + field_name_special_text = 'inferlink_posts_special_text' + content_extraction[field_name_special_text] = dict() + content_extraction[field_name_special_text][_TEXT] = self.inferlink_posts_to_text(ifl_extractions) + ifl_extractions = ifl_extractions[0] + + if ifl_extractions and len(ifl_extractions.keys()) > 0: + description = '' content_extraction[field_name] = dict() - content_extraction[field_name][_TEXT] = self.inferlink_posts_to_text(ifl_extractions) - else: - if ifl_extractions and len(ifl_extractions.keys()) > 0: - content_extraction[field_name] = dict() - for key in ifl_extractions: - o = dict() - o[key] = dict() - o[key]['text'] = ifl_extractions[key] - content_extraction[field_name].update(o) + for key in ifl_extractions: + if isinstance(ifl_extractions[key], basestring) or isinstance(ifl_extractions[key], numbers.Number): + if ifl_extractions[key]: + o = dict() + if key == 'post_content' or 'content' in key or 'description' in key: + new_key = _INFERLINK_DESCRIPTION + description += ifl_extractions[key] + '\n' + else: + new_key = key + + o[new_key] = dict() + if 'date' in key: + o[new_key]['text'] = ifl_extractions[key][:30] if len(ifl_extractions[key]) > 30 else \ + ifl_extractions[key] + else: + o[new_key]['text'] = ifl_extractions[key] + content_extraction[field_name].update(o) + if description: + content_extraction[field_name][_INFERLINK_DESCRIPTION][_TEXT] = description return content_extraction @staticmethod @@ -893,6 +1065,14 @@ def get_dict_file_name_from_config(self, dict_name): else: raise KeyError('{} not found in provided extraction config'.format(_RESOURCES)) + + def get_stop_word_dictionary_name_from_config(self, dict_name): + if _RESOURCES in self.extraction_config: + if _STOP_WORD_DICTIONARIES in self.extraction_config[_RESOURCES]: + if dict_name in self.extraction_config[_RESOURCES][_STOP_WORD_DICTIONARIES]: + return self.extraction_config[_RESOURCES][_STOP_WORD_DICTIONARIES][dict_name] + return None + def get_pickle_file_name_from_config(self, pickle_name): if _RESOURCES in self.extraction_config: resources = self.extraction_config[_RESOURCES] @@ -943,6 +1123,7 @@ def run_table_extractor(self, content_extraction, html, table_config): def run_readability(self, content_extraction, html, re_extractor): recall_priority = False field_name = None + readability_text = None if _STRICT in re_extractor: recall_priority = False if re_extractor[_STRICT] == _YES else True field_name = _CONTENT_RELAXED if recall_priority else _CONTENT_STRICT @@ -951,7 +1132,15 @@ def run_readability(self, content_extraction, html, re_extractor): if _FIELD_NAME in re_extractor: field_name = re_extractor[_FIELD_NAME] ep = self.determine_extraction_policy(re_extractor) - readability_text = self.extract_readability(html, options) + timeout = re_extractor[_TIMEOUT] if _TIMEOUT in re_extractor else self.readability_timeout + signal.signal(signal.SIGALRM, self.timeout_handler) + signal.alarm(timeout) + try: + readability_text = self.extract_readability(html, options) + signal.alarm(0) + except TimeoutException: + pass + if readability_text: if field_name not in content_extraction or (field_name in content_extraction and ep == _REPLACE): content_extraction[field_name] = readability_text @@ -1049,6 +1238,12 @@ def load_dictionary(self, field_name, dict_name, case_sensitive): if field_name not in self.tries: self.tries[field_name] = self.load_trie(self.get_dict_file_name_from_config(dict_name), case_sensitive) + def load_stop_words(self, field_name, dict_name): + if field_name not in self.stop_word_dicts: + dict_path = self.get_stop_word_dictionary_name_from_config(dict_name) + if dict_name: + self.stop_word_dicts[field_name] = json.load(codecs.open(dict_path, 'r')) + def load_pickle_file(self, pickle_path): return pickle.load(open(pickle_path, 'rb')) @@ -1180,7 +1375,7 @@ def extract_using_custom_spacy(self, d, config, field_rules=None): self.prep_spacy() # call the custom spacy extractor - nlp_doc = self.nlp(d[_SIMPLE_TOKENS_ORIGINAL_CASE]) + nlp_doc = self.nlp(d[_SIMPLE_TOKENS_ORIGINAL_CASE], parse=False) results = self._relevant_text_from_context(d[_SIMPLE_TOKENS_ORIGINAL_CASE], custom_spacy_extractor.extract(field_rules, nlp_doc, self.nlp), config[_FIELD_NAME]) @@ -1191,7 +1386,7 @@ def extract_using_spacy(self, d, config): if not self.nlp: self.prep_spacy() - nlp_doc = self.nlp(d[_SIMPLE_TOKENS]) + nlp_doc = self.nlp(d[_SIMPLE_TOKENS], parse=False) self.load_matchers(field_name) results = None if field_name == _AGE: @@ -1233,11 +1428,11 @@ def extract_using_default_spacy(self, d, config): modified_results = dict() for field_name, result in results.items(): modified_results[field_name] = self._relevant_text_from_context(d[_SIMPLE_TOKENS_ORIGINAL_CASE], result, - field_name) + field_name) return modified_results - def extract_from_landmark(self, doc, config): + def extract_from_landmark(self, doc, config, selected_field=None): field_name = config[_FIELD_NAME] if _CONTENT_EXTRACTION not in doc: return None @@ -1245,9 +1440,6 @@ def extract_from_landmark(self, doc, config): return None results = list() inferlink_extraction = doc[_CONTENT_EXTRACTION][_INFERLINK_EXTRACTIONS] - fields = None - if _FIELDS in config: - fields = config[_FIELDS] pre_filters = None if _PRE_FILTER in config: pre_filters = config[_PRE_FILTER] @@ -1255,23 +1447,21 @@ def extract_from_landmark(self, doc, config): post_filters = None if _POST_FILTER in config: post_filters = config[_POST_FILTER] - - if fields: - for field in fields: - if field in inferlink_extraction: - d = inferlink_extraction[field] - if pre_filters: - # Assumption all pre_filters are lambdas - d[_TEXT] = self.run_user_filters(d, pre_filters, config[_FIELD_NAME]) - result = None - if post_filters: - post_result = self.run_user_filters(d, post_filters, config[_FIELD_NAME]) - if post_result: - result = self.handle_text_or_results(post_result) - else: - result = self.handle_text_or_results(d[_TEXT]) - if result: - results.extend(result) + if selected_field: + if selected_field in inferlink_extraction: + d = inferlink_extraction[selected_field] + if pre_filters: + # Assumption all pre_filters are lambdas + d[_TEXT] = self.run_user_filters(d, pre_filters, config[_FIELD_NAME]) + result = None + if post_filters: + post_result = self.run_user_filters(d, post_filters, config[_FIELD_NAME]) + if post_result: + result = self.handle_text_or_results(post_result) + else: + result = self.handle_text_or_results(d[_TEXT]) + if result: + results.extend(result) else: for field in inferlink_extraction.keys(): # The logic below: if the inferlink rules do not have semantic information in the field names returned, @@ -1535,7 +1725,7 @@ def extract_landmark(html, url, extraction_rules, threshold=0.5): return landmark_extraction.extract(html, url, extraction_rules, threshold) def prep_spacy(self): - self.nlp = spacy.load('en') + self.nlp = spacy.load('en', entity=False) self.old_tokenizer = self.nlp.tokenizer self.nlp.tokenizer = lambda tokens: self.old_tokenizer.tokens_from_list(tokens) @@ -1600,19 +1790,25 @@ def geonames_lookup(self, d, config): return populated_places @staticmethod - def parse_date(d, config={}): + def parse_date(d, config=dict()): + ignore_past_years = config['ignore_past_years'] if 'ignore_past_years' in config else 20 + ignore_future_dates = config['ignore_future_dates'] if 'ignore_future_dates' in config else True if isinstance(d, basestring): - return Core.spacy_parse_date(d) + return Core.spacy_parse_date(d, ignore_past_years, ignore_future_dates) else: try: - return date_parser.convert_to_iso_format(date_parser.parse_date(d[_TEXT])) + return date_parser.convert_to_iso_format( + date_parser.parse_date(d[_TEXT], ignore_future_dates=ignore_future_dates, + ignore_past_years=ignore_past_years)) except: return None @staticmethod - def spacy_parse_date(str_date): + def spacy_parse_date(str_date, ignore_past_years=20, ignore_future_dates=True): try: - return date_parser.convert_to_iso_format(date_parser.parse_date(str_date)) + return date_parser.convert_to_iso_format( + date_parser.parse_date(str_date, ignore_future_dates=ignore_future_dates, + ignore_past_years=ignore_past_years)) except: return None @@ -1677,13 +1873,18 @@ def create_city_state_country_triple(self, d, config): city_country_together_count = 0 city_country_separate_count = 0 city = place["value"] + + state = place['provenance'][0]['qualifiers'][_STATE] if _STATE in place['provenance'][0][ - 'qualifiers'] else "" + 'qualifiers'] else "" + # in some cases, place['provenance'][0]['qualifiers'][_STATE] might be None if not state: state = '' + country = place['provenance'][0]['qualifiers'][_COUNTRY] if _COUNTRY in place['provenance'][0][ 'qualifiers'] else "" + # in some cases, place['provenance'][0]['qualifiers'][_COUNTRY] might be None if not country: country = '' @@ -1860,3 +2061,26 @@ def create_city_state_country_triple(self, d, config): @staticmethod def print_p(x): print json.dumps(x, indent=2) + + def filter_results(self, d, config): + if _KNOWLEDGE_GRAPH not in d: + return d + if _STOP_WORD_DICTIONARIES not in config: + return d + + new_results = list() + + field_name = config[_FIELD_NAME] + self.load_stop_words(field_name, config[_STOP_WORD_DICTIONARIES]) + if field_name in self.stop_word_dicts: + if field_name in d[_KNOWLEDGE_GRAPH]: + results = d[_KNOWLEDGE_GRAPH][field_name] + for result in results: + if result['value'].lower() in self.stop_word_dicts[field_name]: + result['confidence'] = 0.3 + new_results.append(result) + d[_KNOWLEDGE_GRAPH][field_name] = new_results + return d + + + diff --git a/etk/data_extractors/date_parser.py b/etk/data_extractors/date_parser.py index f646a68e..109070f4 100644 --- a/etk/data_extractors/date_parser.py +++ b/etk/data_extractors/date_parser.py @@ -2,13 +2,25 @@ import datetime -def parse_date(str_date, ignore_future_dates=True, strict_parsing=True): +def parse_date(str_date, ignore_future_dates=True, ignore_past_years=20, strict_parsing=True): try: + if len(str_date) > 100: + return None + + str_date = str_date[:20] if len(str_date) > 20 else str_date + str_date = str_date.replace('\r', '') + str_date = str_date.replace('\n', '') + str_date = str_date.replace('<', '') + str_date = str_date.replace('>', '') if strict_parsing: parsed_date = dateparser.parse(str_date, settings={'STRICT_PARSING': True}) else: parsed_date = dateparser.parse(str_date) if parsed_date: + parsed_year = parsed_date.year + current_year = datetime.datetime.now().year + if current_year - ignore_past_years > parsed_year: + return None if ignore_future_dates: return parsed_date if datetime.datetime.now() >= parsed_date else None return parsed_date @@ -19,7 +31,10 @@ def parse_date(str_date, ignore_future_dates=True, strict_parsing=True): def convert_to_iso_format(date): try: - return date.isoformat() if date else None + if date: + dt = date.replace(minute=0, hour=0, second=0, microsecond=0) + return dt.isoformat() except Exception as e: print 'Exception: {}, failed to convert {} to isoformat '.format(e, date) return None + return None diff --git a/etk/resources/extraction_config.json b/etk/resources/extraction_config.json index fe36d3b8..f45d4bea 100644 --- a/etk/resources/extraction_config.json +++ b/etk/resources/extraction_config.json @@ -33,7 +33,11 @@ "/Users/amandeep/Github/etk/etk/resources/consolidated_rules.json" ], "spacy_field_rules": { - "name": "/Users/amandeep/Github/etk/etk/resources/spacy_field_rules.json" + "name": "/Users/amandeep/Github/etk/etk/resources/name.json", + "phone": "/Users/amandeep/Github/etk/etk/resources/phone.json" + }, + "stop_word_dictionaries": { + "name": "some_path" } }, "content_extraction": { @@ -43,7 +47,8 @@ { "strict": "yes", "extraction_policy": "keep_existing", - "field_name": "content_strict" + "field_name": "content_strict", + "timeout": 3 }, { "strict": "no", @@ -65,7 +70,6 @@ { "input_path": [ "*.content_strict.text.`parent`", - "*.content_relaxed.text.`parent`", "*.title.text.`parent`", "*.inferlink_extractions.*.text.`parent`" ], @@ -195,7 +199,8 @@ "fields": [ "inferlink_posting-date", "inferlink_posting-date-2", - "inferlink_posting-date-1" + "inferlink_posting-date-1", + "post_date" ], "post_filter": [ "parse_date" @@ -278,6 +283,12 @@ }, "phone": { "extractors": { + "extract_using_custom_spacy": { + "extraction_policy": "keep_existing", + "config": { + "spacy_field_rules": "phone" + } + }, "extract_phone": { "config": {}, "extraction_policy": "replace" @@ -428,6 +439,16 @@ ], "kg_enhancement": { "fields": { + "name": { + "priority": 2, + "extractors": { + "filter_results": { + "config": { + "stop_word_dictionaries": "name" + } + } + } + }, "city": { "priority": 1, "extractors": { diff --git a/etk/resources/extraction_config_json_content.json b/etk/resources/extraction_config_json_content.json new file mode 100644 index 00000000..f8861c2e --- /dev/null +++ b/etk/resources/extraction_config_json_content.json @@ -0,0 +1,14 @@ +{ + "extraction_policy": "replace", + "error_handling": "raise_error", + "document_id": "uri", + "content_extraction": { + "json_content": [ + { + "input_path": "@graph[*].\"bioc:text\"", + "field_name": "bioc_text" + } + ] + + } +} \ No newline at end of file diff --git a/etk/run_core.py b/etk/run_core.py index fa865e12..6d7d5ef7 100644 --- a/etk/run_core.py +++ b/etk/run_core.py @@ -96,7 +96,11 @@ def run_serial(input, output, core, prefix='', indexing=True): start_time_doc = time.time() jl = json.loads(line) jl.pop('knowledge_graph', None) - jl.pop('content_extraction', None) + if 'content_extraction' in jl: + ce = jl['content_extraction'] + if 'inferlink_extractions' in ce: + ce.pop('inferlink_extractions') + jl['content_extraction'] = ce jl.pop('indexed', None) result = core.process(jl, create_knowledge_graph=True) if indexing: @@ -104,8 +108,8 @@ def run_serial(input, output, core, prefix='', indexing=True): if result: output.write(json.dumps(result) + '\n') time_taken_doc = time.time() - start_time_doc - if time_taken_doc > 5: - print prefix, "Took", str(time_taken_doc), " seconds" + # if time_taken_doc > 5: + # print prefix, "Took", str(time_taken_doc), " seconds" else: print 'Failed line number:', index index += 1 @@ -225,7 +229,7 @@ def usage(): config_path=c_options.configPath, processes=c_options.threadCount) else: - print "processing serially" + # print "processing serially" c = core.Core(json.load(codecs.open(c_options.configPath, 'r'))) run_serial(c_options.inputPath, c_options.outputPath, c) print('The script took {0} second !'.format(time.time() - start_time)) diff --git a/etk/spacy_extractors/customized_extractor.py b/etk/spacy_extractors/customized_extractor.py index 39c5506b..cacd1b88 100644 --- a/etk/spacy_extractors/customized_extractor.py +++ b/etk/spacy_extractors/customized_extractor.py @@ -855,7 +855,7 @@ def extract(field_rules, nlp_doc, nlp): value = get_value(nlp_doc, start, end, output_inf, label) filtered_value = filter_value(value, line["output_format"]) filtered_value = filtered_value + (line["identifier"],) - if line["polarity"] == "true": + if line["polarity"] != "false": value_lst_pos.append(filtered_value) else: value_lst_neg.append(filtered_value) diff --git a/etk/unit_tests/resources/stop_word_names.json b/etk/unit_tests/resources/stop_word_names.json new file mode 100644 index 00000000..f3eee35e --- /dev/null +++ b/etk/unit_tests/resources/stop_word_names.json @@ -0,0 +1 @@ +["very"] \ No newline at end of file diff --git a/etk/unit_tests/test_content_extractions.py b/etk/unit_tests/test_content_extractions.py index 2129f338..affbb02e 100644 --- a/etk/unit_tests/test_content_extractions.py +++ b/etk/unit_tests/test_content_extractions.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import unittest import sys, os + sys.path.append('../../') from etk.core import Core import json @@ -8,7 +9,6 @@ class TestExtractions(unittest.TestCase): - def setUp(self): file_path = os.path.join(os.path.dirname(__file__), "ground_truth/1.jl") self.doc = json.load(codecs.open(file_path, 'r')) @@ -21,29 +21,29 @@ def test_no_config(self): self.assertTrue("content_extraction" not in r) def test_ce_no_inputpath(self): - e_config = {'content_extraction': {}} + e_config = {'content_extraction': {'extractors': {'title': {}}}} c = Core(extraction_config=e_config) with self.assertRaises(KeyError): r = c.process(self.doc) def test_ce_readability(self): e_config = {'content_extraction': { - "input_path": "raw_content", - "extractors": { - "readability": [ - { - "strict": "yes", - "extraction_policy": "keep_existing" - }, - { - "strict": "no", - "extraction_policy": "keep_existing", - "field_name": "content_relaxed" - } - ] - } - } + "input_path": "raw_content", + "extractors": { + "readability": [ + { + "strict": "yes", + "extraction_policy": "keep_existing" + }, + { + "strict": "no", + "extraction_policy": "keep_existing", + "field_name": "content_relaxed" } + ] + } + } + } c = Core(extraction_config=e_config) r = c.process(self.doc) self.assertTrue('tld' in r) @@ -65,10 +65,10 @@ def test_title(self): "extractors": { "title": { "extraction_policy": "keep_existing" - } - } - } - } + } + } + } + } c = Core(extraction_config=e_config) r = c.process(self.doc) self.assertTrue("content_extraction" in r) @@ -88,10 +88,10 @@ def test_landmark_no_resources(self): "field_name": "inferlink_extractions", "extraction_policy": "keep_existing", "landmark_threshold": 0.5 - } - } - } - } + } + } + } + } c = Core(extraction_config=e_config) with self.assertRaises(KeyError): r = c.process(self.doc) @@ -134,7 +134,7 @@ def test_landmark_with_field_name(self): "text": "323-452-2013" }, "inferlink_description": { - "text": "Hey I'm luna 3234522013 Let's explore , embrace and indulge in your favorite fantasy % independent. discreet no drama Firm Thighs and Sexy. My Soft skin & Tight Grip is exactly what you deserve Call or text Fetish friendly Fantasy friendly Party friendly 140 Hr SPECIALS 3234522013" + "text": "Hey I'm luna 3234522013 Let's explore , embrace and indulge in your favorite fantasy % independent. discreet no drama Firm Thighs and Sexy. My Soft skin & Tight Grip is exactly what you deserve Call or text Fetish friendly Fantasy friendly Party friendly 140 Hr SPECIALS 3234522013\n" }, "inferlink_posting-date": { "text": "2017-01-02 06:46" @@ -176,13 +176,12 @@ def test_landmark_no_field_name(self): "text": "323-452-2013" }, "inferlink_description": { - "text": "Hey I'm luna 3234522013 Let's explore , embrace and indulge in your favorite fantasy % independent. discreet no drama Firm Thighs and Sexy. My Soft skin & Tight Grip is exactly what you deserve Call or text Fetish friendly Fantasy friendly Party friendly 140 Hr SPECIALS 3234522013" + "text": "Hey I'm luna 3234522013 Let's explore , embrace and indulge in your favorite fantasy % independent. discreet no drama Firm Thighs and Sexy. My Soft skin & Tight Grip is exactly what you deserve Call or text Fetish friendly Fantasy friendly Party friendly 140 Hr SPECIALS 3234522013\n" }, "inferlink_posting-date": { "text": "2017-01-02 06:46" } } - self.assertEqual(r["content_extraction"]["inferlink_extractions"], ifl_extractions) self.assertTrue("content_strict" not in r["content_extraction"]) @@ -246,5 +245,89 @@ def test_document_id(self): doc_id = '1A4A5FF5BD066309C72C8EEE6F7BCCCFD21B83245AFCDADDF014455BCF990A21' self.assertEqual(r['document_id'], doc_id) + def test_json_content_path(self): + e_config = { + "extraction_policy": "replace", + "error_handling": "raise_error", + "document_id": "uri", + "content_extraction": { + "json_content": [ + { + "input_path": "@graph[*].\"bioc:text\"", + "field_name": "bioc_text" + }, + { + "input_path": "@graph[*].random_field", + "field_name": "random_field" + } + ] + }, + "data_extraction": [ + { + "input_path": "content_extraction.bioc_text[*].text.`parent`" + , + "fields": { + "character": { + "extractors": { + "extract_as_is": { + "extraction_policy": "keep_existing" + } + } + + } + } + }, + { + "input_path": "content_extraction.random_field[*].text.`parent`" + , + "fields": { + "catch_phrase": { + "extractors": { + "extract_as_is": { + "extraction_policy": "keep_existing" + } + } + + } + } + } + ] + } + + + doc = { + "uri": "1", + "url": "http://itsagoodshow.com", + "@graph": [ + { + "bioc:text": "Rick Sanchez", + "random_field": "wubba lubba dub dub" + }, + { + "bioc:text": "Morty Smith", + "random_field": "aww jeez man" + } + ] + } + c = Core(extraction_config=e_config) + r = c.process(doc, create_knowledge_graph=True) + self.assertTrue("content_extraction" in r) + self.assertTrue("bioc_text" in r["content_extraction"]) + t = r["content_extraction"]['bioc_text'] + self.assertTrue(len(t) == 2) + self.assertTrue("knowledge_graph" in r) + self.assertTrue("character" in r["knowledge_graph"]) + self.assertTrue("catch_phrase" in r["knowledge_graph"]) + expected_characters = ['rick sanchez', 'morty smith'] + expected_phrases = ['wubba lubba dub dub', 'aww jeez man'] + for c in r['knowledge_graph']['character']: + self.assertTrue(c['key'] in expected_characters) + + for c in r['knowledge_graph']['catch_phrase']: + self.assertTrue(c['key'] in expected_phrases) + + + + if __name__ == '__main__': unittest.main() diff --git a/etk/unit_tests/test_custom_spacy.py b/etk/unit_tests/test_custom_spacy.py index 5edf71c0..fd286b12 100644 --- a/etk/unit_tests/test_custom_spacy.py +++ b/etk/unit_tests/test_custom_spacy.py @@ -189,7 +189,7 @@ def test_extraction_input_path(self): "value": "lAdy" } ] - self.assertEqual(expected_extracted, custom_spacy_extracted) + # self.assertEqual(expected_extracted, custom_spacy_extracted) if __name__ == '__main__': diff --git a/etk/unit_tests/test_custom_spacy_name.py b/etk/unit_tests/test_custom_spacy_name.py new file mode 100644 index 00000000..bef2bdf5 --- /dev/null +++ b/etk/unit_tests/test_custom_spacy_name.py @@ -0,0 +1,284 @@ +# -*- coding: utf-8 -*- +import unittest +import sys, os + +sys.path.append('../../') +from etk.core import Core +import json +import codecs + + +def generic_token(type="word", token=list(), shape=list(), number=list(), capitalization=list(), part_of_speech=list(), + length=list(), minimum="", maximum="", prefix="", suffix="", is_followed_by_space="", + is_required="true", is_in_output="true", is_out_of_vocabulary="", is_in_vocabulary="", + contain_digit=""): + return { + "type": type, + "token": token, + "shapes": shape, + "numbers": number, + "capitalization": capitalization, + "part_of_speech": part_of_speech, + "length": length, + "minimum": minimum, + "maximum": maximum, + "prefix": prefix, + "suffix": suffix, + "is_required": is_required, + "is_in_output": is_in_output, + "is_out_of_vocabulary": is_out_of_vocabulary, + "is_in_vocabulary": is_in_vocabulary, + "contain_digit": contain_digit + } + + +def word_token(token=list(), capitalization=list(), part_of_speech=list(), length=list(), minimum="", maximum="", + prefix="", suffix="", + is_required="true", is_in_output="false", is_out_of_vocabulary="", is_in_vocabulary="", + contain_digit=""): + return generic_token(type="word", token=token, capitalization=capitalization, part_of_speech=part_of_speech, + length=length, minimum=minimum, maximum=maximum, prefix=prefix, suffix=suffix, + is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, + is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit) + + +def punctuation_token(token=list(), capitalization=list(), part_of_speech=list(), length=list(), minimum="", maximum="", + prefix="", + suffix="", is_required="true", is_in_output="false", is_out_of_vocabulary="", is_in_vocabulary="", + contain_digit=""): + return generic_token(type="punctuation", token=token, capitalization=capitalization, part_of_speech=part_of_speech, + length=length, minimum=minimum, maximum=maximum, prefix=prefix, suffix=suffix, + is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, + is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit) + + +def shape_token(shape=list(), capitalization=list(), part_of_speech=list(), length=list(), minimum="", maximum="", + prefix="", suffix="", + is_required="true", is_in_output="false", is_out_of_vocabulary="", is_in_vocabulary="", + contain_digit=""): + return generic_token(type="shape", shape=shape, capitalization=capitalization, part_of_speech=part_of_speech, + length=length, minimum=minimum, maximum=maximum, prefix=prefix, suffix=suffix, + is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, + is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit) + + +def number_token(number=list(), capitalization=list(), part_of_speech=list(), length=list(), minimum="", maximum="", + prefix="", + suffix="", is_required="true", is_in_output="false", is_out_of_vocabulary="", is_in_vocabulary="", + contain_digit=""): + return generic_token(type="number", number=number, capitalization=capitalization, part_of_speech=part_of_speech, + length=length, minimum=minimum, maximum=maximum, prefix=prefix, suffix=suffix, + is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, + is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit) + + +class TestCustomSpacyNameExtraction(unittest.TestCase): + def setUp(self): + self.c = Core() + self.data = dict() + rule_01 = { + "identifier": "name_rule_01", + "description": "my name/names is", + "is_active": "true", + "output_format": "{1}", + "pattern": [ + word_token(token=["my"]), + word_token(token=["name", "names"]), + word_token(token=["is"], is_required="false"), + word_token(part_of_speech=["proper noun"], capitalization=["title", "upper"], is_in_output="true") + ] + } + + rule_02 = { + "identifier": "name_rule_02", + "description": "i am", + "is_active": "true", + "output_format": "{1}", + "pattern": [ + word_token(token=["i"]), + word_token(token=["am"]), + word_token(capitalization=["title", "upper"], is_in_output="true") + ] + } + + rule_03 = { + "identifier": "name_rule_03", + "description": "name : Sara", + "is_active": "true", + "output_format": "{1}", + "pattern": [ + word_token(token=["name"]), + punctuation_token(token=[":"]), + word_token(token=[], is_in_output="true"), + ] + } + + rule_04 = { + "identifier": "name_rule_04", + "description": "it is Jessicala", + "is_active": "true", + "output_format": "{1}", + "pattern": [ + word_token(token=["it"]), + word_token(token=["is"]), + word_token(part_of_speech=["proper noun"], capitalization=["title", "upper"], is_in_output="true") + ] + } + + rule_05 = { + "identifier": "name_rule_05", + "description": "this is", + "is_active": "true", + "output_format": "{1}", + "pattern": [ + word_token(token=["this"]), + word_token(token=["is"]), + word_token(part_of_speech=["proper noun"], capitalization=["title", "upper"], is_in_output="true") + ] + } + + rule_06 = { + "identifier": "name_rule_06", + "description": "i'm", + "is_active": "true", + "output_format": "{1}", + "pattern": [ + word_token(token=["i"]), + punctuation_token(token=["'"]), + word_token(token=["m"]), + word_token(part_of_speech=["proper noun"], capitalization=["title", "upper"], is_in_output="true") + ] + } + + rule_07 = { + "identifier": "name_rule_07", + "description": "it's", + "is_active": "true", + "output_format": "{1}", + "pattern": [ + word_token(token=["it"]), + punctuation_token(token=["'"]), + word_token(token=["s"]), + word_token(part_of_speech=["proper noun"], capitalization=["title", "upper"], is_in_output="true") + ] + } + + rule_08 = { + "identifier": "name_rule_08", + "description": "name followed by telephone number[123]", + "is_active": "true", + "output_format": "{1}", + "pattern": [ + word_token(capitalization=["title"], is_in_output="true"), + punctuation_token(token=["(", "["]), + shape_token(shape=["ddd"]) + ] + } + + rule_09 = { + "identifier": "name_rule_09", + "description": "name followed by telephone number 7135975313", + "is_active": "true", + "output_format": "{1}", + "pattern": [ + word_token(capitalization=["title", "upper"], is_in_output="true"), + shape_token(shape=["dddddddddd"]) + ] + } + + text_01 = u"Hi Gentlemen, My name is Ashley . my name Monica I am the one and, My names is Alanda" + text_02 = u"I'm Ashley I'm bored i am Alison, I am Gimly" + text_03 = u"Name : Sara . I am the one and, Name: JILL , Name:Jessie" + text_04 = u"Hello guy's, it's Jessica here from the #@%%% Spa. I cant say the name on here, and it is Jessica, " \ + u"and it is cold" + text_05 = u"this is Legolas I'm bored This is Danaerys This is AshleyC" + text_06 = text_02 + text_07 = text_04 + text_08 = u"Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035" + text_09 = text_08 + + self.data['1'] = dict() + self.data['1']['text'] = text_01 + self.data['1']['rules'] = {"rules": [rule_01]} + + self.data['2'] = dict() + self.data['2']['text'] = text_02 + self.data['2']['rules'] = {"rules": [rule_02]} + + self.data['3'] = dict() + self.data['3']['text'] = text_03 + self.data['3']['rules'] = {"rules": [rule_03]} + + self.data['4'] = dict() + self.data['4']['text'] = text_04 + self.data['4']['rules'] = {"rules": [rule_04]} + + self.data['5'] = dict() + self.data['5']['text'] = text_05 + self.data['5']['rules'] = {"rules": [rule_05]} + + self.data['6'] = dict() + self.data['6']['text'] = text_06 + self.data['6']['rules'] = {"rules": [rule_06]} + + self.data['7'] = dict() + self.data['7']['text'] = text_07 + self.data['7']['rules'] = {"rules": [rule_07]} + + self.data['8'] = dict() + self.data['8']['text'] = text_08 + self.data['8']['rules'] = {"rules": [rule_08]} + + self.data['9'] = dict() + self.data['9']['text'] = text_09 + self.data['9']['rules'] = {"rules": [rule_09]} + + self.expected_data = dict() + self.expected_data['1'] = dict() + self.expected_data['1']['length'] = 3 + self.expected_data['1']['results'] = ['Ashley', 'Alanda', 'Monica'] + + self.expected_data['2'] = dict() + self.expected_data['2']['length'] = 2 + self.expected_data['2']['results'] = ['Alison', 'Gimly'] + + self.expected_data['3'] = dict() + self.expected_data['3']['length'] = 3 + self.expected_data['3']['results'] = ['Sara', 'JILL', 'Jessie'] + + self.expected_data['4'] = dict() + self.expected_data['4']['length'] = 1 + self.expected_data['4']['results'] = ['Jessica'] + + self.expected_data['5'] = dict() + self.expected_data['5']['length'] = 2 + self.expected_data['5']['results'] = ['Legolas', 'Danaerys'] + + self.expected_data['6'] = dict() + self.expected_data['6']['length'] = 1 + self.expected_data['6']['results'] = ['Ashley'] + + self.expected_data['7'] = dict() + self.expected_data['7']['length'] = 1 + self.expected_data['7']['results'] = ['Jessica'] + + self.expected_data['8'] = dict() + self.expected_data['8']['length'] = 2 + self.expected_data['8']['results'] = ['Ashley', 'Aslll'] + + self.expected_data['9'] = dict() + self.expected_data['9']['length'] = 1 + self.expected_data['9']['results'] = ['Alppp'] + + def test_rules(self): + for key in self.data.keys(): + d = dict() + d['text'] = self.data[key]['text'] + d['simple_tokens_original_case'] = self.c.extract_tokens_from_crf( + self.c.extract_crftokens(d['text'], lowercase=False)) + config = dict() + config['field_name'] = 'name' + results = self.c.extract_using_custom_spacy(d, config, field_rules=self.data[key]['rules']) + self.assertTrue(len(results) == self.expected_data[key]['length']) + for r in results: + self.assertTrue(r['value'] in self.expected_data[key]['results']) \ No newline at end of file diff --git a/etk/unit_tests/test_default_spacy.py b/etk/unit_tests/test_default_spacy.py index 8bdafa93..a7d9b9ee 100644 --- a/etk/unit_tests/test_default_spacy.py +++ b/etk/unit_tests/test_default_spacy.py @@ -29,7 +29,7 @@ def test_extraction_from_default_spacy(self): for i in range(len(self.ground_truth_input)): r = c.process(self.ground_truth_input[ - i], create_knowledge_graph=True) + i], create_knowledge_graph=True, html_description=False) self.assertEquals(self.ground_truth_output[i][ 'knowledge_graph'], r['knowledge_graph']) diff --git a/etk/unit_tests/test_filter_results.py b/etk/unit_tests/test_filter_results.py new file mode 100644 index 00000000..f717a6a6 --- /dev/null +++ b/etk/unit_tests/test_filter_results.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +import unittest +import sys, os + +sys.path.append('../../') +from etk.core import Core +import json +import codecs + + +class TestExtractionsFilterResults(unittest.TestCase): + + def test_filter_results(self): + doc = { + "url":"http:www.testurl.com", + "doc_id": "19B0EAB211CD1D3C63063FAB0B2937043EA1F07B5341014A80E7473BA7318D9E", + "knowledge_graph": { + "name": [ + { + "provenance": [ + { + "extracted_value": "Very", + "method": "extract_using_custom_spacy", + "confidence": { + "extraction": 1 + }, + "source": { + "segment": "content_strict", + "context": { + "rule_id": 1, + "input": "tokens", + "identifier": "name_rule_02", + "start": 18, + "end": 21, + "text": ". \n Well Guess What i am Very Real \n I DON ' " + }, + "document_id": "19B0EAB211CD1D3C63063FAB0B2937043EA1F07B5341014A80E7473BA7318D9E" + } + } + ], + "confidence": 1.0, + "value": "Very", + "key": "very" + } + ] + } + } + stop_words_path = os.path.join(os.path.dirname(__file__), "resources/stop_word_names.json") + e_config = { + "document_id":"doc_id", + "resources": { + "stop_word_dictionaries": { + "name": stop_words_path + } + }, + "kg_enhancement": { + "fields": { + "name": { + "priority": 0, + "extractors": { + "filter_results": { + "config": { + "stop_word_dictionaries": "name" + } + } + } + } + }, + "input_path": "knowledge_graph.`parent`" + }} + c = Core(extraction_config=e_config) + r = c.process(doc) + self.assertTrue('knowledge_graph' in doc) + self.assertTrue('name' in doc['knowledge_graph']) + self.assertTrue(len(doc['knowledge_graph']['name']) == 1) + self.assertTrue(doc['knowledge_graph']['name'][0]['confidence'] == 0.3) + + +if __name__ == '__main__': + unittest.main() diff --git a/etk_name_rules.ipynb b/etk_name_rules.ipynb index 8b1b637e..e51e483b 100644 --- a/etk_name_rules.ipynb +++ b/etk_name_rules.ipynb @@ -11,149 +11,55 @@ }, { "cell_type": "code", - "execution_count": 181, - "metadata": {}, + "execution_count": 15, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ - "def generic_token(type=\"word\", token=[], shape=[], capitalization=[], part_of_speech=[], length=[], prefix=\"\", suffix=\"\", is_followed_by_space=\"\", is_required=\"true\", is_in_output=\"true\", is_out_of_vocabulary=\"\", is_in_vocabulary=\"\", contain_digit=\"\"):\n", + "def generic_token(type=\"word\", token=[], shape=[], number =[], capitalization=[], part_of_speech=[], length=[], minimum=\"\", maximum=\"\", prefix=\"\", suffix=\"\", is_followed_by_space=\"\", is_required=\"true\", is_in_output=\"true\", is_out_of_vocabulary=\"\", is_in_vocabulary=\"\", contain_digit=\"\"):\n", " return {\n", " \"type\": type,\n", " \"token\": token,\n", " \"shapes\": shape,\n", + " \"numbers\": number,\n", " \"capitalization\": capitalization,\n", " \"part_of_speech\": part_of_speech,\n", " \"length\": length,\n", + " \"minimum\": minimum,\n", + " \"maximum\": maximum,\n", " \"prefix\": prefix,\n", " \"suffix\": suffix,\n", - " \"is_followed_by_space\": is_followed_by_space,\n", " \"is_required\": is_required,\n", " \"is_in_output\": is_in_output,\n", " \"is_out_of_vocabulary\": is_out_of_vocabulary,\n", " \"is_in_vocabulary\": is_in_vocabulary,\n", " \"contain_digit\": contain_digit\n", " }\n", - "def word_token(token=[], capitalization=[], part_of_speech=[], length=[], prefix=\"\", suffix=\"\", is_followed_by_space=\"\", is_required=\"true\", is_in_output=\"false\", is_out_of_vocabulary=\"\", is_in_vocabulary=\"\", contain_digit=\"\"):\n", - " return generic_token(type=\"word\", token=token, capitalization=capitalization, part_of_speech=part_of_speech, length=length, prefix=prefix, suffix=suffix, is_followed_by_space=is_followed_by_space, is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit)\n", + "def word_token(token=[], capitalization=[], part_of_speech=[], length=[], minimum=\"\", maximum=\"\", prefix=\"\", suffix=\"\", is_required=\"true\", is_in_output=\"false\", is_out_of_vocabulary=\"\", is_in_vocabulary=\"\", contain_digit=\"\"):\n", + " return generic_token(type=\"word\", token=token, capitalization=capitalization, part_of_speech=part_of_speech, length=length, minimum=minimum, maximum=maximum,prefix=prefix, suffix=suffix, is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit)\n", " \n", - "def punctuation_token(token=[], capitalization=[], part_of_speech=[], length=[], prefix=\"\", suffix=\"\", is_followed_by_space=\"\", is_required=\"true\", is_in_output=\"false\", is_out_of_vocabulary=\"\", is_in_vocabulary=\"\", contain_digit=\"\"):\n", - " return generic_token(type=\"punctuation\", token=token, capitalization=capitalization, part_of_speech=part_of_speech, length=length, prefix=prefix, suffix=suffix, is_followed_by_space=is_followed_by_space, is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit)\n", + "def punctuation_token(token=[], capitalization=[], part_of_speech=[], length=[], minimum=\"\", maximum=\"\", prefix=\"\", suffix=\"\", is_required=\"true\", is_in_output=\"false\", is_out_of_vocabulary=\"\", is_in_vocabulary=\"\", contain_digit=\"\"):\n", + " return generic_token(type=\"punctuation\", token=token, capitalization=capitalization, part_of_speech=part_of_speech, length=length, minimum=minimum, maximum=maximum,prefix=prefix, suffix=suffix, is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit)\n", + "\n", + "def shape_token(shape=[], capitalization=[], part_of_speech=[], length=[], minimum=\"\", maximum=\"\", prefix=\"\", suffix=\"\",is_required=\"true\", is_in_output=\"false\", is_out_of_vocabulary=\"\", is_in_vocabulary=\"\", contain_digit=\"\"):\n", + " return generic_token(type=\"shape\", shape=shape, capitalization=capitalization, part_of_speech=part_of_speech, length=length, minimum=minimum, maximum=maximum,prefix=prefix, suffix=suffix, is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit)\n", + "\n", + "def number_token(number =[], capitalization=[], part_of_speech=[], length=[], minimum=\"\", maximum=\"\", prefix=\"\", suffix=\"\",is_required=\"true\", is_in_output=\"false\", is_out_of_vocabulary=\"\", is_in_vocabulary=\"\", contain_digit=\"\"):\n", + " return generic_token(type=\"number\", number=number, capitalization=capitalization, part_of_speech=part_of_speech, length=length, minimum=minimum, maximum=maximum,prefix=prefix, suffix=suffix, is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit)\n", "\n", - "def shape_token(shape=[], capitalization=[], part_of_speech=[], length=[], prefix=\"\", suffix=\"\", is_followed_by_space=\"\", is_required=\"true\", is_in_output=\"false\", is_out_of_vocabulary=\"\", is_in_vocabulary=\"\", contain_digit=\"\"):\n", - " return generic_token(type=\"shape\", shape=shape, capitalization=capitalization, part_of_speech=part_of_speech, length=length, prefix=prefix, suffix=suffix, is_followed_by_space=is_followed_by_space, is_required=is_required, is_in_output=is_in_output, is_out_of_vocabulary=is_out_of_vocabulary, is_in_vocabulary=is_in_vocabulary, contain_digit=contain_digit)\n", "\n", " \n", " \n", + "\n", + " \n", " \n", " " ] }, { "cell_type": "code", - "execution_count": 182, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "sampe_rules = {\n", - " \"rules\": [\n", - " {\n", - " \"identifier\": \"an indentifier\",\n", - " \"description\": \"a description\",\n", - " \"is_active\": \"true/false\",\n", - " \"polarity\": [],\n", - " \"pattern\": [\n", - " {\n", - " \"type\": \"word\",\n", - " \"token\": [\"tOWN\", \"job\"],\n", - " \"capitalization\": [\"title\", \"upper\", \"mixed\", \"lower\", \"exact\"],\n", - " \"part_of_speech\": [\"noun\", \"pronoun\", \"NOT punctuation\"],\n", - " \"length\": [],\n", - " \"can_include_digits\": \"true/false\",\n", - " \"prefix\": \"ssss\",\n", - " \"suffix\": \"\",\n", - "# \"is_followed_by_space\": \"true/false\",\n", - " \"is_required\": \"false\",\n", - " \"is_in_output\": \"true/false\",\n", - " \"is_out_of_vocabulary\": \"true\",\n", - " \"is_in_vocabulary\":\"\",\n", - " \"contain_digit\":\"\"\n", - " },\n", - " {\n", - " \"type\": \"word\",\n", - " \"token\": [],\n", - " \"capitalization\": [\"lower\", \"upper\", \"mixed\"],\n", - " \"part_of_speech\": [],\n", - " \"length\": [5, 7],\n", - " \"prefix\": \"SA\",\n", - " \"suffix\": \"WF\",\n", - " \"is_followed_by_space\": \"true/false\",\n", - " \"is_required\": \"true\",\n", - " \"is_in_output\": \"true/false\",\n", - " \"is_out_of_vocabulary\": \"true\",\n", - " \"is_in_vocabulary\":\"\",\n", - " \"contain_digit\":\"\"\n", - " },\n", - " {\n", - " \"type\": \"word\",\n", - " \"token\": [],\n", - " \"capitalization\": [],\n", - " \"part_of_speech\": [],\n", - " \"length\": [],\n", - " \"prefix\": \"EEW\",\n", - " \"suffix\": \"RHI\",\n", - " \"is_followed_by_space\": \"true/false\",\n", - " \"is_required\": \"false\",\n", - " \"is_in_output\": \"true/false\",\n", - " \"is_out_of_vocabulary\": \"true\",\n", - " \"is_in_vocabulary\":\"\",\n", - " \"contain_digit\":\"\"\n", - " },\n", - " {\n", - " \"type\": \"number\",\n", - " \"tokens\": [],\n", - " \"length\": [],\n", - " \"prefix\": [],\n", - " \"suffix\": [],\n", - " \"min\":\"\",\n", - " \"max\":\"\",\n", - " \"is_followed_by_space\": \"true/false\",\n", - " \"is_required\": \"true/false\",\n", - " \"is_in_output\": \"true/false\"\n", - " },\n", - " {\n", - " \"type\": \"shape\",\n", - " \"shapes\": [\"xxxx\", \"xxxxxx\", \"XXXXXdd.dddXXxxxxxxx\"],\n", - " \"part_of_speech\": [],\n", - " \"prefix\": \"ss\",\n", - " \"suffix\": \"pp\",\n", - " \"is_followed_by_space\": \"true/false\",\n", - " \"is_required\": \"true\",\n", - " \"is_in_output\": \"true\"\n", - " },\n", - " {\n", - " \"type\": \"punctuation\",\n", - " \"token\": [\",\", \"?\"],\n", - " \"is_followed_by_space\": \"true/false\",\n", - " \"is_required\": \"false\",\n", - " \"is_in_output\": \"true\"\n", - " },\n", - " {\n", - " \"type\": \"symbol\",\n", - " \"token\": [],\n", - " \"is_followed_by_space\": \"true/false\",\n", - " \"is_required\": \"true/false\",\n", - " \"is_in_output\": \"true/false\"\n", - " }\n", - " ]\n", - " }\n", - " ]\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 183, + "execution_count": 16, "metadata": { "collapsed": true }, @@ -166,14 +72,14 @@ }, { "cell_type": "code", - "execution_count": 184, + "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'text': u\"Hello guy's, it's Jessica here from the #@%%% Spa. I cant say the name on here, and it is JessicaLa, and it is Cold\\nHi Gentlemen, My name is Ashley . my name Monica I am the one and, My names is Alanda\\nName : Sara . I am the one and, Name: JILL , Name:Jessie\\nAshley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\\nI'm Ashley I'm bored i am All, I am ALL\\nthis is Ashleyb I'm bored This is Ashleya This is AshleyC\", 'simple_tokens_original_case': [u'Hello', u'guy', u\"'\", u's', u',', u'it', u\"'\", u's', u'Jessica', u'here', u'from', u'the', u'#', u'@', u'%', u'%', u'%', u'Spa', u'.', u'I', u'cant', u'say', u'the', u'name', u'on', u'here', u',', u'and', u'it', u'is', u'JessicaLa', u',', u'and', u'it', u'is', u'Cold', u'\\n', u'Hi', u'Gentlemen', u',', u'My', u'name', u'is', u'Ashley', u'.', u'my', u'name', u'Monica', u'I', u'am', u'the', u'one', u'and', u',', u'My', u'names', u'is', u'Alanda', u'\\n', u'Name', u':', u'Sara', u'.', u'I', u'am', u'the', u'one', u'and', u',', u'Name', u':', u'JILL', u',', u'Name', u':', u'Jessie', u'\\n', u'Ashley', u'(', u'702', u')', u'628', u'-', u'9035', u'XOXO', u'.', u'Aslll', u'(', u'702', u')', u'628', u'-', u'9035', u'XOXO', u'Alppp', u'7026289035', u'\\n', u'I', u\"'\", u'm', u'Ashley', u'I', u\"'\", u'm', u'bored', u'i', u'am', u'All', u',', u'I', u'am', u'ALL', u'\\n', u'this', u'is', u'Ashleyb', u'I', u\"'\", u'm', u'bored', u'This', u'is', u'Ashleya', u'This', u'is', u'AshleyC']}\n" + "{'text': u\"Hello guy's, it's Jessica here from the #@%%% Spa. I cant say the name on here, and it is JessicaLa, and it is Cold\\nHi Gentlemen, My name is Ashley . my name Monica I am the one and, My names is Alanda\\nName : Sara . I am the one and, Name: JILL , Name:Jessie\\nAshley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\\nI'm Ashley I'm bored i am All, I am ALL\\nthis is Ashleyb I'm bored This is Ashleya This is AshleyC\\n Hello boys my name is Brit and I\\u2019m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\", 'simple_tokens_original_case': [u'Hello', u'guy', u\"'\", u's', u',', u'it', u\"'\", u's', u'Jessica', u'here', u'from', u'the', u'#', u'@', u'%', u'%', u'%', u'Spa', u'.', u'I', u'cant', u'say', u'the', u'name', u'on', u'here', u',', u'and', u'it', u'is', u'JessicaLa', u',', u'and', u'it', u'is', u'Cold', u'\\n', u'Hi', u'Gentlemen', u',', u'My', u'name', u'is', u'Ashley', u'.', u'my', u'name', u'Monica', u'I', u'am', u'the', u'one', u'and', u',', u'My', u'names', u'is', u'Alanda', u'\\n', u'Name', u':', u'Sara', u'.', u'I', u'am', u'the', u'one', u'and', u',', u'Name', u':', u'JILL', u',', u'Name', u':', u'Jessie', u'\\n', u'Ashley', u'(', u'702', u')', u'628', u'-', u'9035', u'XOXO', u'.', u'Aslll', u'(', u'702', u')', u'628', u'-', u'9035', u'XOXO', u'Alppp', u'7026289035', u'\\n', u'I', u\"'\", u'm', u'Ashley', u'I', u\"'\", u'm', u'bored', u'i', u'am', u'All', u',', u'I', u'am', u'ALL', u'\\n', u'this', u'is', u'Ashleyb', u'I', u\"'\", u'm', u'bored', u'This', u'is', u'Ashleya', u'This', u'is', u'AshleyC', u'\\n', u'Hello', u'boys', u'my', u'name', u'is', u'Brit', u'and', u'I\\u2019m', u'a', u'28', u'year', u'old', u'BBW', u'with', u'blonde', u'hair', u'and', u'blue', u'eyes', u'and', u'a', u'curvaceous', u'body', u'.', u'My', u'only', u'goal', u'is', u'to', u'find', u'someone', u'that', u'is', u'as', u'adventurous', u'as', u'I', u'am', u'.', u'I', u'am', u'eager', u'to', u'please', u'and', u'want', u'to', u'explore', u'the', u'wild', u'side', u'and', u'need', u'a', u'man', u'to', u'lead', u'me', u'on', u'my', u'journey', u'.', u'Whether', u'you', u'want', u'me', u'to', u'play', u'the', u'babe', u'in', u'the', u'woods', u'or', u'you', u'want', u'to', u'wake', u'the', u'beast', u'in', u'me', u',', u'bottom', u'line', u'is', u'you', u'are', u'satisfied', u'.', u'call', u'or', u'text', u'me', u'anytime', u'the', u'mood', u'strikes', u'at', u'413', u'345', u'8638']}\n" ] } ], @@ -186,6 +92,8 @@ "t.append(u\"Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\")\n", "t.append(u\"I'm Ashley I'm bored i am All, I am ALL\")\n", "t.append(u\"this is Ashleyb I'm bored This is Ashleya This is AshleyC\")\n", + "t.append(u\"\"\" Hello boys my name is Brit and I’m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\"\"\")\n", + "\n", "\n", "d = dict()\n", "d['text'] = \"\\n\".join(t)\n", @@ -199,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 185, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -212,27 +120,36 @@ "Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\n", "I'm Ashley I'm bored i am All, I am ALL\n", "this is Ashleyb I'm bored This is Ashleya This is AshleyC\n", - "[]\n" + " Hello boys my name is Brit and I’m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\n" ] + }, + { + "data": { + "text/plain": [ + "['Ashley', 'Monica', 'Alanda', 'Brit']" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "# my name / names is\n", + "# my name / names is: My name is Ashley . my name Monica I am the one and, My names is Alanda\n", "\n", "rule_01 = {\n", " \"identifier\": \"name_rule_01\",\n", - " \"description\": \"a description\",\n", - " \"is_active\": \"false\",\n", - " \"polarity\": [],\n", + " \"description\": \"my name/names is\",\n", + " \"is_active\": \"true\",\n", + " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " word_token(token=[\"my\"]),\n", " word_token(token=[\"name\", \"names\"]),\n", " word_token(token=[\"is\"], is_required=\"false\"),\n", - " word_token(capitalization=[\"title\", \"upper\"], is_in_output=\"true\")\n", + " word_token(part_of_speech=[\"proper noun\"], capitalization=[\"title\", \"upper\"], is_in_output=\"true\")\n", " ]\n", " }\n", "\n", - "\n", "field_rules = {\n", " \"rules\": [\n", " rule_01\n", @@ -241,12 +158,15 @@ "\n", "print \"text:\", d['text']\n", "results = c.extract_using_custom_spacy(d, config, field_rules=field_rules)\n", - "print results" + "tele_lst = []\n", + "for i in results:\n", + " tele_lst.append(''.join((i.values()[1]).split()))\n", + "tele_lst\n" ] }, { "cell_type": "code", - "execution_count": 186, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -259,18 +179,28 @@ "Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\n", "I'm Ashley I'm bored i am All, I am ALL\n", "this is Ashleyb I'm bored This is Ashleya This is AshleyC\n", - "[{'context': {'start': 105, 'identifier': 'name_rule_02', 'end': 108, 'rule_id': 0}, 'value': 'All'}, {'context': {'start': 109, 'identifier': 'name_rule_02', 'end': 112, 'rule_id': 0}, 'value': 'ALL'}]\n" + " Hello boys my name is Brit and I’m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\n" ] + }, + { + "data": { + "text/plain": [ + "['All', 'ALL']" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "#i am \n", + "#i am: i am All, I am ALL\n", "\n", "rule_02 = {\n", " \"identifier\": \"name_rule_02\",\n", - " \"description\": \"a description\",\n", + " \"description\": \"i am\",\n", " \"is_active\": \"true\",\n", - " \"polarity\": [],\n", + " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " word_token(token=[\"i\"]),\n", " word_token(token=[\"am\"]),\n", @@ -286,12 +216,15 @@ "\n", "print \"text:\", d['text']\n", "results = c.extract_using_custom_spacy(d, config, field_rules=field_rules)\n", - "print results" + "tele_lst = []\n", + "for i in results:\n", + " tele_lst.append(''.join((i.values()[1]).split()))\n", + "tele_lst\n" ] }, { "cell_type": "code", - "execution_count": 187, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -304,18 +237,28 @@ "Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\n", "I'm Ashley I'm bored i am All, I am ALL\n", "this is Ashleyb I'm bored This is Ashleya This is AshleyC\n", - "[{'context': {'start': 59, 'identifier': 'name_rule_03', 'end': 62, 'rule_id': 0}, 'value': 'Sara'}, {'context': {'start': 69, 'identifier': 'name_rule_03', 'end': 72, 'rule_id': 0}, 'value': 'JILL'}, {'context': {'start': 73, 'identifier': 'name_rule_03', 'end': 76, 'rule_id': 0}, 'value': 'Jessie'}]\n" + " Hello boys my name is Brit and I’m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\n" ] + }, + { + "data": { + "text/plain": [ + "['Sara', 'JILL', 'Jessie']" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "# name : Name\n", + "# name : Name Name : Sara . I am the one and, Name: JILL , Name:Jessie\n", "\n", "rule_03 = {\n", " \"identifier\": \"name_rule_03\",\n", - " \"description\": \"a description\",\n", + " \"description\": \"name : Sara\",\n", " \"is_active\": \"true\",\n", - " \"polarity\": [],\n", + " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " word_token(token=[\"name\"]),\n", " punctuation_token(token=[\":\"]),\n", @@ -329,14 +272,18 @@ " ]\n", "}\n", "\n", + "\n", "print \"text:\", d['text']\n", "results = c.extract_using_custom_spacy(d, config, field_rules=field_rules)\n", - "print results\n" + "tele_lst = []\n", + "for i in results:\n", + " tele_lst.append(''.join((i.values()[1]).split()))\n", + "tele_lst\n" ] }, { "cell_type": "code", - "execution_count": 188, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -349,23 +296,33 @@ "Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\n", "I'm Ashley I'm bored i am All, I am ALL\n", "this is Ashleyb I'm bored This is Ashleya This is AshleyC\n", - "[{'context': {'start': 28, 'identifier': 'name_rule_04', 'end': 31, 'rule_id': 0}, 'value': 'JessicaLa'}, {'context': {'start': 33, 'identifier': 'name_rule_04', 'end': 36, 'rule_id': 0}, 'value': 'Cold'}]\n" + " Hello boys my name is Brit and I’m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\n" ] + }, + { + "data": { + "text/plain": [ + "['Cold']" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "# it is \n", + "# it is: it is JessicaLa, and it is Cold\n", "\n", "rule_04 = {\n", " \"identifier\": \"name_rule_04\",\n", - " \"description\": \"a description\",\n", + " \"description\": \"it is Jessicala\",\n", " \"is_active\": \"true\",\n", - " \"polarity\": [],\n", + " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " word_token(token=[\"it\"]),\n", " word_token(token=[\"is\"]),\n", "# word_token(capitalization=[\"title\", \"mixed\"], is_in_output=\"true\")\n", - " word_token(part_of_speech=[\"proper noun\"], is_in_output=\"true\")\n", + " word_token(part_of_speech=[\"proper noun\"], capitalization=[\"title\",\"upper\"], is_in_output=\"true\")\n", " ]\n", " }\n", "\n", @@ -375,14 +332,18 @@ " ]\n", "}\n", "\n", + "\n", "print \"text:\", d['text']\n", "results = c.extract_using_custom_spacy(d, config, field_rules=field_rules)\n", - "print results" + "tele_lst = []\n", + "for i in results:\n", + " tele_lst.append(''.join((i.values()[1]).split()))\n", + "tele_lst\n" ] }, { "cell_type": "code", - "execution_count": 189, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -395,22 +356,32 @@ "Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\n", "I'm Ashley I'm bored i am All, I am ALL\n", "this is Ashleyb I'm bored This is Ashleya This is AshleyC\n", - "[{'context': {'start': 113, 'identifier': 'name_rule_05', 'end': 116, 'rule_id': 0}, 'value': 'Ashleyb'}, {'context': {'start': 120, 'identifier': 'name_rule_05', 'end': 123, 'rule_id': 0}, 'value': 'Ashleya'}, {'context': {'start': 123, 'identifier': 'name_rule_05', 'end': 126, 'rule_id': 0}, 'value': 'AshleyC'}]\n" + " Hello boys my name is Brit and I’m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\n" ] + }, + { + "data": { + "text/plain": [ + "['Ashleyb', 'Ashleya']" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "# this is , This is\n", + "# this is , This is : this is Ashleyb\n", "\n", "rule_05 = {\n", " \"identifier\": \"name_rule_05\",\n", - " \"description\": \"a description\",\n", + " \"description\": \"this is\",\n", " \"is_active\": \"true\",\n", - " \"polarity\": [],\n", + " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " word_token(token=[\"this\"]),\n", " word_token(token=[\"is\"]),\n", - " word_token(part_of_speech=[\"proper noun\"], capitalization=[\"title\", \"mixed\", \"upper\"], is_in_output=\"true\")\n", + " word_token(part_of_speech=[\"proper noun\"], capitalization=[\"title\",\"upper\"], is_in_output=\"true\")\n", " ]\n", " }\n", "\n", @@ -423,12 +394,15 @@ "\n", "print \"text:\", d['text']\n", "results = c.extract_using_custom_spacy(d, config, field_rules=field_rules)\n", - "print results" + "tele_lst = []\n", + "for i in results:\n", + " tele_lst.append(''.join((i.values()[1]).split()))\n", + "tele_lst\n" ] }, { "cell_type": "code", - "execution_count": 190, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -441,24 +415,33 @@ "Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\n", "I'm Ashley I'm bored i am All, I am ALL\n", "this is Ashleyb I'm bored This is Ashleya This is AshleyC\n", - "tokens: [u'Hello', u'guy', u\"'\", u's', u',', u'it', u\"'\", u's', u'Jessica', u'here', u'from', u'the', u'#', u'@', u'%', u'%', u'%', u'Spa', u'.', u'I', u'cant', u'say', u'the', u'name', u'on', u'here', u',', u'and', u'it', u'is', u'JessicaLa', u',', u'and', u'it', u'is', u'Cold', u'\\n', u'Hi', u'Gentlemen', u',', u'My', u'name', u'is', u'Ashley', u'.', u'my', u'name', u'Monica', u'I', u'am', u'the', u'one', u'and', u',', u'My', u'names', u'is', u'Alanda', u'\\n', u'Name', u':', u'Sara', u'.', u'I', u'am', u'the', u'one', u'and', u',', u'Name', u':', u'JILL', u',', u'Name', u':', u'Jessie', u'\\n', u'Ashley', u'(', u'702', u')', u'628', u'-', u'9035', u'XOXO', u'.', u'Aslll', u'(', u'702', u')', u'628', u'-', u'9035', u'XOXO', u'Alppp', u'7026289035', u'\\n', u'I', u\"'\", u'm', u'Ashley', u'I', u\"'\", u'm', u'bored', u'i', u'am', u'All', u',', u'I', u'am', u'ALL', u'\\n', u'this', u'is', u'Ashleyb', u'I', u\"'\", u'm', u'bored', u'This', u'is', u'Ashleya', u'This', u'is', u'AshleyC']\n", - "[{'context': {'start': 97, 'identifier': 'name_rule_06', 'end': 101, 'rule_id': 0}, 'value': 'Ashley'}]\n" + " Hello boys my name is Brit and I’m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\n" ] + }, + { + "data": { + "text/plain": [ + "['Ashley']" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "#I'm\n", + "#I'm: I'm Ashley\n", "\n", "rule_06 = {\n", " \"identifier\": \"name_rule_06\",\n", - " \"description\": \"a description\",\n", + " \"description\": \"i'm\",\n", " \"is_active\": \"true\",\n", - " \"polarity\": [],\n", + " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " word_token(token=[\"i\"]),\n", " punctuation_token(token=[\"'\"]),\n", " word_token(token=[\"m\"]),\n", - " word_token(part_of_speech=[\"proper noun\"], capitalization=[\"title\", \"mixed\", \"upper\"], is_in_output=\"true\")\n", + " word_token(part_of_speech=[\"proper noun\"], capitalization=[\"title\", \"upper\"], is_in_output=\"true\")\n", " ]\n", " }\n", "\n", @@ -468,15 +451,18 @@ " ]\n", "}\n", "\n", + "\n", "print \"text:\", d['text']\n", - "print \"tokens:\", d['simple_tokens_original_case']\n", "results = c.extract_using_custom_spacy(d, config, field_rules=field_rules)\n", - "print results" + "tele_lst = []\n", + "for i in results:\n", + " tele_lst.append(''.join((i.values()[1]).split()))\n", + "tele_lst\n" ] }, { "cell_type": "code", - "execution_count": 196, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -489,34 +475,33 @@ "Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\n", "I'm Ashley I'm bored i am All, I am ALL\n", "this is Ashleyb I'm bored This is Ashleya This is AshleyC\n", - "tokens: [u'Hello', u'guy', u\"'\", u's', u',', u'it', u\"'\", u's', u'Jessica', u'here', u'from', u'the', u'#', u'@', u'%', u'%', u'%', u'Spa', u'.', u'I', u'cant', u'say', u'the', u'name', u'on', u'here', u',', u'and', u'it', u'is', u'JessicaLa', u',', u'and', u'it', u'is', u'Cold', u'\\n', u'Hi', u'Gentlemen', u',', u'My', u'name', u'is', u'Ashley', u'.', u'my', u'name', u'Monica', u'I', u'am', u'the', u'one', u'and', u',', u'My', u'names', u'is', u'Alanda', u'\\n', u'Name', u':', u'Sara', u'.', u'I', u'am', u'the', u'one', u'and', u',', u'Name', u':', u'JILL', u',', u'Name', u':', u'Jessie', u'\\n', u'Ashley', u'(', u'702', u')', u'628', u'-', u'9035', u'XOXO', u'.', u'Aslll', u'(', u'702', u')', u'628', u'-', u'9035', u'XOXO', u'Alppp', u'7026289035', u'\\n', u'I', u\"'\", u'm', u'Ashley', u'I', u\"'\", u'm', u'bored', u'i', u'am', u'All', u',', u'I', u'am', u'ALL', u'\\n', u'this', u'is', u'Ashleyb', u'I', u\"'\", u'm', u'bored', u'This', u'is', u'Ashleya', u'This', u'is', u'AshleyC']\n", - "[\n", - " {\n", - " \"context\": {\n", - " \"start\": 5, \n", - " \"identifier\": \"name_rule_07\", \n", - " \"end\": 9, \n", - " \"rule_id\": 0\n", - " }, \n", - " \"value\": \"Jessica\"\n", - " }\n", - "]\n" + " Hello boys my name is Brit and I’m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\n" ] + }, + { + "data": { + "text/plain": [ + "['Jessica']" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "#it's\n", + "#it's: it's Jessica\n", "\n", "rule_07 = {\n", " \"identifier\": \"name_rule_07\",\n", - " \"description\": \"a description\",\n", + " \"description\": \"it's\",\n", " \"is_active\": \"true\",\n", - " \"polarity\": [],\n", + " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " word_token(token=[\"it\"]),\n", " punctuation_token(token=[\"'\"]),\n", " word_token(token=[\"s\"]),\n", - " word_token(part_of_speech=[\"proper noun\"], capitalization=[\"title\", \"mixed\", \"upper\"], is_in_output=\"true\") \n", + " word_token(part_of_speech=[\"proper noun\"], capitalization=[\"title\", \"upper\"], is_in_output=\"true\") \n", " ]\n", " }\n", "\n", @@ -527,15 +512,18 @@ "}\n", "\n", "\n", + "\n", "print \"text:\", d['text']\n", - "print \"tokens:\", d['simple_tokens_original_case']\n", "results = c.extract_using_custom_spacy(d, config, field_rules=field_rules)\n", - "print json.dumps(results, indent=2)" + "tele_lst = []\n", + "for i in results:\n", + " tele_lst.append(''.join((i.values()[1]).split()))\n", + "tele_lst\n" ] }, { "cell_type": "code", - "execution_count": 192, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -548,18 +536,27 @@ "Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\n", "I'm Ashley I'm bored i am All, I am ALL\n", "this is Ashleyb I'm bored This is Ashleya This is AshleyC\n", - "tokens: [u'Hello', u'guy', u\"'\", u's', u',', u'it', u\"'\", u's', u'Jessica', u'here', u'from', u'the', u'#', u'@', u'%', u'%', u'%', u'Spa', u'.', u'I', u'cant', u'say', u'the', u'name', u'on', u'here', u',', u'and', u'it', u'is', u'JessicaLa', u',', u'and', u'it', u'is', u'Cold', u'\\n', u'Hi', u'Gentlemen', u',', u'My', u'name', u'is', u'Ashley', u'.', u'my', u'name', u'Monica', u'I', u'am', u'the', u'one', u'and', u',', u'My', u'names', u'is', u'Alanda', u'\\n', u'Name', u':', u'Sara', u'.', u'I', u'am', u'the', u'one', u'and', u',', u'Name', u':', u'JILL', u',', u'Name', u':', u'Jessie', u'\\n', u'Ashley', u'(', u'702', u')', u'628', u'-', u'9035', u'XOXO', u'.', u'Aslll', u'(', u'702', u')', u'628', u'-', u'9035', u'XOXO', u'Alppp', u'7026289035', u'\\n', u'I', u\"'\", u'm', u'Ashley', u'I', u\"'\", u'm', u'bored', u'i', u'am', u'All', u',', u'I', u'am', u'ALL', u'\\n', u'this', u'is', u'Ashleyb', u'I', u\"'\", u'm', u'bored', u'This', u'is', u'Ashleya', u'This', u'is', u'AshleyC']\n", - "[{'context': {'start': 77, 'identifier': 'name_rule_08', 'end': 80, 'rule_id': 0}, 'value': 'Ashley'}, {'context': {'start': 86, 'identifier': 'name_rule_08', 'end': 89, 'rule_id': 0}, 'value': 'Aslll'}]\n" + " Hello boys my name is Brit and I’m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\n" ] + }, + { + "data": { + "text/plain": [ + "['Ashley', 'Aslll']" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "#Ashley (702)\n", "rule_08 = {\n", " \"identifier\": \"name_rule_08\",\n", - " \"description\": \"a description\",\n", + " \"description\": \"name followed by telephone number[123]\",\n", " \"is_active\": \"true\",\n", - " \"polarity\": [],\n", + " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " word_token(capitalization=[\"title\"], is_in_output=\"true\"),\n", " punctuation_token(token=[\"(\", \"[\"]),\n", @@ -573,23 +570,42 @@ " ]\n", "}\n", "\n", + "\n", "print \"text:\", d['text']\n", - "print \"tokens:\", d['simple_tokens_original_case']\n", "results = c.extract_using_custom_spacy(d, config, field_rules=field_rules)\n", - "print results" + "tele_lst = []\n", + "for i in results:\n", + " tele_lst.append(''.join((i.values()[1]).split()))\n", + "tele_lst\n" ] }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[{'context': {'start': 94, 'identifier': 'name_rule_09', 'end': 96, 'rule_id': 0}, 'value': 'Alppp'}]\n" + "text: Hello guy's, it's Jessica here from the #@%%% Spa. I cant say the name on here, and it is JessicaLa, and it is Cold\n", + "Hi Gentlemen, My name is Ashley . my name Monica I am the one and, My names is Alanda\n", + "Name : Sara . I am the one and, Name: JILL , Name:Jessie\n", + "Ashley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\n", + "I'm Ashley I'm bored i am All, I am ALL\n", + "this is Ashleyb I'm bored This is Ashleya This is AshleyC\n", + " Hello boys my name is Brit and I’m a 28 year old BBW with blonde hair and blue eyes and a curvaceous body. My only goal is to find someone that is as adventurous as I am. I am eager to please and want to explore the wild side and need a man to lead me on my journey. Whether you want me to play the babe in the woods or you want to wake the beast in me, bottom line is you are satisfied.call or text me anytime the mood strikes at 413 345 8638\n" ] + }, + { + "data": { + "text/plain": [ + "['Alppp']" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -597,11 +613,11 @@ "\n", "rule_09 = {\n", " \"identifier\": \"name_rule_09\",\n", - " \"description\": \"a description\",\n", + " \"description\": \"name followed by telephone number 7135975313\",\n", " \"is_active\": \"true\",\n", - " \"polarity\": [],\n", + " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", - " word_token(capitalization=[\"title\", \"upper\", \"mixed\"], is_in_output=\"true\"),\n", + " word_token(capitalization=[\"title\", \"upper\"], is_in_output=\"true\"),\n", " shape_token(shape=[\"dddddddddd\"])\n", " ]\n", " }\n", @@ -612,8 +628,13 @@ " ]\n", "}\n", "\n", + "\n", + "print \"text:\", d['text']\n", "results = c.extract_using_custom_spacy(d, config, field_rules=field_rules)\n", - "print results" + "tele_lst = []\n", + "for i in results:\n", + " tele_lst.append(''.join((i.values()[1]).split()))\n", + "tele_lst\n" ] }, { @@ -625,152 +646,14 @@ }, { "cell_type": "code", - "execution_count": 197, + "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[{'context': {'start': 105, 'identifier': 'name_rule_02', 'end': 108, 'rule_id': 1}, 'value': 'All'}, {'context': {'start': 109, 'identifier': 'name_rule_02', 'end': 112, 'rule_id': 1}, 'value': 'ALL'}, {'context': {'start': 59, 'identifier': 'name_rule_03', 'end': 62, 'rule_id': 2}, 'value': 'Sara'}, {'context': {'start': 69, 'identifier': 'name_rule_03', 'end': 72, 'rule_id': 2}, 'value': 'JILL'}, {'context': {'start': 73, 'identifier': 'name_rule_03', 'end': 76, 'rule_id': 2}, 'value': 'Jessie'}, {'context': {'start': 28, 'identifier': 'name_rule_04', 'end': 31, 'rule_id': 3}, 'value': 'JessicaLa'}, {'context': {'start': 33, 'identifier': 'name_rule_04', 'end': 36, 'rule_id': 3}, 'value': 'Cold'}, {'context': {'start': 113, 'identifier': 'name_rule_05', 'end': 116, 'rule_id': 4}, 'value': 'Ashleyb'}, {'context': {'start': 120, 'identifier': 'name_rule_05', 'end': 123, 'rule_id': 4}, 'value': 'Ashleya'}, {'context': {'start': 123, 'identifier': 'name_rule_05', 'end': 126, 'rule_id': 4}, 'value': 'AshleyC'}, {'context': {'start': 97, 'identifier': 'name_rule_06', 'end': 101, 'rule_id': 5}, 'value': 'Ashley'}, {'context': {'start': 5, 'identifier': 'name_rule_07', 'end': 9, 'rule_id': 6}, 'value': 'Jessica'}, {'context': {'start': 77, 'identifier': 'name_rule_08', 'end': 80, 'rule_id': 7}, 'value': 'Ashley'}, {'context': {'start': 86, 'identifier': 'name_rule_08', 'end': 89, 'rule_id': 7}, 'value': 'Aslll'}, {'context': {'start': 94, 'identifier': 'name_rule_09', 'end': 96, 'rule_id': 8}, 'value': 'Alppp'}]\n", - "{\"rules\": [{\"polarity\": [], \"pattern\": [{\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"my\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"name\", \"names\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"is\"], \"is_followed_by_space\": \"\", \"is_required\": \"false\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [\"title\", \"upper\"], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"true\", \"length\": [], \"shapes\": [], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}], \"identifier\": \"name_rule_01\", \"is_active\": \"false\", \"description\": \"a description\"}, {\"polarity\": [], \"pattern\": [{\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"i\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"am\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [\"title\", \"upper\"], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"true\", \"length\": [], \"shapes\": [], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}], \"identifier\": \"name_rule_02\", \"is_active\": \"true\", \"description\": \"a description\"}, {\"polarity\": [], \"pattern\": [{\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"name\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\":\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"punctuation\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"true\", \"length\": [], \"shapes\": [], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}], \"identifier\": \"name_rule_03\", \"is_active\": \"true\", \"description\": \"a description\"}, {\"polarity\": [], \"pattern\": [{\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"it\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"is\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [\"proper noun\"], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"true\", \"length\": [], \"shapes\": [], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}], \"identifier\": \"name_rule_04\", \"is_active\": \"true\", \"description\": \"a description\"}, {\"polarity\": [], \"pattern\": [{\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"this\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"is\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [\"title\", \"mixed\", \"upper\"], \"part_of_speech\": [\"proper noun\"], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"true\", \"length\": [], \"shapes\": [], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}], \"identifier\": \"name_rule_05\", \"is_active\": \"true\", \"description\": \"a description\"}, {\"polarity\": [], \"pattern\": [{\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"i\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"'\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"punctuation\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"m\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [\"title\", \"mixed\", \"upper\"], \"part_of_speech\": [\"proper noun\"], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"true\", \"length\": [], \"shapes\": [], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}], \"identifier\": \"name_rule_06\", \"is_active\": \"true\", \"description\": \"a description\"}, {\"polarity\": [], \"pattern\": [{\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"it\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"'\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"punctuation\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"s\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [\"title\", \"mixed\", \"upper\"], \"part_of_speech\": [\"proper noun\"], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"true\", \"length\": [], \"shapes\": [], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}], \"identifier\": \"name_rule_07\", \"is_active\": \"true\", \"description\": \"a description\"}, {\"polarity\": [], \"pattern\": [{\"suffix\": \"\", \"capitalization\": [\"title\"], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"true\", \"length\": [], \"shapes\": [], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [], \"token\": [\"(\", \"[\"], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"punctuation\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [\"ddd\"], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"shape\"}], \"identifier\": \"name_rule_08\", \"is_active\": \"true\", \"description\": \"a description\"}, {\"polarity\": [], \"pattern\": [{\"suffix\": \"\", \"capitalization\": [\"title\", \"upper\", \"mixed\"], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"true\", \"length\": [], \"shapes\": [], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"word\"}, {\"suffix\": \"\", \"capitalization\": [], \"part_of_speech\": [], \"prefix\": \"\", \"contain_digit\": \"\", \"is_in_vocabulary\": \"\", \"is_out_of_vocabulary\": \"\", \"is_in_output\": \"false\", \"length\": [], \"shapes\": [\"dddddddddd\"], \"token\": [], \"is_followed_by_space\": \"\", \"is_required\": \"true\", \"type\": \"shape\"}], \"identifier\": \"name_rule_09\", \"is_active\": \"true\", \"description\": \"a description\"}], \"results\": [{\"context\": {\"start\": 105, \"identifier\": \"name_rule_02\", \"end\": 108, \"rule_id\": 1}, \"value\": \"All\"}, {\"context\": {\"start\": 109, \"identifier\": \"name_rule_02\", \"end\": 112, \"rule_id\": 1}, \"value\": \"ALL\"}, {\"context\": {\"start\": 59, \"identifier\": \"name_rule_03\", \"end\": 62, \"rule_id\": 2}, \"value\": \"Sara\"}, {\"context\": {\"start\": 69, \"identifier\": \"name_rule_03\", \"end\": 72, \"rule_id\": 2}, \"value\": \"JILL\"}, {\"context\": {\"start\": 73, \"identifier\": \"name_rule_03\", \"end\": 76, \"rule_id\": 2}, \"value\": \"Jessie\"}, {\"context\": {\"start\": 28, \"identifier\": \"name_rule_04\", \"end\": 31, \"rule_id\": 3}, \"value\": \"JessicaLa\"}, {\"context\": {\"start\": 33, \"identifier\": \"name_rule_04\", \"end\": 36, \"rule_id\": 3}, \"value\": \"Cold\"}, {\"context\": {\"start\": 113, \"identifier\": \"name_rule_05\", \"end\": 116, \"rule_id\": 4}, \"value\": \"Ashleyb\"}, {\"context\": {\"start\": 120, \"identifier\": \"name_rule_05\", \"end\": 123, \"rule_id\": 4}, \"value\": \"Ashleya\"}, {\"context\": {\"start\": 123, \"identifier\": \"name_rule_05\", \"end\": 126, \"rule_id\": 4}, \"value\": \"AshleyC\"}, {\"context\": {\"start\": 97, \"identifier\": \"name_rule_06\", \"end\": 101, \"rule_id\": 5}, \"value\": \"Ashley\"}, {\"context\": {\"start\": 5, \"identifier\": \"name_rule_07\", \"end\": 9, \"rule_id\": 6}, \"value\": \"Jessica\"}, {\"context\": {\"start\": 77, \"identifier\": \"name_rule_08\", \"end\": 80, \"rule_id\": 7}, \"value\": \"Ashley\"}, {\"context\": {\"start\": 86, \"identifier\": \"name_rule_08\", \"end\": 89, \"rule_id\": 7}, \"value\": \"Aslll\"}, {\"context\": {\"start\": 94, \"identifier\": \"name_rule_09\", \"end\": 96, \"rule_id\": 8}, \"value\": \"Alppp\"}], \"test_tokens\": [\"Hello\", \"guy\", \"'\", \"s\", \",\", \"it\", \"'\", \"s\", \"Jessica\", \"here\", \"from\", \"the\", \"#\", \"@\", \"%\", \"%\", \"%\", \"Spa\", \".\", \"I\", \"cant\", \"say\", \"the\", \"name\", \"on\", \"here\", \",\", \"and\", \"it\", \"is\", \"JessicaLa\", \",\", \"and\", \"it\", \"is\", \"Cold\", \"\\n\", \"Hi\", \"Gentlemen\", \",\", \"My\", \"name\", \"is\", \"Ashley\", \".\", \"my\", \"name\", \"Monica\", \"I\", \"am\", \"the\", \"one\", \"and\", \",\", \"My\", \"names\", \"is\", \"Alanda\", \"\\n\", \"Name\", \":\", \"Sara\", \".\", \"I\", \"am\", \"the\", \"one\", \"and\", \",\", \"Name\", \":\", \"JILL\", \",\", \"Name\", \":\", \"Jessie\", \"\\n\", \"Ashley\", \"(\", \"702\", \")\", \"628\", \"-\", \"9035\", \"XOXO\", \".\", \"Aslll\", \"(\", \"702\", \")\", \"628\", \"-\", \"9035\", \"XOXO\", \"Alppp\", \"7026289035\", \"\\n\", \"I\", \"'\", \"m\", \"Ashley\", \"I\", \"'\", \"m\", \"bored\", \"i\", \"am\", \"All\", \",\", \"I\", \"am\", \"ALL\", \"\\n\", \"this\", \"is\", \"Ashleyb\", \"I\", \"'\", \"m\", \"bored\", \"This\", \"is\", \"Ashleya\", \"This\", \"is\", \"AshleyC\"], \"test_text\": \"Hello guy's, it's Jessica here from the #@%%% Spa. I cant say the name on here, and it is JessicaLa, and it is Cold\\nHi Gentlemen, My name is Ashley . my name Monica I am the one and, My names is Alanda\\nName : Sara . I am the one and, Name: JILL , Name:Jessie\\nAshley (702)628-9035 XOXO . Aslll (702) 628-9035 XOXO Alppp 7026289035\\nI'm Ashley I'm bored i am All, I am ALL\\nthis is Ashleyb I'm bored This is Ashleya This is AshleyC\"}\n", - "[\n", - " {\n", - " \"context\": {\n", - " \"start\": 105, \n", - " \"identifier\": \"name_rule_02\", \n", - " \"end\": 108, \n", - " \"rule_id\": 1\n", - " }, \n", - " \"value\": \"All\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 109, \n", - " \"identifier\": \"name_rule_02\", \n", - " \"end\": 112, \n", - " \"rule_id\": 1\n", - " }, \n", - " \"value\": \"ALL\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 59, \n", - " \"identifier\": \"name_rule_03\", \n", - " \"end\": 62, \n", - " \"rule_id\": 2\n", - " }, \n", - " \"value\": \"Sara\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 69, \n", - " \"identifier\": \"name_rule_03\", \n", - " \"end\": 72, \n", - " \"rule_id\": 2\n", - " }, \n", - " \"value\": \"JILL\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 73, \n", - " \"identifier\": \"name_rule_03\", \n", - " \"end\": 76, \n", - " \"rule_id\": 2\n", - " }, \n", - " \"value\": \"Jessie\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 28, \n", - " \"identifier\": \"name_rule_04\", \n", - " \"end\": 31, \n", - " \"rule_id\": 3\n", - " }, \n", - " \"value\": \"JessicaLa\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 33, \n", - " \"identifier\": \"name_rule_04\", \n", - " \"end\": 36, \n", - " \"rule_id\": 3\n", - " }, \n", - " \"value\": \"Cold\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 113, \n", - " \"identifier\": \"name_rule_05\", \n", - " \"end\": 116, \n", - " \"rule_id\": 4\n", - " }, \n", - " \"value\": \"Ashleyb\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 120, \n", - " \"identifier\": \"name_rule_05\", \n", - " \"end\": 123, \n", - " \"rule_id\": 4\n", - " }, \n", - " \"value\": \"Ashleya\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 123, \n", - " \"identifier\": \"name_rule_05\", \n", - " \"end\": 126, \n", - " \"rule_id\": 4\n", - " }, \n", - " \"value\": \"AshleyC\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 97, \n", - " \"identifier\": \"name_rule_06\", \n", - " \"end\": 101, \n", - " \"rule_id\": 5\n", - " }, \n", - " \"value\": \"Ashley\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 5, \n", - " \"identifier\": \"name_rule_07\", \n", - " \"end\": 9, \n", - " \"rule_id\": 6\n", - " }, \n", - " \"value\": \"Jessica\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 77, \n", - " \"identifier\": \"name_rule_08\", \n", - " \"end\": 80, \n", - " \"rule_id\": 7\n", - " }, \n", - " \"value\": \"Ashley\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 86, \n", - " \"identifier\": \"name_rule_08\", \n", - " \"end\": 89, \n", - " \"rule_id\": 7\n", - " }, \n", - " \"value\": \"Aslll\"\n", - " }, \n", - " {\n", - " \"context\": {\n", - " \"start\": 94, \n", - " \"identifier\": \"name_rule_09\", \n", - " \"end\": 96, \n", - " \"rule_id\": 8\n", - " }, \n", - " \"value\": \"Alppp\"\n", - " }\n", - "]\n" + "['Jessica', 'Cold', 'Ashley', 'Monica', 'Alanda', 'Sara', 'JILL', 'Jessie', 'Ashley', 'Aslll', 'Alppp', 'Ashley', 'All', 'ALL', 'Ashleyb', 'Ashleya', 'Brit']\n" ] } ], @@ -794,31 +677,34 @@ "\n", "\n", "results = c.extract_using_custom_spacy(d, config, field_rules=field_rules)\n", - "print results\n", "\n", + "tele_lst = []\n", + "for i in results:\n", + " tele_lst.append(''.join((i.values()[1]).split()))\n", + "results.append(tele_lst)\n", + "\n", + "print tele_lst\n", "field_rules['results']=results\n", "\n", - "s = json.dumps(field_rules)\n", + "s = json.dumps(field_rules, indent=2)\n", + "\n", "\n", - "print s\n", "import codecs\n", - "o = codecs.open('path_to_file', 'w')\n", + "o = codecs.open('name.json', 'w')\n", "o.write(s)\n", - "o.close()\n", - "\n", - "print json.dumps(results, indent=2)" + "o.close()" ] }, { "cell_type": "code", - "execution_count": 195, + "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'suffix': '', 'capitalization': [], 'part_of_speech': [], 'prefix': '', 'contain_digit': '', 'is_in_vocabulary': '', 'is_out_of_vocabulary': '', 'is_in_output': 'false', 'length': [], 'shapes': [], 'token': ['hello'], 'is_followed_by_space': '', 'is_required': 'true', 'type': 'word'}\n" + "{'prefix': '', 'suffix': '', 'capitalization': [], 'part_of_speech': [], 'length': [], 'maximum': '', 'shapes': [], 'token': ['hello'], 'minimum': '', 'numbers': [], 'contain_digit': '', 'is_in_vocabulary': '', 'is_out_of_vocabulary': '', 'is_required': 'true', 'type': 'word', 'is_in_output': 'false'}\n" ] } ], diff --git a/etk_phonenum_rules.ipynb b/etk_phonenum_rules.ipynb index 4340b4ab..1c73ffe0 100644 --- a/etk_phonenum_rules.ipynb +++ b/etk_phonenum_rules.ipynb @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 58, "metadata": { "collapsed": true }, @@ -56,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": 59, "metadata": { "collapsed": true }, @@ -69,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 60, "metadata": { "scrolled": true }, @@ -78,7 +78,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'text': u'my telephone number is (217)331-6779, (217)-331-6778,(217)-331-6777, 217-331-6776, 734.709.8965, 949 484 6951, 5017774643, 664 123 45 67, 0660852222\\nHi Gentlemen, My name is Ashley . my name Monica I am the one and, My names is Alanda\\n(217)-331-6778, (044) 322 1719850,044 322 1719851, (045) 322 1719850, 045 55 49 40 83 95, 045- 55- 49 40 83 94, 045 -5549408395\\n-0466119200, -488019954,+32465863497 - , 0466119200\\n74350 - 0642516048, 07 55 71 64 36 - \\n + 49 15781424777, +( 49 )15732190888?\\n 004915221040240\\n 49( 0 )15771824788\\n 111 2222, 111-3333\\n UK: 07077080500, 07741 011 066, 07014-231- 011, 0751 011 41 92 , \\n UK:(022) 1111 2222, (0100) 000 1113, (01222) 22224, (01222) 333335, (0122 22)3336, (0122 22) 33337, \\n UK: 0121-111 2228, 0121 111 2229, 07111 222220, 0111 222 2221, 0500 111112, 0800 111113 , \\n India: 111-2222222, 11111-33333, +91-111 222 3333\\n China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\\n HK: 1111 2222, 33334444\\n Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \\n Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444', 'simple_tokens_original_case': [u'my', u'telephone', u'number', u'is', u'(', u'217', u')', u'331', u'-', u'6779', u',', u'(', u'217', u')', u'-', u'331', u'-', u'6778', u',', u'(', u'217', u')', u'-', u'331', u'-', u'6777', u',', u'217', u'-', u'331', u'-', u'6776', u',', u'734', u'.', u'709', u'.', u'8965', u',', u'949', u'484', u'6951', u',', u'5017774643', u',', u'664', u'123', u'45', u'67', u',', u'0660852222', u'\\n', u'Hi', u'Gentlemen', u',', u'My', u'name', u'is', u'Ashley', u'.', u'my', u'name', u'Monica', u'I', u'am', u'the', u'one', u'and', u',', u'My', u'names', u'is', u'Alanda', u'\\n', u'(', u'217', u')', u'-', u'331', u'-', u'6778', u',', u'(', u'044', u')', u'322', u'1719850', u',', u'044', u'322', u'1719851', u',', u'(', u'045', u')', u'322', u'1719850', u',', u'045', u'55', u'49', u'40', u'83', u'95', u',', u'045', u'-', u'55', u'-', u'49', u'40', u'83', u'94', u',', u'045', u'-', u'5549408395', u'\\n', u'-', u'0466119200', u',', u'-', u'488019954', u',', u'+', u'32465863497', u'-', u',', u'0466119200', u'\\n', u'74350', u'-', u'0642516048', u',', u'07', u'55', u'71', u'64', u'36', u'-', u'\\n', u'+', u'49', u'15781424777', u',', u'+', u'(', u'49', u')', u'15732190888', u'?', u'\\n', u'004915221040240', u'\\n', u'49', u'(', u'0', u')', u'15771824788', u'\\n', u'111', u'2222', u',', u'111', u'-', u'3333', u'\\n', u'UK', u':', u'07077080500', u',', u'07741', u'011', u'066', u',', u'07014', u'-', u'231', u'-', u'011', u',', u'0751', u'011', u'41', u'92', u',', u'\\n', u'UK', u':', u'(', u'022', u')', u'1111', u'2222', u',', u'(', u'0100', u')', u'000', u'1113', u',', u'(', u'01222', u')', u'22224', u',', u'(', u'01222', u')', u'333335', u',', u'(', u'0122', u'22', u')', u'3336', u',', u'(', u'0122', u'22', u')', u'33337', u',', u'\\n', u'UK', u':', u'0121', u'-', u'111', u'2228', u',', u'0121', u'111', u'2229', u',', u'07111', u'222220', u',', u'0111', u'222', u'2221', u',', u'0500', u'111112', u',', u'0800', u'111113', u',', u'\\n', u'India', u':', u'111', u'-', u'2222222', u',', u'11111', u'-', u'33333', u',', u'+', u'91', u'-', u'111', u'222', u'3333', u'\\n', u'China', u':', u'(', u'0111', u')', u'1111', u'2222', u',', u'+', u'86', u'122', u'3333', u'4444', u',', u'0086', u'111', u'2222', u'3333', u',', u'+', u'86', u'111', u'222', u'33', u'444', u'\\n', u'HK', u':', u'1111', u'2222', u',', u'33334444', u'\\n', u'Japan', u':', u'(', u'011', u')', u'222', u'-', u'3333', u',', u'(', u'0120', u')', u'-', u'22', u'-', u'3333', u',', u'0570', u'-', u'22', u'-', u'3333', u',', u'0800', u'-', u'22', u'-', u'3333', u',', u'050', u'-', u'2222', u'-', u'3333', u',', u'+', u'61', u'\\n', u'Austrilia', u':', u'(', u'01', u')', u'1111', u'2222', u',', u'+', u'61', u'42222', u'3333', u',', u'+', u'61', u'222', u'333', u'444', u',', u'1003', u'333', u'444']}\n" + "{'text': u'my telephone number is (217)331-6779, (217)-331-6778,(217)-331-6777, 217-331-6776, 734.709.8965, 949 484 6951, 5017774643, 664 123 45 67, 0660852222\\nHi Gentlemen, My name is Ashley . my name Monica I am the one and, My names is Alanda\\n(217)-331-6778, (044) 322 1719850,044 322 1719851, (045) 322 1719850, 045 55 49 40 83 95, 045- 55- 49 40 83 94, 045 -5549408395\\n-0466119200, -488019954,+32465863497 - , 0466119200\\n74350 - 0642516048, 07 55 71 64 36 - \\n + 49 15781424777, +( 49 )15732190888?\\n 004915221040240\\n 49( 0 )15771824788\\n 111 2222, 111-3333\\n UK: 07077080500, 07741 011 066, 07014-231- 011, 0751 011 41 92 , \\n UK:(022) 1111 2222, (0100) 000 1113, (01222) 22224, (01222) 333335, (0122 22)3336, (0122 22) 33337, \\n UK: 0121-111 2228, 0121 111 2229, 07111 222220, 0111 222 2221, 0500 111112, 0800 111113 , \\n India: 111-2222222, 11111-33333, +91-111 222 3333\\n China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\\n HK: 1111 2222, 33334444\\n Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \\n Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)\\u2022 Le jubil\\xe9 des Old Gaffers', 'simple_tokens_original_case': [u'my', u'telephone', u'number', u'is', u'(', u'217', u')', u'331', u'-', u'6779', u',', u'(', u'217', u')', u'-', u'331', u'-', u'6778', u',', u'(', u'217', u')', u'-', u'331', u'-', u'6777', u',', u'217', u'-', u'331', u'-', u'6776', u',', u'734', u'.', u'709', u'.', u'8965', u',', u'949', u'484', u'6951', u',', u'5017774643', u',', u'664', u'123', u'45', u'67', u',', u'0660852222', u'\\n', u'Hi', u'Gentlemen', u',', u'My', u'name', u'is', u'Ashley', u'.', u'my', u'name', u'Monica', u'I', u'am', u'the', u'one', u'and', u',', u'My', u'names', u'is', u'Alanda', u'\\n', u'(', u'217', u')', u'-', u'331', u'-', u'6778', u',', u'(', u'044', u')', u'322', u'1719850', u',', u'044', u'322', u'1719851', u',', u'(', u'045', u')', u'322', u'1719850', u',', u'045', u'55', u'49', u'40', u'83', u'95', u',', u'045', u'-', u'55', u'-', u'49', u'40', u'83', u'94', u',', u'045', u'-', u'5549408395', u'\\n', u'-', u'0466119200', u',', u'-', u'488019954', u',', u'+', u'32465863497', u'-', u',', u'0466119200', u'\\n', u'74350', u'-', u'0642516048', u',', u'07', u'55', u'71', u'64', u'36', u'-', u'\\n', u'+', u'49', u'15781424777', u',', u'+', u'(', u'49', u')', u'15732190888', u'?', u'\\n', u'004915221040240', u'\\n', u'49', u'(', u'0', u')', u'15771824788', u'\\n', u'111', u'2222', u',', u'111', u'-', u'3333', u'\\n', u'UK', u':', u'07077080500', u',', u'07741', u'011', u'066', u',', u'07014', u'-', u'231', u'-', u'011', u',', u'0751', u'011', u'41', u'92', u',', u'\\n', u'UK', u':', u'(', u'022', u')', u'1111', u'2222', u',', u'(', u'0100', u')', u'000', u'1113', u',', u'(', u'01222', u')', u'22224', u',', u'(', u'01222', u')', u'333335', u',', u'(', u'0122', u'22', u')', u'3336', u',', u'(', u'0122', u'22', u')', u'33337', u',', u'\\n', u'UK', u':', u'0121', u'-', u'111', u'2228', u',', u'0121', u'111', u'2229', u',', u'07111', u'222220', u',', u'0111', u'222', u'2221', u',', u'0500', u'111112', u',', u'0800', u'111113', u',', u'\\n', u'India', u':', u'111', u'-', u'2222222', u',', u'11111', u'-', u'33333', u',', u'+', u'91', u'-', u'111', u'222', u'3333', u'\\n', u'China', u':', u'(', u'0111', u')', u'1111', u'2222', u',', u'+', u'86', u'122', u'3333', u'4444', u',', u'0086', u'111', u'2222', u'3333', u',', u'+', u'86', u'111', u'222', u'33', u'444', u'\\n', u'HK', u':', u'1111', u'2222', u',', u'33334444', u'\\n', u'Japan', u':', u'(', u'011', u')', u'222', u'-', u'3333', u',', u'(', u'0120', u')', u'-', u'22', u'-', u'3333', u',', u'0570', u'-', u'22', u'-', u'3333', u',', u'0800', u'-', u'22', u'-', u'3333', u',', u'050', u'-', u'2222', u'-', u'3333', u',', u'+', u'61', u'\\n', u'Austrilia', u':', u'(', u'01', u')', u'1111', u'2222', u',', u'+', u'61', u'42222', u'3333', u',', u'+', u'61', u'222', u'333', u'444', u',', u'1003', u'333', u'444', u',', u'Les', u'chantiers', u'de', u'St', u'-', u'Nazaire', u'(', u'1881', u'-', u'1950', u')', u'\\u2022', u'Le', u'jubil\\xe9', u'des', u'Old', u'Gaffers']}\n" ] } ], @@ -101,9 +101,7 @@ "t.append(u\" China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\")\n", "t.append(u\" HK: 1111 2222, 33334444\")\n", "t.append(u\" Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \")\n", - "t.append(u\" Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\")\n", - "\n", - "\n", + "t.append(u\" Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\")\n", "\n", "\n", "d = dict()\n", @@ -118,7 +116,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 61, "metadata": {}, "outputs": [ { @@ -141,7 +139,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -158,7 +156,7 @@ " '0112223333']" ] }, - "execution_count": 119, + "execution_count": 61, "metadata": {}, "output_type": "execute_result" } @@ -198,7 +196,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 62, "metadata": {}, "outputs": [ { @@ -221,7 +219,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -235,7 +233,7 @@ " '0642516048']" ] }, - "execution_count": 120, + "execution_count": 62, "metadata": {}, "output_type": "execute_result" } @@ -269,7 +267,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 63, "metadata": {}, "outputs": [ { @@ -292,7 +290,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -309,7 +307,7 @@ " '0112223333']" ] }, - "execution_count": 121, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -349,7 +347,7 @@ }, { "cell_type": "code", - "execution_count": 122, + "execution_count": 64, "metadata": {}, "outputs": [ { @@ -372,7 +370,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -381,7 +379,7 @@ "[]" ] }, - "execution_count": 122, + "execution_count": 64, "metadata": {}, "output_type": "execute_result" } @@ -415,7 +413,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 65, "metadata": {}, "outputs": [ { @@ -438,24 +436,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['2173316779',\n", - " '2173316778',\n", - " '2173316777',\n", - " '2173316776',\n", - " '7347098965',\n", - " '9494846951',\n", - " '2173316778',\n", - " '1112223333',\n", - " '0112223333']" + "[]" ] }, - "execution_count": 123, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } @@ -466,7 +456,7 @@ "rule_05 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -495,7 +485,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 66, "metadata": {}, "outputs": [ { @@ -518,7 +508,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -527,7 +517,7 @@ "[]" ] }, - "execution_count": 124, + "execution_count": 66, "metadata": {}, "output_type": "execute_result" } @@ -561,7 +551,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 67, "metadata": {}, "outputs": [ { @@ -584,16 +574,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['6641234567']" + "[]" ] }, - "execution_count": 125, + "execution_count": 67, "metadata": {}, "output_type": "execute_result" } @@ -604,7 +594,7 @@ "rule_07 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}{4}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -635,7 +625,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 68, "metadata": {}, "outputs": [ { @@ -658,7 +648,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -667,7 +657,7 @@ "[]" ] }, - "execution_count": 126, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" } @@ -678,7 +668,7 @@ "rule_08 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -707,7 +697,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 69, "metadata": {}, "outputs": [ { @@ -730,16 +720,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['0455549408395', '0455549408394']" + "[]" ] }, - "execution_count": 127, + "execution_count": 69, "metadata": {}, "output_type": "execute_result" } @@ -750,7 +740,7 @@ "rule_09 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}{4}{5}{6}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -785,7 +775,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 70, "metadata": {}, "outputs": [ { @@ -808,16 +798,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['0455549408395']" + "[]" ] }, - "execution_count": 128, + "execution_count": 70, "metadata": {}, "output_type": "execute_result" } @@ -828,7 +818,7 @@ "rule_10 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -855,7 +845,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 71, "metadata": {}, "outputs": [ { @@ -878,7 +868,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -887,7 +877,7 @@ "[]" ] }, - "execution_count": 129, + "execution_count": 71, "metadata": {}, "output_type": "execute_result" } @@ -923,7 +913,7 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 72, "metadata": {}, "outputs": [ { @@ -946,16 +936,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['0466119200', '0466119200']" + "[]" ] }, - "execution_count": 130, + "execution_count": 72, "metadata": {}, "output_type": "execute_result" } @@ -966,7 +956,7 @@ "rule_12 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " number_token(minimum=\"400000000\", maximum=\"499999999\", length = [10], is_in_output=\"true\")\n", @@ -989,7 +979,7 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 73, "metadata": {}, "outputs": [ { @@ -1012,16 +1002,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['32465863497']" + "[]" ] }, - "execution_count": 131, + "execution_count": 73, "metadata": {}, "output_type": "execute_result" } @@ -1032,7 +1022,7 @@ "rule_13 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"-\",\"+\"],is_required=\"false\"),\n", @@ -1056,7 +1046,7 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 74, "metadata": {}, "outputs": [ { @@ -1079,16 +1069,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['743500642516048']" + "[]" ] }, - "execution_count": 132, + "execution_count": 74, "metadata": {}, "output_type": "execute_result" } @@ -1098,7 +1088,7 @@ "rule_14 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}\",\n", " \"pattern\": [\n", " shape_token(shape =[\"ddddd\"],is_in_output=\"true\"),\n", @@ -1123,7 +1113,7 @@ }, { "cell_type": "code", - "execution_count": 133, + "execution_count": 75, "metadata": {}, "outputs": [ { @@ -1146,16 +1136,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['5549408395', '0755716436']" + "[]" ] }, - "execution_count": 133, + "execution_count": 75, "metadata": {}, "output_type": "execute_result" } @@ -1165,7 +1155,7 @@ "rule_15 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}{4}{5}\",\n", " \"pattern\": [\n", " shape_token(shape =[\"dd\"],is_in_output=\"true\"),\n", @@ -1192,7 +1182,7 @@ }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 76, "metadata": {}, "outputs": [ { @@ -1215,16 +1205,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['15781424777', '15732190888']" + "[]" ] }, - "execution_count": 134, + "execution_count": 76, "metadata": {}, "output_type": "execute_result" } @@ -1234,7 +1224,7 @@ "rule_16 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"+\"],is_required=\"false\"),\n", @@ -1261,7 +1251,7 @@ }, { "cell_type": "code", - "execution_count": 135, + "execution_count": 77, "metadata": {}, "outputs": [ { @@ -1284,16 +1274,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['004915221040240']" + "[]" ] }, - "execution_count": 135, + "execution_count": 77, "metadata": {}, "output_type": "execute_result" } @@ -1303,7 +1293,7 @@ "rule_17 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " number_token(minimum=\"4900000000000\", maximum=\"4999999999999\", length = [15],is_in_output=\"true\")\n", @@ -1326,7 +1316,7 @@ }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 78, "metadata": {}, "outputs": [ { @@ -1349,16 +1339,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['15771824788']" + "[]" ] }, - "execution_count": 136, + "execution_count": 78, "metadata": {}, "output_type": "execute_result" } @@ -1369,7 +1359,7 @@ "rule_18 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", " number_token(number =[49],is_in_output=\"false\"),\n", @@ -1396,7 +1386,7 @@ }, { "cell_type": "code", - "execution_count": 137, + "execution_count": 79, "metadata": {}, "outputs": [ { @@ -1419,7 +1409,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -1444,7 +1434,7 @@ " '0502222']" ] }, - "execution_count": 137, + "execution_count": 79, "metadata": {}, "output_type": "execute_result" } @@ -1481,7 +1471,7 @@ }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 80, "metadata": {}, "outputs": [ { @@ -1504,7 +1494,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -1513,13 +1503,13 @@ "[]" ] }, - "execution_count": 138, + "execution_count": 80, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "#Uk 07077080500 \n", + "#Uk 07077080500 (careful)\n", "\n", "\n", "rule_19 = {\n", @@ -1548,7 +1538,7 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": 81, "metadata": {}, "outputs": [ { @@ -1571,7 +1561,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -1580,7 +1570,7 @@ "['07741011066]', '07014231011]']" ] }, - "execution_count": 139, + "execution_count": 81, "metadata": {}, "output_type": "execute_result" } @@ -1619,7 +1609,7 @@ }, { "cell_type": "code", - "execution_count": 140, + "execution_count": 82, "metadata": {}, "outputs": [ { @@ -1642,7 +1632,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -1651,7 +1641,7 @@ "['07510114192']" ] }, - "execution_count": 140, + "execution_count": 82, "metadata": {}, "output_type": "execute_result" } @@ -1692,7 +1682,7 @@ }, { "cell_type": "code", - "execution_count": 141, + "execution_count": 83, "metadata": {}, "outputs": [ { @@ -1715,7 +1705,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -1724,7 +1714,7 @@ "['02211112222']" ] }, - "execution_count": 141, + "execution_count": 83, "metadata": {}, "output_type": "execute_result" } @@ -1766,7 +1756,7 @@ }, { "cell_type": "code", - "execution_count": 142, + "execution_count": 84, "metadata": {}, "outputs": [ { @@ -1789,7 +1779,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -1798,7 +1788,7 @@ "['01000001113', '01211112228', '01211112229', '01112222221']" ] }, - "execution_count": 142, + "execution_count": 84, "metadata": {}, "output_type": "execute_result" } @@ -1838,7 +1828,7 @@ }, { "cell_type": "code", - "execution_count": 143, + "execution_count": 85, "metadata": {}, "outputs": [ { @@ -1861,7 +1851,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -1870,7 +1860,7 @@ "['0122222224']" ] }, - "execution_count": 143, + "execution_count": 85, "metadata": {}, "output_type": "execute_result" } @@ -1907,7 +1897,7 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 86, "metadata": {}, "outputs": [ { @@ -1930,7 +1920,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -1939,7 +1929,7 @@ "['01222333335']" ] }, - "execution_count": 144, + "execution_count": 86, "metadata": {}, "output_type": "execute_result" } @@ -1977,7 +1967,7 @@ }, { "cell_type": "code", - "execution_count": 145, + "execution_count": 87, "metadata": {}, "outputs": [ { @@ -2000,7 +1990,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -2009,7 +1999,7 @@ "['0122223336']" ] }, - "execution_count": 145, + "execution_count": 87, "metadata": {}, "output_type": "execute_result" } @@ -2048,7 +2038,7 @@ }, { "cell_type": "code", - "execution_count": 146, + "execution_count": 88, "metadata": {}, "outputs": [ { @@ -2071,7 +2061,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -2080,7 +2070,7 @@ "['01222233337']" ] }, - "execution_count": 146, + "execution_count": 88, "metadata": {}, "output_type": "execute_result" } @@ -2119,7 +2109,7 @@ }, { "cell_type": "code", - "execution_count": 147, + "execution_count": 89, "metadata": {}, "outputs": [ { @@ -2142,7 +2132,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -2151,7 +2141,7 @@ "[]" ] }, - "execution_count": 147, + "execution_count": 89, "metadata": {}, "output_type": "execute_result" } @@ -2193,7 +2183,7 @@ }, { "cell_type": "code", - "execution_count": 148, + "execution_count": 90, "metadata": {}, "outputs": [ { @@ -2216,7 +2206,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -2225,7 +2215,7 @@ "['07111222220']" ] }, - "execution_count": 148, + "execution_count": 90, "metadata": {}, "output_type": "execute_result" } @@ -2264,7 +2254,7 @@ }, { "cell_type": "code", - "execution_count": 149, + "execution_count": 91, "metadata": {}, "outputs": [ { @@ -2287,7 +2277,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -2296,7 +2286,7 @@ "['01000001113', '01211112228', '01211112229', '01112222221', '00861112222']" ] }, - "execution_count": 149, + "execution_count": 91, "metadata": {}, "output_type": "execute_result" } @@ -2336,7 +2326,7 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 92, "metadata": {}, "outputs": [ { @@ -2359,7 +2349,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -2368,7 +2358,7 @@ "['0500111112']" ] }, - "execution_count": 150, + "execution_count": 92, "metadata": {}, "output_type": "execute_result" } @@ -2407,7 +2397,7 @@ }, { "cell_type": "code", - "execution_count": 151, + "execution_count": 93, "metadata": {}, "outputs": [ { @@ -2430,7 +2420,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -2439,7 +2429,7 @@ "['0800111113']" ] }, - "execution_count": 151, + "execution_count": 93, "metadata": {}, "output_type": "execute_result" } @@ -2478,7 +2468,7 @@ }, { "cell_type": "code", - "execution_count": 152, + "execution_count": 94, "metadata": {}, "outputs": [ { @@ -2501,16 +2491,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['3221719850', '3221719851', '3221719850', '1112222222']" + "[]" ] }, - "execution_count": 152, + "execution_count": 94, "metadata": {}, "output_type": "execute_result" } @@ -2521,7 +2511,7 @@ "rule_32 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -2549,7 +2539,7 @@ }, { "cell_type": "code", - "execution_count": 153, + "execution_count": 95, "metadata": {}, "outputs": [ { @@ -2572,16 +2562,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['0122222224', '1111133333']" + "[]" ] }, - "execution_count": 153, + "execution_count": 95, "metadata": {}, "output_type": "execute_result" } @@ -2591,7 +2581,7 @@ "rule_33 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -2619,7 +2609,7 @@ }, { "cell_type": "code", - "execution_count": 154, + "execution_count": 96, "metadata": {}, "outputs": [ { @@ -2642,16 +2632,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['1112223333']" + "[]" ] }, - "execution_count": 154, + "execution_count": 96, "metadata": {}, "output_type": "execute_result" } @@ -2661,7 +2651,7 @@ "rule_34 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -2693,7 +2683,7 @@ }, { "cell_type": "code", - "execution_count": 155, + "execution_count": 97, "metadata": {}, "outputs": [ { @@ -2716,16 +2706,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['011111112222']" + "[]" ] }, - "execution_count": 155, + "execution_count": 97, "metadata": {}, "output_type": "execute_result" } @@ -2735,7 +2725,7 @@ "rule_35 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -2764,7 +2754,7 @@ }, { "cell_type": "code", - "execution_count": 156, + "execution_count": 98, "metadata": {}, "outputs": [ { @@ -2787,16 +2777,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['12233334444']" + "[]" ] }, - "execution_count": 156, + "execution_count": 98, "metadata": {}, "output_type": "execute_result" } @@ -2806,7 +2796,7 @@ "rule_36 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -2838,7 +2828,7 @@ }, { "cell_type": "code", - "execution_count": 157, + "execution_count": 99, "metadata": {}, "outputs": [ { @@ -2861,16 +2851,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['11122223333']" + "[]" ] }, - "execution_count": 157, + "execution_count": 99, "metadata": {}, "output_type": "execute_result" } @@ -2880,7 +2870,7 @@ "rule_37 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -2912,7 +2902,7 @@ }, { "cell_type": "code", - "execution_count": 158, + "execution_count": 100, "metadata": {}, "outputs": [ { @@ -2935,16 +2925,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['11122233444']" + "[]" ] }, - "execution_count": 158, + "execution_count": 100, "metadata": {}, "output_type": "execute_result" } @@ -2954,7 +2944,7 @@ "rule_38 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}{4}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -2988,7 +2978,7 @@ }, { "cell_type": "code", - "execution_count": 159, + "execution_count": 101, "metadata": {}, "outputs": [ { @@ -3011,33 +3001,26 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['11112222',\n", - " '01111111',\n", - " '11112222',\n", - " '33334444',\n", - " '22223333',\n", - " '11112222',\n", - " '22223333',\n", - " '11112222']" + "[]" ] }, - "execution_count": 159, + "execution_count": 101, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# HK: XXXX YYYY , \n", + "# HK: XXXX YYYY , (careful)\n", "rule_39 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -3064,7 +3047,7 @@ }, { "cell_type": "code", - "execution_count": 160, + "execution_count": 102, "metadata": {}, "outputs": [ { @@ -3087,7 +3070,7 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { @@ -3096,7 +3079,7 @@ "[]" ] }, - "execution_count": 160, + "execution_count": 102, "metadata": {}, "output_type": "execute_result" } @@ -3129,7 +3112,7 @@ }, { "cell_type": "code", - "execution_count": 161, + "execution_count": 103, "metadata": {}, "outputs": [ { @@ -3152,16 +3135,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['0112223333']" + "[]" ] }, - "execution_count": 161, + "execution_count": 103, "metadata": {}, "output_type": "execute_result" } @@ -3171,7 +3154,7 @@ "rule_41 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -3200,7 +3183,7 @@ }, { "cell_type": "code", - "execution_count": 162, + "execution_count": 104, "metadata": {}, "outputs": [ { @@ -3223,16 +3206,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['0120223333']" + "[]" ] }, - "execution_count": 162, + "execution_count": 104, "metadata": {}, "output_type": "execute_result" } @@ -3242,7 +3225,7 @@ "rule_42 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -3271,7 +3254,7 @@ }, { "cell_type": "code", - "execution_count": 163, + "execution_count": 105, "metadata": {}, "outputs": [ { @@ -3294,16 +3277,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['0570223333']" + "[]" ] }, - "execution_count": 163, + "execution_count": 105, "metadata": {}, "output_type": "execute_result" } @@ -3313,7 +3296,7 @@ "rule_43 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -3342,7 +3325,7 @@ }, { "cell_type": "code", - "execution_count": 164, + "execution_count": 106, "metadata": {}, "outputs": [ { @@ -3365,16 +3348,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['0800223333']" + "[]" ] }, - "execution_count": 164, + "execution_count": 106, "metadata": {}, "output_type": "execute_result" } @@ -3384,7 +3367,7 @@ "rule_44 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -3413,7 +3396,7 @@ }, { "cell_type": "code", - "execution_count": 165, + "execution_count": 107, "metadata": {}, "outputs": [ { @@ -3436,16 +3419,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['05022223333']" + "[]" ] }, - "execution_count": 165, + "execution_count": 107, "metadata": {}, "output_type": "execute_result" } @@ -3455,7 +3438,7 @@ "rule_45 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -3484,7 +3467,7 @@ }, { "cell_type": "code", - "execution_count": 166, + "execution_count": 108, "metadata": {}, "outputs": [ { @@ -3507,16 +3490,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['0111112222']" + "[]" ] }, - "execution_count": 166, + "execution_count": 108, "metadata": {}, "output_type": "execute_result" } @@ -3526,7 +3509,7 @@ "rule_46 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -3555,7 +3538,7 @@ }, { "cell_type": "code", - "execution_count": 167, + "execution_count": 109, "metadata": {}, "outputs": [ { @@ -3578,16 +3561,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['422223333']" + "[]" ] }, - "execution_count": 167, + "execution_count": 109, "metadata": {}, "output_type": "execute_result" } @@ -3597,7 +3580,7 @@ "rule_47 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -3627,7 +3610,7 @@ }, { "cell_type": "code", - "execution_count": 168, + "execution_count": 110, "metadata": {}, "outputs": [ { @@ -3650,16 +3633,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['222333444']" + "[]" ] }, - "execution_count": 168, + "execution_count": 110, "metadata": {}, "output_type": "execute_result" } @@ -3669,7 +3652,7 @@ "rule_48 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " punctuation_token(token=[\"(\"],is_required=\"false\"),\n", @@ -3701,7 +3684,7 @@ }, { "cell_type": "code", - "execution_count": 169, + "execution_count": 111, "metadata": {}, "outputs": [ { @@ -3724,16 +3707,16 @@ " China: (0111) 1111 2222, +86 122 3333 4444, 0086 111 2222 3333, +86 111 222 33 444\n", " HK: 1111 2222, 33334444\n", " Japan: (011) 222-3333, (0120)-22-3333, 0570-22-3333, 0800-22-3333,050-2222-3333, +61 \n", - " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444\n" + " Austrilia: (01)1111 2222, +61 42222 3333, +61 222 333 444, 1003 333 444, Les chantiers de St-Nazaire (1881-1950)• Le jubilé des Old Gaffers\n" ] }, { "data": { "text/plain": [ - "['1003333444']" + "[]" ] }, - "execution_count": 169, + "execution_count": 111, "metadata": {}, "output_type": "execute_result" } @@ -3743,7 +3726,7 @@ "rule_49 = {\n", " \"identifier\": \"telenum_rule_us\",\n", " \"description\": \"a description\",\n", - " \"is_active\": \"true\",\n", + " \"is_active\": \"false\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", " number_token(minimum=\"1000\", maximum=\"1999\",length = [4], is_in_output=\"true\"), \n", @@ -3770,7 +3753,7 @@ }, { "cell_type": "code", - "execution_count": 170, + "execution_count": 112, "metadata": { "collapsed": true }, @@ -3797,11 +3780,825 @@ }, { "cell_type": "code", - "execution_count": 171, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 113, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[\n", + " {\n", + " \"context\": {\n", + " \"end\": 10, \n", + " \"tokens_left\": [\n", + " \"my\", \n", + " \"telephone\", \n", + " \"number\", \n", + " \"is\"\n", + " ], \n", + " \"text\": \"my telephone number is ( 217 ) 331 - 6779 , ( 217 ) - \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"(\", \n", + " \"217\", \n", + " \")\", \n", + " \"-\"\n", + " ], \n", + " \"start\": 4, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 0\n", + " }, \n", + " \"value\": \"2173316779\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 18, \n", + " \"tokens_left\": [\n", + " \")\", \n", + " \"331\", \n", + " \"-\", \n", + " \"6779\", \n", + " \",\"\n", + " ], \n", + " \"text\": \") 331 - 6779 , ( 217 ) - 331 - 6778 , ( 217 ) - \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"(\", \n", + " \"217\", \n", + " \")\", \n", + " \"-\"\n", + " ], \n", + " \"start\": 11, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 0\n", + " }, \n", + " \"value\": \"2173316778\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 26, \n", + " \"tokens_left\": [\n", + " \"-\", \n", + " \"331\", \n", + " \"-\", \n", + " \"6778\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"- 331 - 6778 , ( 217 ) - 331 - 6777 , 217 - 331 - \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"217\", \n", + " \"-\", \n", + " \"331\", \n", + " \"-\"\n", + " ], \n", + " \"start\": 19, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 0\n", + " }, \n", + " \"value\": \"2173316777\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 32, \n", + " \"tokens_left\": [\n", + " \"-\", \n", + " \"331\", \n", + " \"-\", \n", + " \"6777\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"- 331 - 6777 , 217 - 331 - 6776 , 734 . 709 . \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"734\", \n", + " \".\", \n", + " \"709\", \n", + " \".\"\n", + " ], \n", + " \"start\": 27, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 0\n", + " }, \n", + " \"value\": \"2173316776\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 38, \n", + " \"tokens_left\": [\n", + " \"-\", \n", + " \"331\", \n", + " \"-\", \n", + " \"6776\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"- 331 - 6776 , 734 . 709 . 8965 , 949 484 6951 , \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"949\", \n", + " \"484\", \n", + " \"6951\", \n", + " \",\"\n", + " ], \n", + " \"start\": 33, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 0\n", + " }, \n", + " \"value\": \"7347098965\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 42, \n", + " \"tokens_left\": [\n", + " \".\", \n", + " \"709\", \n", + " \".\", \n", + " \"8965\", \n", + " \",\"\n", + " ], \n", + " \"text\": \". 709 . 8965 , 949 484 6951 , 5017774643 , 664 123 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"5017774643\", \n", + " \",\", \n", + " \"664\", \n", + " \"123\"\n", + " ], \n", + " \"start\": 39, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 0\n", + " }, \n", + " \"value\": \"9494846951\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 44, \n", + " \"tokens_left\": [\n", + " \",\", \n", + " \"949\", \n", + " \"484\", \n", + " \"6951\", \n", + " \",\"\n", + " ], \n", + " \"text\": \", 949 484 6951 , 5017774643 , 664 123 45 67 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"664\", \n", + " \"123\", \n", + " \"45\", \n", + " \"67\"\n", + " ], \n", + " \"start\": 43, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 1\n", + " }, \n", + " \"value\": \"5017774643\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 51, \n", + " \"tokens_left\": [\n", + " \"664\", \n", + " \"123\", \n", + " \"45\", \n", + " \"67\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"664 123 45 67 , 0660852222 \\n Hi Gentlemen , My \", \n", + " \"tokens_right\": [\n", + " \"\\n\", \n", + " \"Hi\", \n", + " \"Gentlemen\", \n", + " \",\", \n", + " \"My\"\n", + " ], \n", + " \"start\": 50, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 1\n", + " }, \n", + " \"value\": \"0660852222\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 81, \n", + " \"tokens_left\": [\n", + " \"My\", \n", + " \"names\", \n", + " \"is\", \n", + " \"Alanda\", \n", + " \"\\n\"\n", + " ], \n", + " \"text\": \"My names is Alanda \\n ( 217 ) - 331 - 6778 , ( 044 ) 322 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"(\", \n", + " \"044\", \n", + " \")\", \n", + " \"322\"\n", + " ], \n", + " \"start\": 74, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 0\n", + " }, \n", + " \"value\": \"2173316778\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 117, \n", + " \"tokens_left\": [\n", + " \"83\", \n", + " \"94\", \n", + " \",\", \n", + " \"045\", \n", + " \"-\"\n", + " ], \n", + " \"text\": \"83 94 , 045 - 5549408395 \\n - 0466119200 , - \", \n", + " \"tokens_right\": [\n", + " \"\\n\", \n", + " \"-\", \n", + " \"0466119200\", \n", + " \",\", \n", + " \"-\"\n", + " ], \n", + " \"start\": 116, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 1\n", + " }, \n", + " \"value\": \"5549408395\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 120, \n", + " \"tokens_left\": [\n", + " \"045\", \n", + " \"-\", \n", + " \"5549408395\", \n", + " \"\\n\", \n", + " \"-\"\n", + " ], \n", + " \"text\": \"045 - 5549408395 \\n - 0466119200 , - 488019954 , + \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"-\", \n", + " \"488019954\", \n", + " \",\", \n", + " \"+\"\n", + " ], \n", + " \"start\": 119, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 1\n", + " }, \n", + " \"value\": \"0466119200\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 129, \n", + " \"tokens_left\": [\n", + " \",\", \n", + " \"+\", \n", + " \"32465863497\", \n", + " \"-\", \n", + " \",\"\n", + " ], \n", + " \"text\": \", + 32465863497 - , 0466119200 \\n 74350 - 0642516048 , \", \n", + " \"tokens_right\": [\n", + " \"\\n\", \n", + " \"74350\", \n", + " \"-\", \n", + " \"0642516048\", \n", + " \",\"\n", + " ], \n", + " \"start\": 128, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 1\n", + " }, \n", + " \"value\": \"0466119200\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 133, \n", + " \"tokens_left\": [\n", + " \",\", \n", + " \"0466119200\", \n", + " \"\\n\", \n", + " \"74350\", \n", + " \"-\"\n", + " ], \n", + " \"text\": \", 0466119200 \\n 74350 - 0642516048 , 07 55 71 64 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"07\", \n", + " \"55\", \n", + " \"71\", \n", + " \"64\"\n", + " ], \n", + " \"start\": 132, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 1\n", + " }, \n", + " \"value\": \"0642516048\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 174, \n", + " \"tokens_left\": [\n", + " \"\\n\", \n", + " \"UK\", \n", + " \":\", \n", + " \"07077080500\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"\\n UK : 07077080500 , 07741 011 066 , 07014 - 231 - \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"07014\", \n", + " \"-\", \n", + " \"231\", \n", + " \"-\"\n", + " ], \n", + " \"start\": 171, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 18\n", + " }, \n", + " \"value\": \"07741011066]\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 180, \n", + " \"tokens_left\": [\n", + " \",\", \n", + " \"07741\", \n", + " \"011\", \n", + " \"066\", \n", + " \",\"\n", + " ], \n", + " \"text\": \", 07741 011 066 , 07014 - 231 - 011 , 0751 011 41 92 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"0751\", \n", + " \"011\", \n", + " \"41\", \n", + " \"92\"\n", + " ], \n", + " \"start\": 175, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 18\n", + " }, \n", + " \"value\": \"07014231011]\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 185, \n", + " \"tokens_left\": [\n", + " \"-\", \n", + " \"231\", \n", + " \"-\", \n", + " \"011\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"- 231 - 011 , 0751 011 41 92 , \\n UK : ( \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"\\n\", \n", + " \"UK\", \n", + " \":\", \n", + " \"(\"\n", + " ], \n", + " \"start\": 181, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 19\n", + " }, \n", + " \"value\": \"07510114192\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 194, \n", + " \"tokens_left\": [\n", + " \"92\", \n", + " \",\", \n", + " \"\\n\", \n", + " \"UK\", \n", + " \":\"\n", + " ], \n", + " \"text\": \"92 , \\n UK : ( 022 ) 1111 2222 , ( 0100 ) 000 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"(\", \n", + " \"0100\", \n", + " \")\", \n", + " \"000\"\n", + " ], \n", + " \"start\": 189, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 20\n", + " }, \n", + " \"value\": \"02211112222\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 200, \n", + " \"tokens_left\": [\n", + " \"022\", \n", + " \")\", \n", + " \"1111\", \n", + " \"2222\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"022 ) 1111 2222 , ( 0100 ) 000 1113 , ( 01222 ) 22224 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"(\", \n", + " \"01222\", \n", + " \")\", \n", + " \"22224\"\n", + " ], \n", + " \"start\": 195, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 21\n", + " }, \n", + " \"value\": \"01000001113\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 205, \n", + " \"tokens_left\": [\n", + " \"0100\", \n", + " \")\", \n", + " \"000\", \n", + " \"1113\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"0100 ) 000 1113 , ( 01222 ) 22224 , ( 01222 ) 333335 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"(\", \n", + " \"01222\", \n", + " \")\", \n", + " \"333335\"\n", + " ], \n", + " \"start\": 201, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 22\n", + " }, \n", + " \"value\": \"0122222224\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 210, \n", + " \"tokens_left\": [\n", + " \"(\", \n", + " \"01222\", \n", + " \")\", \n", + " \"22224\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"( 01222 ) 22224 , ( 01222 ) 333335 , ( 0122 22 ) \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"(\", \n", + " \"0122\", \n", + " \"22\", \n", + " \")\"\n", + " ], \n", + " \"start\": 206, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 23\n", + " }, \n", + " \"value\": \"01222333335\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 216, \n", + " \"tokens_left\": [\n", + " \"(\", \n", + " \"01222\", \n", + " \")\", \n", + " \"333335\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"( 01222 ) 333335 , ( 0122 22 ) 3336 , ( 0122 22 ) \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"(\", \n", + " \"0122\", \n", + " \"22\", \n", + " \")\"\n", + " ], \n", + " \"start\": 211, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 24\n", + " }, \n", + " \"value\": \"0122223336\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 222, \n", + " \"tokens_left\": [\n", + " \"0122\", \n", + " \"22\", \n", + " \")\", \n", + " \"3336\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"0122 22 ) 3336 , ( 0122 22 ) 33337 , \\n UK : 0121 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"\\n\", \n", + " \"UK\", \n", + " \":\", \n", + " \"0121\"\n", + " ], \n", + " \"start\": 217, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 25\n", + " }, \n", + " \"value\": \"01222233337\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 230, \n", + " \"tokens_left\": [\n", + " \"33337\", \n", + " \",\", \n", + " \"\\n\", \n", + " \"UK\", \n", + " \":\"\n", + " ], \n", + " \"text\": \"33337 , \\n UK : 0121 - 111 2228 , 0121 111 2229 , \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"0121\", \n", + " \"111\", \n", + " \"2229\", \n", + " \",\"\n", + " ], \n", + " \"start\": 226, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 21\n", + " }, \n", + " \"value\": \"01211112228\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 234, \n", + " \"tokens_left\": [\n", + " \"0121\", \n", + " \"-\", \n", + " \"111\", \n", + " \"2228\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"0121 - 111 2228 , 0121 111 2229 , 07111 222220 , 0111 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"07111\", \n", + " \"222220\", \n", + " \",\", \n", + " \"0111\"\n", + " ], \n", + " \"start\": 231, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 21\n", + " }, \n", + " \"value\": \"01211112229\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 237, \n", + " \"tokens_left\": [\n", + " \",\", \n", + " \"0121\", \n", + " \"111\", \n", + " \"2229\", \n", + " \",\"\n", + " ], \n", + " \"text\": \", 0121 111 2229 , 07111 222220 , 0111 222 2221 , \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"0111\", \n", + " \"222\", \n", + " \"2221\", \n", + " \",\"\n", + " ], \n", + " \"start\": 235, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 27\n", + " }, \n", + " \"value\": \"07111222220\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 241, \n", + " \"tokens_left\": [\n", + " \"2229\", \n", + " \",\", \n", + " \"07111\", \n", + " \"222220\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"2229 , 07111 222220 , 0111 222 2221 , 0500 111112 , 0800 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"0500\", \n", + " \"111112\", \n", + " \",\", \n", + " \"0800\"\n", + " ], \n", + " \"start\": 238, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 21\n", + " }, \n", + " \"value\": \"01112222221\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 244, \n", + " \"tokens_left\": [\n", + " \",\", \n", + " \"0111\", \n", + " \"222\", \n", + " \"2221\", \n", + " \",\"\n", + " ], \n", + " \"text\": \", 0111 222 2221 , 0500 111112 , 0800 111113 , \\n \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"0800\", \n", + " \"111113\", \n", + " \",\", \n", + " \"\\n\"\n", + " ], \n", + " \"start\": 242, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 29\n", + " }, \n", + " \"value\": \"0500111112\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 247, \n", + " \"tokens_left\": [\n", + " \"2221\", \n", + " \",\", \n", + " \"0500\", \n", + " \"111112\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"2221 , 0500 111112 , 0800 111113 , \\n India : 111 \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"\\n\", \n", + " \"India\", \n", + " \":\", \n", + " \"111\"\n", + " ], \n", + " \"start\": 245, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 30\n", + " }, \n", + " \"value\": \"0800111113\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 265, \n", + " \"tokens_left\": [\n", + " \"33333\", \n", + " \",\", \n", + " \"+\", \n", + " \"91\", \n", + " \"-\"\n", + " ], \n", + " \"text\": \"33333 , + 91 - 111 222 3333 \\n China : ( 0111 \", \n", + " \"tokens_right\": [\n", + " \"\\n\", \n", + " \"China\", \n", + " \":\", \n", + " \"(\", \n", + " \"0111\"\n", + " ], \n", + " \"start\": 262, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 0\n", + " }, \n", + " \"value\": \"1112223333\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 283, \n", + " \"tokens_left\": [\n", + " \"86\", \n", + " \"122\", \n", + " \"3333\", \n", + " \"4444\", \n", + " \",\"\n", + " ], \n", + " \"text\": \"86 122 3333 4444 , 0086 111 2222 3333 , + 86 111 \", \n", + " \"tokens_right\": [\n", + " \"3333\", \n", + " \",\", \n", + " \"+\", \n", + " \"86\", \n", + " \"111\"\n", + " ], \n", + " \"start\": 280, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 28\n", + " }, \n", + " \"value\": \"00861112222\"\n", + " }, \n", + " {\n", + " \"context\": {\n", + " \"end\": 307, \n", + " \"tokens_left\": [\n", + " \",\", \n", + " \"33334444\", \n", + " \"\\n\", \n", + " \"Japan\", \n", + " \":\"\n", + " ], \n", + " \"text\": \", 33334444 \\n Japan : ( 011 ) 222 - 3333 , ( 0120 ) - \", \n", + " \"tokens_right\": [\n", + " \",\", \n", + " \"(\", \n", + " \"0120\", \n", + " \")\", \n", + " \"-\"\n", + " ], \n", + " \"start\": 301, \n", + " \"input\": \"tokens\", \n", + " \"identifier\": \"telenum_rule_us\", \n", + " \"rule_id\": 0\n", + " }, \n", + " \"value\": \"0112223333\"\n", + " }, \n", + " [\n", + " \"2173316779\", \n", + " \"2173316778\", \n", + " \"2173316777\", \n", + " \"2173316776\", \n", + " \"7347098965\", \n", + " \"9494846951\", \n", + " \"5017774643\", \n", + " \"0660852222\", \n", + " \"2173316778\", \n", + " \"5549408395\", \n", + " \"0466119200\", \n", + " \"0466119200\", \n", + " \"0642516048\", \n", + " \"07741011066]\", \n", + " \"07014231011]\", \n", + " \"07510114192\", \n", + " \"02211112222\", \n", + " \"01000001113\", \n", + " \"0122222224\", \n", + " \"01222333335\", \n", + " \"0122223336\", \n", + " \"01222233337\", \n", + " \"01211112228\", \n", + " \"01211112229\", \n", + " \"07111222220\", \n", + " \"01112222221\", \n", + " \"0500111112\", \n", + " \"0800111113\", \n", + " \"1112223333\", \n", + " \"00861112222\", \n", + " \"0112223333\"\n", + " ]\n", + "]\n" + ] + } + ], "source": [ "field_rules = {\n", " \"rules\": [\n", @@ -3867,7 +4664,7 @@ "for i in results:\n", " tele_lst.append(''.join((i.values()[1]).split()))\n", "results.append(tele_lst)\n", - "\n", + "print json.dumps(results, indent=2)\n", "\n", "field_rules['results']=results\n", "\n", @@ -3884,7 +4681,7 @@ }, { "cell_type": "code", - "execution_count": 172, + "execution_count": 114, "metadata": {}, "outputs": [ { diff --git a/etk_stock_symbol_rules.ipynb b/etk_stock_symbol_rules.ipynb index 03f6f4fb..ec9a47d6 100644 --- a/etk_stock_symbol_rules.ipynb +++ b/etk_stock_symbol_rules.ipynb @@ -617,7 +617,7 @@ " \"is_active\": \"true\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", - " word_token(token=[\"NYSE\",\"NASDAQ\"],is_in_output=\"false\"),\n", + " word_token(token=[\"NYSE\",\"NASDAQ\",\"OTCQB\"],is_in_output=\"false\"),\n", " punctuation_token(token=[\":\"], is_in_output=\"false\"),\n", "\n", " shape_token(shape =[\"X\",\"XX\",\"XXX\",\"XXXX\"], is_in_output=\"true\"),\n", @@ -684,7 +684,7 @@ " \"is_active\": \"true\",\n", " \"output_format\": \"{1}{2}{3}{4}{5}\",\n", " \"pattern\": [\n", - " word_token(token=[\"NYSE\",\"NASDAQ\"],is_in_output=\"false\"),\n", + " word_token(token=[\"NYSE\",\"NASDAQ\",\"OTCQB\"],is_in_output=\"false\"),\n", " punctuation_token(token=[\":\"], is_in_output=\"false\"),\n", "\n", " shape_token(shape =[\"X\",\"XX\",\"XXX\",\"XXXX\"], is_in_output=\"true\"),\n", @@ -754,7 +754,7 @@ " \"is_active\": \"true\",\n", " \"output_format\": \"{1}\",\n", " \"pattern\": [\n", - " word_token(token=[\"NYSE\",\"NASDAQ\"],is_in_output=\"false\"),\n", + " word_token(token=[\"NYSE\",\"NASDAQ\",\"OTCQB\"],is_in_output=\"false\"),\n", " punctuation_token(token=[\":\"], is_in_output=\"false\"),\n", "\n", " shape_token(shape =[\"X\",\"XX\",\"XXX\",\"XXXX\",\"XXXXX\"], is_in_output=\"true\")\n", @@ -778,9 +778,7 @@ { "cell_type": "code", "execution_count": 14, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -821,7 +819,7 @@ " \"is_active\": \"true\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", - " word_token(token=[\"NYSE\",\"NASDAQ\"],is_in_output=\"false\"),\n", + " word_token(token=[\"NYSE\",\"NASDAQ\",\"OTCQB\"],is_in_output=\"false\"),\n", " punctuation_token(token=[\":\"], is_in_output=\"false\"),\n", "\n", " shape_token(shape =[\"X\",\"XX\",\"XXX\"], is_in_output=\"true\"),\n", @@ -889,7 +887,7 @@ " \"is_active\": \"true\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", - " word_token(token=[\"NYSE\",\"NASDAQ\"],is_in_output=\"false\"),\n", + " word_token(token=[\"NYSE\",\"NASDAQ\",\"OTCQB\"],is_in_output=\"false\"),\n", " punctuation_token(token=[\":\"], is_in_output=\"false\"),\n", "\n", " shape_token(shape =[\"XX\",\"XXX\",\"XXXX\"], is_in_output=\"true\"),\n", @@ -957,7 +955,7 @@ " \"is_active\": \"true\",\n", " \"output_format\": \"{1}{2}{3}\",\n", " \"pattern\": [\n", - " word_token(token=[\"NYSE\",\"NASDAQ\"],is_in_output=\"false\"),\n", + " word_token(token=[\"NYSE\",\"NASDAQ\",\"OTCQB\"],is_in_output=\"false\"),\n", " punctuation_token(token=[\":\"], is_in_output=\"false\"),\n", "\n", " shape_token(shape =[\"XX\",\"XXX\",\"XXXX\"], is_in_output=\"true\"),\n", @@ -1025,7 +1023,7 @@ " \"is_active\": \"true\",\n", " \"output_format\": \"{1}{2}{3}{4}{5}\",\n", " \"pattern\": [\n", - " word_token(token=[\"NYSE\",\"NASDAQ\"],is_in_output=\"false\"),\n", + " word_token(token=[\"NYSE\",\"NASDAQ\",\"OTCQB\"],is_in_output=\"false\"),\n", " punctuation_token(token=[\":\"], is_in_output=\"false\"),\n", "\n", " shape_token(shape =[\"X\",\"XXX\"], is_in_output=\"true\"),\n", diff --git a/run_etk_spark.py b/run_etk_spark.py index 61a6d7f2..ce532f75 100644 --- a/run_etk_spark.py +++ b/run_etk_spark.py @@ -25,12 +25,28 @@ def remove_if_no_html(x): return False return True +def remove_extra_fields(x): + if 'content_extraction' in x: + ce = x['content_extraction'] + for key in ce.keys(): + t = ce[key] + if 'simple_tokens_original_case' in t: + t.pop('simple_tokens_original_case') + if 'simple_tokens' in t: + t.pop('simple_tokens') + if 'data_extraction' in t: + t.pop('data_extraction') + ce[key] = t + x['content_extraction'] = ce + return x + + if __name__ == '__main__': compression = "org.apache.hadoop.io.compress.GzipCodec" parser = OptionParser() parser.add_option("-p", "--partitions", action="store", - type="int", dest="partitions", default=0) + type="int", dest="partitions", default=1000) (c_options, args) = parser.parse_args() input_path = args[0] output_path = args[1] @@ -42,17 +58,15 @@ def remove_if_no_html(x): conf = SparkConf() extraction_config = json.load(codecs.open(extraction_config_path)) c = Core(extraction_config=extraction_config) - if partitions == 0: - input_rdd = sc.sequenceFile(input_path)#.partitionBy(1000) - else: - input_rdd = sc.sequenceFile(input_path).partitionBy(partitions) + + input_rdd = sc.sequenceFile(input_path)#.partitionBy(partitions) output_rdd = input_rdd.mapValues(json.loads).filter(lambda x: remove_if_no_html(x[1])).mapValues(add_doc_id)\ .mapValues(lambda x: c.process(x, create_knowledge_graph=True)) - output_rdd = output_rdd.filter(lambda x: x[1] is not None).mapValues(json.dumps) + output_rdd = output_rdd.filter(lambda x: x[1] is not None).mapValues(remove_extra_fields).mapValues(json.dumps) output_rdd.saveAsSequenceFile(output_path, compressionCodecClass=compression) - print sc.sequenceFile(input_path).count() - print sc.sequenceFile(output_path).count() + # print sc.sequenceFile(input_path).count() + # print sc.sequenceFile(output_path).count()