问Python连接由文本组成的关键字

EN

keywords = ['machine learning', 'data science', 'artificial intelligence']
texts = [
    'the new machine learning model built by google is revolutionary for the current state of artificial intelligence. it may change the way we are thinking', 
    'data science and artificial intelligence are two different fields, although they are interconnected. scientists from harvard are explaining it in a detailed presentation that could be found on our page.'
]

combinations_dict = {tuple(kwsplit):('_'.join(kwsplit), ()) 
                     for kwsplit in [kwstr.split() for kwstr in keywords]}
combinations_dict

{('machine', 'learning'): ('machine_learning', ()),
 ('data', 'science'): ('data_science', ()),
 ('artificial', 'intelligence'): ('artificial_intelligence', ())}

def combining_generator(tokens, comb_dict):
    buff = ()  # start with empty buffer
    for in_tok in tokens:
        buff += (in_tok,)  # add latest to buffer
        if len(buff) < 2:  # grow buffer to 2 tokens if possible
            continue
        # lookup what to do for current pair... 
        # ...defaulting to emit-[0]-item, keep-[1]-item in new buff
        out_tok, buff = comb_dict.get(buff, (buff[0], (buff[1],)))
        yield out_tok 
    if buff:
        yield buff[0]  # last solo token if any

tokenized_texts = [text.split() for text in texts]
retokenized_texts = [list(combining_generator(tokens, combinations_dict)) for tokens in tokenized_texts]
retokenized_texts

[
  ['the', 'new', 'machine_learning', 'model', 'built', 'by', 'google', 'is', 'revolutionary', 'for', 'the', 'current', 'state', 'of', 'artificial', 'intelligence.', 'it', 'may', 'change', 'the', 'way', 'we', 'are', 'thinking'], 
  ['data_science', 'and', 'artificial_intelligence', 'are', 'two', 'different', 'fields,', 'although', 'they', 'are', 'interconnected.', 'scientists', 'from', 'harvard', 'are', 'explaining', 'it', 'in', 'a', 'detailed', 'presentation', 'that', 'could', 'be', 'found', 'on', 'our', 'page.']
]

import re
tokenized_texts = [re.findall('\w+', text) for text in texts]
tokenized_texts

tokenized_texts = [re.findall(r'\w+|(?:[^\w\s])', text) for text in texts]
tokenized_texts

keywords = ['machine learning', 'data science', 'artificial intelligence']

texts = ['the new machine learning model built by google is revolutionary for the current state of artificial intelligence. it may change the way we are thinking', 'data science and artificial intelligence are two different fields, although they are interconnected. scientists from harvard are explaining it in a detailed presentation that could be found on our page.']

#Add underscore
for idx, text in enumerate(texts):
  for keyword in keywords:
    reload_text = texts[idx]
    if keyword in text:
      texts[idx] = reload_text.replace(keyword, keyword.replace(" ", "_"))

#Split text for each "." encountered
for idx, text in enumerate(texts):
  texts[idx] = list(filter(None, text.split(".")))
print(texts)

#Split text to get each word
for idx,text in enumerate(texts):
  for idx_s,sentence in enumerate(text):
    texts[idx][idx_s] = list(map(lambda x: re.sub("[,\.!?]", "", x), sentence.split())) #map to delete every undesired characters

print(texts)

[
    [
        ['the', 'new', 'machine_learning', 'model', 'built', 'by', 'google', 'is', 'revolutionary', 'for', 'the', 'current', 'state', 'of', 'artificial_intelligence'], 
        ['it', 'may', 'change', 'the', 'way', 'we', 'are', 'thinking']
    ], 
    [
        ['data_science', 'and', 'artificial_intelligence', 'are', 'two', 'different', 'fields', 'although', 'they', 'are', 'interconnected'], 
        ['scientists', 'from', 'harvard', 'are', 'explaining', 'it', 'in', 'a', 'detailed', 'presentation', 'that', 'could', 'be', 'found', 'on', 'our', 'page']
    ]
]