From e7122b862f061102510f01151e767e1ad6807296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ch=C3=A9rifBALDE?= Date: Thu, 31 Mar 2022 22:28:59 +0200 Subject: [PATCH] 31/03/22 - 22h30 --- Ela_Spacy.py | 1 + data_indexees.csv | 14 ++------------ ela_index_bdd_classes.py | 28 ++++++++++++++++++++-------- ela_output_test_file_pandas_2.txt | 16 +++------------- ela_spacy_common.py | 4 ++-- 5 files changed, 28 insertions(+), 35 deletions(-) diff --git a/Ela_Spacy.py b/Ela_Spacy.py index 8904972..201d6e2 100644 --- a/Ela_Spacy.py +++ b/Ela_Spacy.py @@ -189,6 +189,7 @@ INTJ : interjection, NOUN : nom, NUM : chiffre, PART : particule, PRON : pronom, SCONJ : conjonction subordonnée, SYM : symbole, VERB : verbe, X : autre ''' + def Ela_remove_pronoun(tab_tokens): mywords = [] diff --git a/data_indexees.csv b/data_indexees.csv index 8ef3a36..8a39a25 100644 --- a/data_indexees.csv +++ b/data_indexees.csv @@ -1,13 +1,3 @@ ,index,mots,occurence,moyenne,id_formation,source_field -0,0,reglement,1,0.08,8866,objectif -1,1,publiqu,1,0.08,8866,objectif -2,2,const,1,0.08,8866,objectif -3,3,format,1,0.08,8866,objectif -4,4,appliqu,1,0.08,8866,objectif -5,5,comptabilit,1,0.08,8866,objectif -6,6,object,1,0.08,8866,objectif -7,7,regl,1,0.08,8866,objectif -8,8,evolu,1,0.08,8866,objectif -9,9,princip,1,0.08,8866,objectif -10,10,maitris,1,0.08,8866,objectif -11,11,impose,1,0.08,8866,objectif +0,0,recherch,4,0.67,AUTO_CH03,default +1,1,mot,2,0.33,AUTO_CH03,default diff --git a/ela_index_bdd_classes.py b/ela_index_bdd_classes.py index f8d865e..5000a4f 100644 --- a/ela_index_bdd_classes.py +++ b/ela_index_bdd_classes.py @@ -5,6 +5,7 @@ import prj_common as mycommon from pymongo import ReturnDocument import inspect import sys, os +from operator import itemgetter, attrgetter client = MongoClient("mongodb://localhost:27017/") # database @@ -416,6 +417,8 @@ cette recherche concerne les 'tuple' avec "source_field":"default" ==> l'indexat Elle retourne l'id_externe de la formation ''' def ela_recherche_tokens(sentence): + + tab_training_id = [] tab_tokens = ls.Ela_Tokenize(sentence) tab_tokens2 = ls.Ela_remove_stop_words(tab_tokens) @@ -453,16 +456,25 @@ def ela_recherche_tokens(sentence): # collection collection = db["elaindex"] + print(" mot recherché tab_tokens4 = "+str(tab_tokens4)) + + pipe2 = [{ '$match': { 'mots': { '$in': tab_tokens4 } } }, + { '$group': { + '_id': "$id_formation", + 'count_class': {'$sum': 1}, + 'sum_occurence': {'$sum': '$occurence'} + } + }, + {'$sort': {'count_class': -1, 'sum_occurence': -1}} + ] + + for result in collection.aggregate(pipe2): + tab_training_id.append(str(result["_id"])) + print(result) + + print("resultat tab_training_id = "+str(tab_training_id)) - for token in tab_tokens4: - print(" #### Le tooken recherché est : "+str(token)) - for doc in collection.find({"mots":token, "source_field":"default"}): - #print(" Trouvé dans le cours N° "+str(doc["id_formation"])) - if str(doc["id_formation"]) not in tab_training_id : - tab_training_id.append(str(doc["id_formation"])) - #print(" Pour la phrase "+str(sentence)+", voici les formations correspondantes") - #print(tab_training_id) return tab_training_id diff --git a/ela_output_test_file_pandas_2.txt b/ela_output_test_file_pandas_2.txt index 5f3e9f7..7b1c8e2 100644 --- a/ela_output_test_file_pandas_2.txt +++ b/ela_output_test_file_pandas_2.txt @@ -1,13 +1,3 @@ - mots occurence moyenne id_formation source_field -0 reglement 1 0.08 8866 objectif -1 publiqu 1 0.08 8866 objectif -2 const 1 0.08 8866 objectif -3 format 1 0.08 8866 objectif -4 appliqu 1 0.08 8866 objectif -5 comptabilit 1 0.08 8866 objectif -6 object 1 0.08 8866 objectif -7 regl 1 0.08 8866 objectif -8 evolu 1 0.08 8866 objectif -9 princip 1 0.08 8866 objectif -10 maitris 1 0.08 8866 objectif -11 impose 1 0.08 8866 objectif \ No newline at end of file + mots occurence moyenne id_formation source_field +0 recherch 4 0.67 AUTO_CH03 default +1 mot 2 0.33 AUTO_CH03 default \ No newline at end of file diff --git a/ela_spacy_common.py b/ela_spacy_common.py index baa921c..fcd9726 100644 --- a/ela_spacy_common.py +++ b/ela_spacy_common.py @@ -1,6 +1,6 @@ def update_token_fr_pontuation(token_fr_pontuation): - tab_ponctuation = ['...', '.', ';', ',', ':', '!', '?', ')', '(', '[', ']', '{', '}', + tab_ponctuation = ['...', '.', ';', ',', ':', '!', '?', ')', '(', '[', ']', '{', '}','-', '=', '°', '#', '-', '/', '~', '&', '\\'] for val in tab_ponctuation: @@ -28,7 +28,7 @@ def update_stopWords(stopWords): 'le', 'mon', 'mais', 'futes', 'auraient', 'vais', 'va', 'faire', 'veux', 'voir', 'a', 'ca', 'donc', 'salut', 'allons', 'mets', 'bien', 'ici', 'aller', 'tous', 'avoir', 'alors', 'plus', 'petit', 'tres', 'etre', 'oui', 'toto', 'moins', 'heureux', 'rien', 'quand', 'trois', 'bientot', - 'ceci', 'voila', 'tout', 'touts', 'toute', 'fait', 'si'] + 'ceci', 'voila', 'tout', 'touts', 'toute', 'fait', 'si', '-'] for val in tab_stop_word: