From 1cec3cfb818f5a70c760948690789731212b7a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ch=C3=A9rifBALDE?= Date: Mon, 28 Mar 2022 20:34:49 +0200 Subject: [PATCH] Merge remote-tracking branch 'origin/master' # Conflicts: # Log/log_file.log --- Ela_Spacy.py | 14 +++++++- data_indexees.csv | 53 +++++++------------------------ ela_index_bdd_classes.py | 6 ++-- ela_output_test_file_pandas_2.txt | 52 ++++++++---------------------- prj_common.py | 12 +++++-- 5 files changed, 50 insertions(+), 87 deletions(-) diff --git a/Ela_Spacy.py b/Ela_Spacy.py index 17fcf50..c992bd7 100644 --- a/Ela_Spacy.py +++ b/Ela_Spacy.py @@ -173,7 +173,19 @@ d’un mot comme un seul et même mot. Par exemple, une fois que l’on applique le mot résultant est le même. ''' def Ela_stemmize(tab_tokens): - return [stemmer.stem(token.text) for token in tab_tokens] + + tab_ret_val = [] + print(" VERIFICATION SI LE MOT EST FR : " + str(tab_tokens)) + for mot in tab_tokens: + if( mycommon.check_word_in_fr_dict(str(mot)) ): + tab_ret_val.append(stemmer.stem(mot.text)) + else: + tab_ret_val.append(mot.text) + + print(" STMISATION TAB = "+str(tab_ret_val)) + + return tab_ret_val + #return [stemmer.stem(token.text) for token in tab_tokens] diff --git a/data_indexees.csv b/data_indexees.csv index ddf86bf..b16ada9 100644 --- a/data_indexees.csv +++ b/data_indexees.csv @@ -1,42 +1,13 @@ ,index,mots,occurence,moyenne,id_formation,source_field -0,0," - -",1,0.02,mysy_ytubes_04,default -1,1,analys,1,0.02,mysy_ytubes_04,default -2,2,bt,1,0.02,mysy_ytubes_04,default -3,3,licenc,1,0.02,mysy_ytubes_04,default -4,4,charg,1,0.02,mysy_ytubes_04,default -5,5,notion,1,0.02,mysy_ytubes_04,default -6,6,cout,3,0.05,mysy_ytubes_04,default -7,7,analyt,1,0.02,mysy_ytubes_04,default -8,8,ge,3,0.05,mysy_ytubes_04,default -9,9,gratuit,1,0.02,mysy_ytubes_04,default -10,10,depart,1,0.02,mysy_ytubes_04,default -11,11,iut,1,0.02,mysy_ytubes_04,default -12,12,comptabilit,6,0.1,mysy_ytubes_04,default -13,13,different,1,0.02,mysy_ytubes_04,default -14,14,niveau,1,0.02,mysy_ytubes_04,default -15,15,prix,1,0.02,mysy_ytubes_04,default -16,16,professeur,1,0.02,mysy_ytubes_04,default -17,17,sorbon,1,0.02,mysy_ytubes_04,default -18,18,introduct,1,0.02,mysy_ytubes_04,default -19,19,nord,1,0.02,mysy_ytubes_04,default -20,20,cour,3,0.05,mysy_ytubes_04,default -21,21,incorpore,1,0.02,mysy_ytubes_04,default -22,22,lign,1,0.02,mysy_ytubes_04,default -23,23,universit,1,0.02,mysy_ytubes_04,default -24,24,stmg,1,0.02,mysy_ytubes_04,default -25,25,villetan,1,0.02,mysy_ytubes_04,default -26,26,dcg,1,0.02,mysy_ytubes_04,default -27,27,marg,1,0.02,mysy_ytubes_04,default -28,28,general,1,0.02,mysy_ytubes_04,default -29,29,paris,1,0.02,mysy_ytubes_04,default -30,30,corinn,1,0.02,mysy_ytubes_04,default -31,31,debut,1,0.02,mysy_ytubes_04,default -32,32,calcul,1,0.02,mysy_ytubes_04,default -33,33,1,2,0.03,mysy_ytubes_04,default -34,34,gestion,4,0.07,mysy_ytubes_04,default -35,35," -",8,0.13,mysy_ytubes_04,default -36,36,zambotto,1,0.02,mysy_ytubes_04,default -37,37,resultat,1,0.02,mysy_ytubes_04,default +0,0,sommeil,1,0.07,8220,default +1,1,stress,1,0.07,8220,default +2,2,coaching,1,0.07,8220,default +3,3,equilibre,1,0.07,8220,default +4,4,relax,1,0.07,8220,default +5,5,typ,1,0.07,8220,default +6,6,trouv,1,0.07,8220,default +7,7,accompagn,2,0.14,8220,default +8,8,sante,2,0.14,8220,default +9,9,travail,1,0.07,8220,default +10,10,nutrit,1,0.07,8220,default +11,11,respir,1,0.07,8220,default diff --git a/ela_index_bdd_classes.py b/ela_index_bdd_classes.py index e4ab098..143b555 100644 --- a/ela_index_bdd_classes.py +++ b/ela_index_bdd_classes.py @@ -407,9 +407,6 @@ def ela_index_all_classes_obj(): return False, "Impossible d'indexer la BDD des formations ==> KO" - - - ''' Cette fonction recherche une chaine de caractère dans la base de données. @@ -435,6 +432,9 @@ def ela_recherche_tokens(sentence): mycommon.check_word_in_fr_dict(str(mot)) tab_tokens4 = ls.Ela_stemmize(tab_tokens3) + + print(" VERIF APRES STEMISATION : " + str(tab_tokens4)) + tab_tokens4.sort() ''' diff --git a/ela_output_test_file_pandas_2.txt b/ela_output_test_file_pandas_2.txt index 95fc463..532f8db 100644 --- a/ela_output_test_file_pandas_2.txt +++ b/ela_output_test_file_pandas_2.txt @@ -1,39 +1,13 @@ - mots occurence moyenne id_formation source_field -0 \n\n 1 0.02 mysy_ytubes_04 default -1 analys 1 0.02 mysy_ytubes_04 default -2 bt 1 0.02 mysy_ytubes_04 default -3 licenc 1 0.02 mysy_ytubes_04 default -4 charg 1 0.02 mysy_ytubes_04 default -5 notion 1 0.02 mysy_ytubes_04 default -6 cout 3 0.05 mysy_ytubes_04 default -7 analyt 1 0.02 mysy_ytubes_04 default -8 ge 3 0.05 mysy_ytubes_04 default -9 gratuit 1 0.02 mysy_ytubes_04 default -10 depart 1 0.02 mysy_ytubes_04 default -11 iut 1 0.02 mysy_ytubes_04 default -12 comptabilit 6 0.1 mysy_ytubes_04 default -13 different 1 0.02 mysy_ytubes_04 default -14 niveau 1 0.02 mysy_ytubes_04 default -15 prix 1 0.02 mysy_ytubes_04 default -16 professeur 1 0.02 mysy_ytubes_04 default -17 sorbon 1 0.02 mysy_ytubes_04 default -18 introduct 1 0.02 mysy_ytubes_04 default -19 nord 1 0.02 mysy_ytubes_04 default -20 cour 3 0.05 mysy_ytubes_04 default -21 incorpore 1 0.02 mysy_ytubes_04 default -22 lign 1 0.02 mysy_ytubes_04 default -23 universit 1 0.02 mysy_ytubes_04 default -24 stmg 1 0.02 mysy_ytubes_04 default -25 villetan 1 0.02 mysy_ytubes_04 default -26 dcg 1 0.02 mysy_ytubes_04 default -27 marg 1 0.02 mysy_ytubes_04 default -28 general 1 0.02 mysy_ytubes_04 default -29 paris 1 0.02 mysy_ytubes_04 default -30 corinn 1 0.02 mysy_ytubes_04 default -31 debut 1 0.02 mysy_ytubes_04 default -32 calcul 1 0.02 mysy_ytubes_04 default -33 1 2 0.03 mysy_ytubes_04 default -34 gestion 4 0.07 mysy_ytubes_04 default -35 \n 8 0.13 mysy_ytubes_04 default -36 zambotto 1 0.02 mysy_ytubes_04 default -37 resultat 1 0.02 mysy_ytubes_04 default \ No newline at end of file + mots occurence moyenne id_formation source_field +0 sommeil 1 0.07 8220 default +1 stress 1 0.07 8220 default +2 coaching 1 0.07 8220 default +3 equilibre 1 0.07 8220 default +4 relax 1 0.07 8220 default +5 typ 1 0.07 8220 default +6 trouv 1 0.07 8220 default +7 accompagn 2 0.14 8220 default +8 sante 2 0.14 8220 default +9 travail 1 0.07 8220 default +10 nutrit 1 0.07 8220 default +11 respir 1 0.07 8220 default \ No newline at end of file diff --git a/prj_common.py b/prj_common.py index 18777c7..02d8e5c 100644 --- a/prj_common.py +++ b/prj_common.py @@ -13,6 +13,7 @@ import csv import inspect import sys from datetime import datetime +from pymongo import ReturnDocument TOKEN_SIZE = 25 CONNECTION_STRING = "mongodb://localhost/cherifdb" @@ -295,7 +296,7 @@ si non, le mot est enregistré dans une table pour traitement utérieur def check_word_in_fr_dict(mot=None): try: - print("#### analuse du mot "+str(mot)) + print("#### analyse du mot "+str(mot)) col_name = dbname["list_mots_fr"] col_name_not_fr = dbname["list_mots_not_fr"] mydata = {} @@ -306,9 +307,14 @@ def check_word_in_fr_dict(mot=None): mydata['mot'] = mot mydata['treated'] = int("0") mydata['update_date'] = datetime.now() - ret_val_tmp = col_name_not_fr.insert_one(mydata) - if (ret_val_tmp is False): + + ret_val = col_name_not_fr.find_one_and_update( + {'mot': str(mot) }, {"$set": mydata},upsert=True, + return_document=ReturnDocument.AFTER + ) + + if ( ret_val['_id'] is False): print(" Impossible d'enregistrer le '" + mot + "'") return False