Merge remote-tracking branch 'origin/master'

# Conflicts:
#	Log/log_file.log
master
ChérifBALDE 2022-03-28 20:34:49 +02:00 committed by cherif
parent 267febf45b
commit 1cec3cfb81
5 changed files with 50 additions and 87 deletions

View File

@ -173,7 +173,19 @@ dun mot comme un seul et même mot. Par exemple, une fois que lon applique
le mot résultant est le même. le mot résultant est le même.
''' '''
def Ela_stemmize(tab_tokens): def Ela_stemmize(tab_tokens):
return [stemmer.stem(token.text) for token in tab_tokens]
tab_ret_val = []
print(" VERIFICATION SI LE MOT EST FR : " + str(tab_tokens))
for mot in tab_tokens:
if( mycommon.check_word_in_fr_dict(str(mot)) ):
tab_ret_val.append(stemmer.stem(mot.text))
else:
tab_ret_val.append(mot.text)
print(" STMISATION TAB = "+str(tab_ret_val))
return tab_ret_val
#return [stemmer.stem(token.text) for token in tab_tokens]

View File

@ -1,42 +1,13 @@
,index,mots,occurence,moyenne,id_formation,source_field ,index,mots,occurence,moyenne,id_formation,source_field
0,0," 0,0,sommeil,1,0.07,8220,default
1,1,stress,1,0.07,8220,default
",1,0.02,mysy_ytubes_04,default 2,2,coaching,1,0.07,8220,default
1,1,analys,1,0.02,mysy_ytubes_04,default 3,3,equilibre,1,0.07,8220,default
2,2,bt,1,0.02,mysy_ytubes_04,default 4,4,relax,1,0.07,8220,default
3,3,licenc,1,0.02,mysy_ytubes_04,default 5,5,typ,1,0.07,8220,default
4,4,charg,1,0.02,mysy_ytubes_04,default 6,6,trouv,1,0.07,8220,default
5,5,notion,1,0.02,mysy_ytubes_04,default 7,7,accompagn,2,0.14,8220,default
6,6,cout,3,0.05,mysy_ytubes_04,default 8,8,sante,2,0.14,8220,default
7,7,analyt,1,0.02,mysy_ytubes_04,default 9,9,travail,1,0.07,8220,default
8,8,ge,3,0.05,mysy_ytubes_04,default 10,10,nutrit,1,0.07,8220,default
9,9,gratuit,1,0.02,mysy_ytubes_04,default 11,11,respir,1,0.07,8220,default
10,10,depart,1,0.02,mysy_ytubes_04,default
11,11,iut,1,0.02,mysy_ytubes_04,default
12,12,comptabilit,6,0.1,mysy_ytubes_04,default
13,13,different,1,0.02,mysy_ytubes_04,default
14,14,niveau,1,0.02,mysy_ytubes_04,default
15,15,prix,1,0.02,mysy_ytubes_04,default
16,16,professeur,1,0.02,mysy_ytubes_04,default
17,17,sorbon,1,0.02,mysy_ytubes_04,default
18,18,introduct,1,0.02,mysy_ytubes_04,default
19,19,nord,1,0.02,mysy_ytubes_04,default
20,20,cour,3,0.05,mysy_ytubes_04,default
21,21,incorpore,1,0.02,mysy_ytubes_04,default
22,22,lign,1,0.02,mysy_ytubes_04,default
23,23,universit,1,0.02,mysy_ytubes_04,default
24,24,stmg,1,0.02,mysy_ytubes_04,default
25,25,villetan,1,0.02,mysy_ytubes_04,default
26,26,dcg,1,0.02,mysy_ytubes_04,default
27,27,marg,1,0.02,mysy_ytubes_04,default
28,28,general,1,0.02,mysy_ytubes_04,default
29,29,paris,1,0.02,mysy_ytubes_04,default
30,30,corinn,1,0.02,mysy_ytubes_04,default
31,31,debut,1,0.02,mysy_ytubes_04,default
32,32,calcul,1,0.02,mysy_ytubes_04,default
33,33,1,2,0.03,mysy_ytubes_04,default
34,34,gestion,4,0.07,mysy_ytubes_04,default
35,35,"
",8,0.13,mysy_ytubes_04,default
36,36,zambotto,1,0.02,mysy_ytubes_04,default
37,37,resultat,1,0.02,mysy_ytubes_04,default

1 index mots occurence moyenne id_formation source_field
2 0 0 sommeil 1 0.02 0.07 mysy_ytubes_04 8220 default
3 1 1 analys stress 1 0.02 0.07 mysy_ytubes_04 8220 default
4 2 2 bt coaching 1 0.02 0.07 mysy_ytubes_04 8220 default
5 3 3 licenc equilibre 1 0.02 0.07 mysy_ytubes_04 8220 default
6 4 4 charg relax 1 0.02 0.07 mysy_ytubes_04 8220 default
7 5 5 notion typ 1 0.02 0.07 mysy_ytubes_04 8220 default
8 6 6 cout trouv 3 1 0.05 0.07 mysy_ytubes_04 8220 default
9 7 7 analyt accompagn 1 2 0.02 0.14 mysy_ytubes_04 8220 default
10 8 8 ge sante 3 2 0.05 0.14 mysy_ytubes_04 8220 default
11 9 9 gratuit travail 1 0.02 0.07 mysy_ytubes_04 8220 default
12 10 10 depart nutrit 1 0.02 0.07 mysy_ytubes_04 8220 default
13 11 11 iut respir 1 0.02 0.07 mysy_ytubes_04 8220 default
12 12 comptabilit 6 0.1 mysy_ytubes_04 default
13 13 different 1 0.02 mysy_ytubes_04 default
14 14 niveau 1 0.02 mysy_ytubes_04 default
15 15 prix 1 0.02 mysy_ytubes_04 default
16 16 professeur 1 0.02 mysy_ytubes_04 default
17 17 sorbon 1 0.02 mysy_ytubes_04 default
18 18 introduct 1 0.02 mysy_ytubes_04 default
19 19 nord 1 0.02 mysy_ytubes_04 default
20 20 cour 3 0.05 mysy_ytubes_04 default
21 21 incorpore 1 0.02 mysy_ytubes_04 default
22 22 lign 1 0.02 mysy_ytubes_04 default
23 23 universit 1 0.02 mysy_ytubes_04 default
24 24 stmg 1 0.02 mysy_ytubes_04 default
25 25 villetan 1 0.02 mysy_ytubes_04 default
26 26 dcg 1 0.02 mysy_ytubes_04 default
27 27 marg 1 0.02 mysy_ytubes_04 default
28 28 general 1 0.02 mysy_ytubes_04 default
29 29 paris 1 0.02 mysy_ytubes_04 default
30 30 corinn 1 0.02 mysy_ytubes_04 default
31 31 debut 1 0.02 mysy_ytubes_04 default
32 32 calcul 1 0.02 mysy_ytubes_04 default
33 33 1 2 0.03 mysy_ytubes_04 default
34 34 gestion 4 0.07 mysy_ytubes_04 default
35 35 8 0.13 mysy_ytubes_04 default
36 36 zambotto 1 0.02 mysy_ytubes_04 default
37 37 resultat 1 0.02 mysy_ytubes_04 default

View File

@ -407,9 +407,6 @@ def ela_index_all_classes_obj():
return False, "Impossible d'indexer la BDD des formations ==> KO" return False, "Impossible d'indexer la BDD des formations ==> KO"
''' '''
Cette fonction recherche une chaine de caractère Cette fonction recherche une chaine de caractère
dans la base de données. dans la base de données.
@ -435,6 +432,9 @@ def ela_recherche_tokens(sentence):
mycommon.check_word_in_fr_dict(str(mot)) mycommon.check_word_in_fr_dict(str(mot))
tab_tokens4 = ls.Ela_stemmize(tab_tokens3) tab_tokens4 = ls.Ela_stemmize(tab_tokens3)
print(" VERIF APRES STEMISATION : " + str(tab_tokens4))
tab_tokens4.sort() tab_tokens4.sort()
''' '''

View File

@ -1,39 +1,13 @@
mots occurence moyenne id_formation source_field mots occurence moyenne id_formation source_field
0 \n\n 1 0.02 mysy_ytubes_04 default 0 sommeil 1 0.07 8220 default
1 analys 1 0.02 mysy_ytubes_04 default 1 stress 1 0.07 8220 default
2 bt 1 0.02 mysy_ytubes_04 default 2 coaching 1 0.07 8220 default
3 licenc 1 0.02 mysy_ytubes_04 default 3 equilibre 1 0.07 8220 default
4 charg 1 0.02 mysy_ytubes_04 default 4 relax 1 0.07 8220 default
5 notion 1 0.02 mysy_ytubes_04 default 5 typ 1 0.07 8220 default
6 cout 3 0.05 mysy_ytubes_04 default 6 trouv 1 0.07 8220 default
7 analyt 1 0.02 mysy_ytubes_04 default 7 accompagn 2 0.14 8220 default
8 ge 3 0.05 mysy_ytubes_04 default 8 sante 2 0.14 8220 default
9 gratuit 1 0.02 mysy_ytubes_04 default 9 travail 1 0.07 8220 default
10 depart 1 0.02 mysy_ytubes_04 default 10 nutrit 1 0.07 8220 default
11 iut 1 0.02 mysy_ytubes_04 default 11 respir 1 0.07 8220 default
12 comptabilit 6 0.1 mysy_ytubes_04 default
13 different 1 0.02 mysy_ytubes_04 default
14 niveau 1 0.02 mysy_ytubes_04 default
15 prix 1 0.02 mysy_ytubes_04 default
16 professeur 1 0.02 mysy_ytubes_04 default
17 sorbon 1 0.02 mysy_ytubes_04 default
18 introduct 1 0.02 mysy_ytubes_04 default
19 nord 1 0.02 mysy_ytubes_04 default
20 cour 3 0.05 mysy_ytubes_04 default
21 incorpore 1 0.02 mysy_ytubes_04 default
22 lign 1 0.02 mysy_ytubes_04 default
23 universit 1 0.02 mysy_ytubes_04 default
24 stmg 1 0.02 mysy_ytubes_04 default
25 villetan 1 0.02 mysy_ytubes_04 default
26 dcg 1 0.02 mysy_ytubes_04 default
27 marg 1 0.02 mysy_ytubes_04 default
28 general 1 0.02 mysy_ytubes_04 default
29 paris 1 0.02 mysy_ytubes_04 default
30 corinn 1 0.02 mysy_ytubes_04 default
31 debut 1 0.02 mysy_ytubes_04 default
32 calcul 1 0.02 mysy_ytubes_04 default
33 1 2 0.03 mysy_ytubes_04 default
34 gestion 4 0.07 mysy_ytubes_04 default
35 \n 8 0.13 mysy_ytubes_04 default
36 zambotto 1 0.02 mysy_ytubes_04 default
37 resultat 1 0.02 mysy_ytubes_04 default

View File

@ -13,6 +13,7 @@ import csv
import inspect import inspect
import sys import sys
from datetime import datetime from datetime import datetime
from pymongo import ReturnDocument
TOKEN_SIZE = 25 TOKEN_SIZE = 25
CONNECTION_STRING = "mongodb://localhost/cherifdb" CONNECTION_STRING = "mongodb://localhost/cherifdb"
@ -295,7 +296,7 @@ si non, le mot est enregistré dans une table pour traitement utérieur
def check_word_in_fr_dict(mot=None): def check_word_in_fr_dict(mot=None):
try: try:
print("#### analuse du mot "+str(mot)) print("#### analyse du mot "+str(mot))
col_name = dbname["list_mots_fr"] col_name = dbname["list_mots_fr"]
col_name_not_fr = dbname["list_mots_not_fr"] col_name_not_fr = dbname["list_mots_not_fr"]
mydata = {} mydata = {}
@ -306,9 +307,14 @@ def check_word_in_fr_dict(mot=None):
mydata['mot'] = mot mydata['mot'] = mot
mydata['treated'] = int("0") mydata['treated'] = int("0")
mydata['update_date'] = datetime.now() mydata['update_date'] = datetime.now()
ret_val_tmp = col_name_not_fr.insert_one(mydata)
if (ret_val_tmp is False):
ret_val = col_name_not_fr.find_one_and_update(
{'mot': str(mot) }, {"$set": mydata},upsert=True,
return_document=ReturnDocument.AFTER
)
if ( ret_val['_id'] is False):
print(" Impossible d'enregistrer le '" + mot + "'") print(" Impossible d'enregistrer le '" + mot + "'")
return False return False