Merge remote-tracking branch 'origin/master'

# Conflicts:
#	Log/log_file.log
master
ChérifBALDE 2022-03-28 20:34:49 +02:00 committed by cherif
parent 267febf45b
commit 1cec3cfb81
5 changed files with 50 additions and 87 deletions

View File

@ -173,7 +173,19 @@ dun mot comme un seul et même mot. Par exemple, une fois que lon applique
le mot résultant est le même.
'''
def Ela_stemmize(tab_tokens):
return [stemmer.stem(token.text) for token in tab_tokens]
tab_ret_val = []
print(" VERIFICATION SI LE MOT EST FR : " + str(tab_tokens))
for mot in tab_tokens:
if( mycommon.check_word_in_fr_dict(str(mot)) ):
tab_ret_val.append(stemmer.stem(mot.text))
else:
tab_ret_val.append(mot.text)
print(" STMISATION TAB = "+str(tab_ret_val))
return tab_ret_val
#return [stemmer.stem(token.text) for token in tab_tokens]

View File

@ -1,42 +1,13 @@
,index,mots,occurence,moyenne,id_formation,source_field
0,0,"
",1,0.02,mysy_ytubes_04,default
1,1,analys,1,0.02,mysy_ytubes_04,default
2,2,bt,1,0.02,mysy_ytubes_04,default
3,3,licenc,1,0.02,mysy_ytubes_04,default
4,4,charg,1,0.02,mysy_ytubes_04,default
5,5,notion,1,0.02,mysy_ytubes_04,default
6,6,cout,3,0.05,mysy_ytubes_04,default
7,7,analyt,1,0.02,mysy_ytubes_04,default
8,8,ge,3,0.05,mysy_ytubes_04,default
9,9,gratuit,1,0.02,mysy_ytubes_04,default
10,10,depart,1,0.02,mysy_ytubes_04,default
11,11,iut,1,0.02,mysy_ytubes_04,default
12,12,comptabilit,6,0.1,mysy_ytubes_04,default
13,13,different,1,0.02,mysy_ytubes_04,default
14,14,niveau,1,0.02,mysy_ytubes_04,default
15,15,prix,1,0.02,mysy_ytubes_04,default
16,16,professeur,1,0.02,mysy_ytubes_04,default
17,17,sorbon,1,0.02,mysy_ytubes_04,default
18,18,introduct,1,0.02,mysy_ytubes_04,default
19,19,nord,1,0.02,mysy_ytubes_04,default
20,20,cour,3,0.05,mysy_ytubes_04,default
21,21,incorpore,1,0.02,mysy_ytubes_04,default
22,22,lign,1,0.02,mysy_ytubes_04,default
23,23,universit,1,0.02,mysy_ytubes_04,default
24,24,stmg,1,0.02,mysy_ytubes_04,default
25,25,villetan,1,0.02,mysy_ytubes_04,default
26,26,dcg,1,0.02,mysy_ytubes_04,default
27,27,marg,1,0.02,mysy_ytubes_04,default
28,28,general,1,0.02,mysy_ytubes_04,default
29,29,paris,1,0.02,mysy_ytubes_04,default
30,30,corinn,1,0.02,mysy_ytubes_04,default
31,31,debut,1,0.02,mysy_ytubes_04,default
32,32,calcul,1,0.02,mysy_ytubes_04,default
33,33,1,2,0.03,mysy_ytubes_04,default
34,34,gestion,4,0.07,mysy_ytubes_04,default
35,35,"
",8,0.13,mysy_ytubes_04,default
36,36,zambotto,1,0.02,mysy_ytubes_04,default
37,37,resultat,1,0.02,mysy_ytubes_04,default
0,0,sommeil,1,0.07,8220,default
1,1,stress,1,0.07,8220,default
2,2,coaching,1,0.07,8220,default
3,3,equilibre,1,0.07,8220,default
4,4,relax,1,0.07,8220,default
5,5,typ,1,0.07,8220,default
6,6,trouv,1,0.07,8220,default
7,7,accompagn,2,0.14,8220,default
8,8,sante,2,0.14,8220,default
9,9,travail,1,0.07,8220,default
10,10,nutrit,1,0.07,8220,default
11,11,respir,1,0.07,8220,default

1 index mots occurence moyenne id_formation source_field
2 0 0 sommeil 1 0.02 0.07 mysy_ytubes_04 8220 default
3 1 1 analys stress 1 0.02 0.07 mysy_ytubes_04 8220 default
4 2 2 bt coaching 1 0.02 0.07 mysy_ytubes_04 8220 default
5 3 3 licenc equilibre 1 0.02 0.07 mysy_ytubes_04 8220 default
6 4 4 charg relax 1 0.02 0.07 mysy_ytubes_04 8220 default
7 5 5 notion typ 1 0.02 0.07 mysy_ytubes_04 8220 default
8 6 6 cout trouv 3 1 0.05 0.07 mysy_ytubes_04 8220 default
9 7 7 analyt accompagn 1 2 0.02 0.14 mysy_ytubes_04 8220 default
10 8 8 ge sante 3 2 0.05 0.14 mysy_ytubes_04 8220 default
11 9 9 gratuit travail 1 0.02 0.07 mysy_ytubes_04 8220 default
12 10 10 depart nutrit 1 0.02 0.07 mysy_ytubes_04 8220 default
13 11 11 iut respir 1 0.02 0.07 mysy_ytubes_04 8220 default
12 12 comptabilit 6 0.1 mysy_ytubes_04 default
13 13 different 1 0.02 mysy_ytubes_04 default
14 14 niveau 1 0.02 mysy_ytubes_04 default
15 15 prix 1 0.02 mysy_ytubes_04 default
16 16 professeur 1 0.02 mysy_ytubes_04 default
17 17 sorbon 1 0.02 mysy_ytubes_04 default
18 18 introduct 1 0.02 mysy_ytubes_04 default
19 19 nord 1 0.02 mysy_ytubes_04 default
20 20 cour 3 0.05 mysy_ytubes_04 default
21 21 incorpore 1 0.02 mysy_ytubes_04 default
22 22 lign 1 0.02 mysy_ytubes_04 default
23 23 universit 1 0.02 mysy_ytubes_04 default
24 24 stmg 1 0.02 mysy_ytubes_04 default
25 25 villetan 1 0.02 mysy_ytubes_04 default
26 26 dcg 1 0.02 mysy_ytubes_04 default
27 27 marg 1 0.02 mysy_ytubes_04 default
28 28 general 1 0.02 mysy_ytubes_04 default
29 29 paris 1 0.02 mysy_ytubes_04 default
30 30 corinn 1 0.02 mysy_ytubes_04 default
31 31 debut 1 0.02 mysy_ytubes_04 default
32 32 calcul 1 0.02 mysy_ytubes_04 default
33 33 1 2 0.03 mysy_ytubes_04 default
34 34 gestion 4 0.07 mysy_ytubes_04 default
35 35 8 0.13 mysy_ytubes_04 default
36 36 zambotto 1 0.02 mysy_ytubes_04 default
37 37 resultat 1 0.02 mysy_ytubes_04 default

View File

@ -407,9 +407,6 @@ def ela_index_all_classes_obj():
return False, "Impossible d'indexer la BDD des formations ==> KO"
'''
Cette fonction recherche une chaine de caractère
dans la base de données.
@ -435,6 +432,9 @@ def ela_recherche_tokens(sentence):
mycommon.check_word_in_fr_dict(str(mot))
tab_tokens4 = ls.Ela_stemmize(tab_tokens3)
print(" VERIF APRES STEMISATION : " + str(tab_tokens4))
tab_tokens4.sort()
'''

View File

@ -1,39 +1,13 @@
mots occurence moyenne id_formation source_field
0 \n\n 1 0.02 mysy_ytubes_04 default
1 analys 1 0.02 mysy_ytubes_04 default
2 bt 1 0.02 mysy_ytubes_04 default
3 licenc 1 0.02 mysy_ytubes_04 default
4 charg 1 0.02 mysy_ytubes_04 default
5 notion 1 0.02 mysy_ytubes_04 default
6 cout 3 0.05 mysy_ytubes_04 default
7 analyt 1 0.02 mysy_ytubes_04 default
8 ge 3 0.05 mysy_ytubes_04 default
9 gratuit 1 0.02 mysy_ytubes_04 default
10 depart 1 0.02 mysy_ytubes_04 default
11 iut 1 0.02 mysy_ytubes_04 default
12 comptabilit 6 0.1 mysy_ytubes_04 default
13 different 1 0.02 mysy_ytubes_04 default
14 niveau 1 0.02 mysy_ytubes_04 default
15 prix 1 0.02 mysy_ytubes_04 default
16 professeur 1 0.02 mysy_ytubes_04 default
17 sorbon 1 0.02 mysy_ytubes_04 default
18 introduct 1 0.02 mysy_ytubes_04 default
19 nord 1 0.02 mysy_ytubes_04 default
20 cour 3 0.05 mysy_ytubes_04 default
21 incorpore 1 0.02 mysy_ytubes_04 default
22 lign 1 0.02 mysy_ytubes_04 default
23 universit 1 0.02 mysy_ytubes_04 default
24 stmg 1 0.02 mysy_ytubes_04 default
25 villetan 1 0.02 mysy_ytubes_04 default
26 dcg 1 0.02 mysy_ytubes_04 default
27 marg 1 0.02 mysy_ytubes_04 default
28 general 1 0.02 mysy_ytubes_04 default
29 paris 1 0.02 mysy_ytubes_04 default
30 corinn 1 0.02 mysy_ytubes_04 default
31 debut 1 0.02 mysy_ytubes_04 default
32 calcul 1 0.02 mysy_ytubes_04 default
33 1 2 0.03 mysy_ytubes_04 default
34 gestion 4 0.07 mysy_ytubes_04 default
35 \n 8 0.13 mysy_ytubes_04 default
36 zambotto 1 0.02 mysy_ytubes_04 default
37 resultat 1 0.02 mysy_ytubes_04 default
mots occurence moyenne id_formation source_field
0 sommeil 1 0.07 8220 default
1 stress 1 0.07 8220 default
2 coaching 1 0.07 8220 default
3 equilibre 1 0.07 8220 default
4 relax 1 0.07 8220 default
5 typ 1 0.07 8220 default
6 trouv 1 0.07 8220 default
7 accompagn 2 0.14 8220 default
8 sante 2 0.14 8220 default
9 travail 1 0.07 8220 default
10 nutrit 1 0.07 8220 default
11 respir 1 0.07 8220 default

View File

@ -13,6 +13,7 @@ import csv
import inspect
import sys
from datetime import datetime
from pymongo import ReturnDocument
TOKEN_SIZE = 25
CONNECTION_STRING = "mongodb://localhost/cherifdb"
@ -295,7 +296,7 @@ si non, le mot est enregistré dans une table pour traitement utérieur
def check_word_in_fr_dict(mot=None):
try:
print("#### analuse du mot "+str(mot))
print("#### analyse du mot "+str(mot))
col_name = dbname["list_mots_fr"]
col_name_not_fr = dbname["list_mots_not_fr"]
mydata = {}
@ -306,9 +307,14 @@ def check_word_in_fr_dict(mot=None):
mydata['mot'] = mot
mydata['treated'] = int("0")
mydata['update_date'] = datetime.now()
ret_val_tmp = col_name_not_fr.insert_one(mydata)
if (ret_val_tmp is False):
ret_val = col_name_not_fr.find_one_and_update(
{'mot': str(mot) }, {"$set": mydata},upsert=True,
return_document=ReturnDocument.AFTER
)
if ( ret_val['_id'] is False):
print(" Impossible d'enregistrer le '" + mot + "'")
return False