parent
267febf45b
commit
1cec3cfb81
14
Ela_Spacy.py
14
Ela_Spacy.py
|
@ -173,7 +173,19 @@ d’un mot comme un seul et même mot. Par exemple, une fois que l’on applique
|
||||||
le mot résultant est le même.
|
le mot résultant est le même.
|
||||||
'''
|
'''
|
||||||
def Ela_stemmize(tab_tokens):
|
def Ela_stemmize(tab_tokens):
|
||||||
return [stemmer.stem(token.text) for token in tab_tokens]
|
|
||||||
|
tab_ret_val = []
|
||||||
|
print(" VERIFICATION SI LE MOT EST FR : " + str(tab_tokens))
|
||||||
|
for mot in tab_tokens:
|
||||||
|
if( mycommon.check_word_in_fr_dict(str(mot)) ):
|
||||||
|
tab_ret_val.append(stemmer.stem(mot.text))
|
||||||
|
else:
|
||||||
|
tab_ret_val.append(mot.text)
|
||||||
|
|
||||||
|
print(" STMISATION TAB = "+str(tab_ret_val))
|
||||||
|
|
||||||
|
return tab_ret_val
|
||||||
|
#return [stemmer.stem(token.text) for token in tab_tokens]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,42 +1,13 @@
|
||||||
,index,mots,occurence,moyenne,id_formation,source_field
|
,index,mots,occurence,moyenne,id_formation,source_field
|
||||||
0,0,"
|
0,0,sommeil,1,0.07,8220,default
|
||||||
|
1,1,stress,1,0.07,8220,default
|
||||||
",1,0.02,mysy_ytubes_04,default
|
2,2,coaching,1,0.07,8220,default
|
||||||
1,1,analys,1,0.02,mysy_ytubes_04,default
|
3,3,equilibre,1,0.07,8220,default
|
||||||
2,2,bt,1,0.02,mysy_ytubes_04,default
|
4,4,relax,1,0.07,8220,default
|
||||||
3,3,licenc,1,0.02,mysy_ytubes_04,default
|
5,5,typ,1,0.07,8220,default
|
||||||
4,4,charg,1,0.02,mysy_ytubes_04,default
|
6,6,trouv,1,0.07,8220,default
|
||||||
5,5,notion,1,0.02,mysy_ytubes_04,default
|
7,7,accompagn,2,0.14,8220,default
|
||||||
6,6,cout,3,0.05,mysy_ytubes_04,default
|
8,8,sante,2,0.14,8220,default
|
||||||
7,7,analyt,1,0.02,mysy_ytubes_04,default
|
9,9,travail,1,0.07,8220,default
|
||||||
8,8,ge,3,0.05,mysy_ytubes_04,default
|
10,10,nutrit,1,0.07,8220,default
|
||||||
9,9,gratuit,1,0.02,mysy_ytubes_04,default
|
11,11,respir,1,0.07,8220,default
|
||||||
10,10,depart,1,0.02,mysy_ytubes_04,default
|
|
||||||
11,11,iut,1,0.02,mysy_ytubes_04,default
|
|
||||||
12,12,comptabilit,6,0.1,mysy_ytubes_04,default
|
|
||||||
13,13,different,1,0.02,mysy_ytubes_04,default
|
|
||||||
14,14,niveau,1,0.02,mysy_ytubes_04,default
|
|
||||||
15,15,prix,1,0.02,mysy_ytubes_04,default
|
|
||||||
16,16,professeur,1,0.02,mysy_ytubes_04,default
|
|
||||||
17,17,sorbon,1,0.02,mysy_ytubes_04,default
|
|
||||||
18,18,introduct,1,0.02,mysy_ytubes_04,default
|
|
||||||
19,19,nord,1,0.02,mysy_ytubes_04,default
|
|
||||||
20,20,cour,3,0.05,mysy_ytubes_04,default
|
|
||||||
21,21,incorpore,1,0.02,mysy_ytubes_04,default
|
|
||||||
22,22,lign,1,0.02,mysy_ytubes_04,default
|
|
||||||
23,23,universit,1,0.02,mysy_ytubes_04,default
|
|
||||||
24,24,stmg,1,0.02,mysy_ytubes_04,default
|
|
||||||
25,25,villetan,1,0.02,mysy_ytubes_04,default
|
|
||||||
26,26,dcg,1,0.02,mysy_ytubes_04,default
|
|
||||||
27,27,marg,1,0.02,mysy_ytubes_04,default
|
|
||||||
28,28,general,1,0.02,mysy_ytubes_04,default
|
|
||||||
29,29,paris,1,0.02,mysy_ytubes_04,default
|
|
||||||
30,30,corinn,1,0.02,mysy_ytubes_04,default
|
|
||||||
31,31,debut,1,0.02,mysy_ytubes_04,default
|
|
||||||
32,32,calcul,1,0.02,mysy_ytubes_04,default
|
|
||||||
33,33,1,2,0.03,mysy_ytubes_04,default
|
|
||||||
34,34,gestion,4,0.07,mysy_ytubes_04,default
|
|
||||||
35,35,"
|
|
||||||
",8,0.13,mysy_ytubes_04,default
|
|
||||||
36,36,zambotto,1,0.02,mysy_ytubes_04,default
|
|
||||||
37,37,resultat,1,0.02,mysy_ytubes_04,default
|
|
||||||
|
|
|
|
@ -407,9 +407,6 @@ def ela_index_all_classes_obj():
|
||||||
return False, "Impossible d'indexer la BDD des formations ==> KO"
|
return False, "Impossible d'indexer la BDD des formations ==> KO"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
'''
|
'''
|
||||||
Cette fonction recherche une chaine de caractère
|
Cette fonction recherche une chaine de caractère
|
||||||
dans la base de données.
|
dans la base de données.
|
||||||
|
@ -435,6 +432,9 @@ def ela_recherche_tokens(sentence):
|
||||||
mycommon.check_word_in_fr_dict(str(mot))
|
mycommon.check_word_in_fr_dict(str(mot))
|
||||||
|
|
||||||
tab_tokens4 = ls.Ela_stemmize(tab_tokens3)
|
tab_tokens4 = ls.Ela_stemmize(tab_tokens3)
|
||||||
|
|
||||||
|
print(" VERIF APRES STEMISATION : " + str(tab_tokens4))
|
||||||
|
|
||||||
tab_tokens4.sort()
|
tab_tokens4.sort()
|
||||||
|
|
||||||
'''
|
'''
|
||||||
|
|
|
@ -1,39 +1,13 @@
|
||||||
mots occurence moyenne id_formation source_field
|
mots occurence moyenne id_formation source_field
|
||||||
0 \n\n 1 0.02 mysy_ytubes_04 default
|
0 sommeil 1 0.07 8220 default
|
||||||
1 analys 1 0.02 mysy_ytubes_04 default
|
1 stress 1 0.07 8220 default
|
||||||
2 bt 1 0.02 mysy_ytubes_04 default
|
2 coaching 1 0.07 8220 default
|
||||||
3 licenc 1 0.02 mysy_ytubes_04 default
|
3 equilibre 1 0.07 8220 default
|
||||||
4 charg 1 0.02 mysy_ytubes_04 default
|
4 relax 1 0.07 8220 default
|
||||||
5 notion 1 0.02 mysy_ytubes_04 default
|
5 typ 1 0.07 8220 default
|
||||||
6 cout 3 0.05 mysy_ytubes_04 default
|
6 trouv 1 0.07 8220 default
|
||||||
7 analyt 1 0.02 mysy_ytubes_04 default
|
7 accompagn 2 0.14 8220 default
|
||||||
8 ge 3 0.05 mysy_ytubes_04 default
|
8 sante 2 0.14 8220 default
|
||||||
9 gratuit 1 0.02 mysy_ytubes_04 default
|
9 travail 1 0.07 8220 default
|
||||||
10 depart 1 0.02 mysy_ytubes_04 default
|
10 nutrit 1 0.07 8220 default
|
||||||
11 iut 1 0.02 mysy_ytubes_04 default
|
11 respir 1 0.07 8220 default
|
||||||
12 comptabilit 6 0.1 mysy_ytubes_04 default
|
|
||||||
13 different 1 0.02 mysy_ytubes_04 default
|
|
||||||
14 niveau 1 0.02 mysy_ytubes_04 default
|
|
||||||
15 prix 1 0.02 mysy_ytubes_04 default
|
|
||||||
16 professeur 1 0.02 mysy_ytubes_04 default
|
|
||||||
17 sorbon 1 0.02 mysy_ytubes_04 default
|
|
||||||
18 introduct 1 0.02 mysy_ytubes_04 default
|
|
||||||
19 nord 1 0.02 mysy_ytubes_04 default
|
|
||||||
20 cour 3 0.05 mysy_ytubes_04 default
|
|
||||||
21 incorpore 1 0.02 mysy_ytubes_04 default
|
|
||||||
22 lign 1 0.02 mysy_ytubes_04 default
|
|
||||||
23 universit 1 0.02 mysy_ytubes_04 default
|
|
||||||
24 stmg 1 0.02 mysy_ytubes_04 default
|
|
||||||
25 villetan 1 0.02 mysy_ytubes_04 default
|
|
||||||
26 dcg 1 0.02 mysy_ytubes_04 default
|
|
||||||
27 marg 1 0.02 mysy_ytubes_04 default
|
|
||||||
28 general 1 0.02 mysy_ytubes_04 default
|
|
||||||
29 paris 1 0.02 mysy_ytubes_04 default
|
|
||||||
30 corinn 1 0.02 mysy_ytubes_04 default
|
|
||||||
31 debut 1 0.02 mysy_ytubes_04 default
|
|
||||||
32 calcul 1 0.02 mysy_ytubes_04 default
|
|
||||||
33 1 2 0.03 mysy_ytubes_04 default
|
|
||||||
34 gestion 4 0.07 mysy_ytubes_04 default
|
|
||||||
35 \n 8 0.13 mysy_ytubes_04 default
|
|
||||||
36 zambotto 1 0.02 mysy_ytubes_04 default
|
|
||||||
37 resultat 1 0.02 mysy_ytubes_04 default
|
|
|
@ -13,6 +13,7 @@ import csv
|
||||||
import inspect
|
import inspect
|
||||||
import sys
|
import sys
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from pymongo import ReturnDocument
|
||||||
|
|
||||||
TOKEN_SIZE = 25
|
TOKEN_SIZE = 25
|
||||||
CONNECTION_STRING = "mongodb://localhost/cherifdb"
|
CONNECTION_STRING = "mongodb://localhost/cherifdb"
|
||||||
|
@ -295,7 +296,7 @@ si non, le mot est enregistré dans une table pour traitement utérieur
|
||||||
|
|
||||||
def check_word_in_fr_dict(mot=None):
|
def check_word_in_fr_dict(mot=None):
|
||||||
try:
|
try:
|
||||||
print("#### analuse du mot "+str(mot))
|
print("#### analyse du mot "+str(mot))
|
||||||
col_name = dbname["list_mots_fr"]
|
col_name = dbname["list_mots_fr"]
|
||||||
col_name_not_fr = dbname["list_mots_not_fr"]
|
col_name_not_fr = dbname["list_mots_not_fr"]
|
||||||
mydata = {}
|
mydata = {}
|
||||||
|
@ -306,9 +307,14 @@ def check_word_in_fr_dict(mot=None):
|
||||||
mydata['mot'] = mot
|
mydata['mot'] = mot
|
||||||
mydata['treated'] = int("0")
|
mydata['treated'] = int("0")
|
||||||
mydata['update_date'] = datetime.now()
|
mydata['update_date'] = datetime.now()
|
||||||
ret_val_tmp = col_name_not_fr.insert_one(mydata)
|
|
||||||
|
|
||||||
if (ret_val_tmp is False):
|
|
||||||
|
ret_val = col_name_not_fr.find_one_and_update(
|
||||||
|
{'mot': str(mot) }, {"$set": mydata},upsert=True,
|
||||||
|
return_document=ReturnDocument.AFTER
|
||||||
|
)
|
||||||
|
|
||||||
|
if ( ret_val['_id'] is False):
|
||||||
print(" Impossible d'enregistrer le '" + mot + "'")
|
print(" Impossible d'enregistrer le '" + mot + "'")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue