parent
267febf45b
commit
1cec3cfb81
14
Ela_Spacy.py
14
Ela_Spacy.py
|
@ -173,7 +173,19 @@ d’un mot comme un seul et même mot. Par exemple, une fois que l’on applique
|
|||
le mot résultant est le même.
|
||||
'''
|
||||
def Ela_stemmize(tab_tokens):
|
||||
return [stemmer.stem(token.text) for token in tab_tokens]
|
||||
|
||||
tab_ret_val = []
|
||||
print(" VERIFICATION SI LE MOT EST FR : " + str(tab_tokens))
|
||||
for mot in tab_tokens:
|
||||
if( mycommon.check_word_in_fr_dict(str(mot)) ):
|
||||
tab_ret_val.append(stemmer.stem(mot.text))
|
||||
else:
|
||||
tab_ret_val.append(mot.text)
|
||||
|
||||
print(" STMISATION TAB = "+str(tab_ret_val))
|
||||
|
||||
return tab_ret_val
|
||||
#return [stemmer.stem(token.text) for token in tab_tokens]
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -1,42 +1,13 @@
|
|||
,index,mots,occurence,moyenne,id_formation,source_field
|
||||
0,0,"
|
||||
|
||||
",1,0.02,mysy_ytubes_04,default
|
||||
1,1,analys,1,0.02,mysy_ytubes_04,default
|
||||
2,2,bt,1,0.02,mysy_ytubes_04,default
|
||||
3,3,licenc,1,0.02,mysy_ytubes_04,default
|
||||
4,4,charg,1,0.02,mysy_ytubes_04,default
|
||||
5,5,notion,1,0.02,mysy_ytubes_04,default
|
||||
6,6,cout,3,0.05,mysy_ytubes_04,default
|
||||
7,7,analyt,1,0.02,mysy_ytubes_04,default
|
||||
8,8,ge,3,0.05,mysy_ytubes_04,default
|
||||
9,9,gratuit,1,0.02,mysy_ytubes_04,default
|
||||
10,10,depart,1,0.02,mysy_ytubes_04,default
|
||||
11,11,iut,1,0.02,mysy_ytubes_04,default
|
||||
12,12,comptabilit,6,0.1,mysy_ytubes_04,default
|
||||
13,13,different,1,0.02,mysy_ytubes_04,default
|
||||
14,14,niveau,1,0.02,mysy_ytubes_04,default
|
||||
15,15,prix,1,0.02,mysy_ytubes_04,default
|
||||
16,16,professeur,1,0.02,mysy_ytubes_04,default
|
||||
17,17,sorbon,1,0.02,mysy_ytubes_04,default
|
||||
18,18,introduct,1,0.02,mysy_ytubes_04,default
|
||||
19,19,nord,1,0.02,mysy_ytubes_04,default
|
||||
20,20,cour,3,0.05,mysy_ytubes_04,default
|
||||
21,21,incorpore,1,0.02,mysy_ytubes_04,default
|
||||
22,22,lign,1,0.02,mysy_ytubes_04,default
|
||||
23,23,universit,1,0.02,mysy_ytubes_04,default
|
||||
24,24,stmg,1,0.02,mysy_ytubes_04,default
|
||||
25,25,villetan,1,0.02,mysy_ytubes_04,default
|
||||
26,26,dcg,1,0.02,mysy_ytubes_04,default
|
||||
27,27,marg,1,0.02,mysy_ytubes_04,default
|
||||
28,28,general,1,0.02,mysy_ytubes_04,default
|
||||
29,29,paris,1,0.02,mysy_ytubes_04,default
|
||||
30,30,corinn,1,0.02,mysy_ytubes_04,default
|
||||
31,31,debut,1,0.02,mysy_ytubes_04,default
|
||||
32,32,calcul,1,0.02,mysy_ytubes_04,default
|
||||
33,33,1,2,0.03,mysy_ytubes_04,default
|
||||
34,34,gestion,4,0.07,mysy_ytubes_04,default
|
||||
35,35,"
|
||||
",8,0.13,mysy_ytubes_04,default
|
||||
36,36,zambotto,1,0.02,mysy_ytubes_04,default
|
||||
37,37,resultat,1,0.02,mysy_ytubes_04,default
|
||||
0,0,sommeil,1,0.07,8220,default
|
||||
1,1,stress,1,0.07,8220,default
|
||||
2,2,coaching,1,0.07,8220,default
|
||||
3,3,equilibre,1,0.07,8220,default
|
||||
4,4,relax,1,0.07,8220,default
|
||||
5,5,typ,1,0.07,8220,default
|
||||
6,6,trouv,1,0.07,8220,default
|
||||
7,7,accompagn,2,0.14,8220,default
|
||||
8,8,sante,2,0.14,8220,default
|
||||
9,9,travail,1,0.07,8220,default
|
||||
10,10,nutrit,1,0.07,8220,default
|
||||
11,11,respir,1,0.07,8220,default
|
||||
|
|
|
|
@ -407,9 +407,6 @@ def ela_index_all_classes_obj():
|
|||
return False, "Impossible d'indexer la BDD des formations ==> KO"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
'''
|
||||
Cette fonction recherche une chaine de caractère
|
||||
dans la base de données.
|
||||
|
@ -435,6 +432,9 @@ def ela_recherche_tokens(sentence):
|
|||
mycommon.check_word_in_fr_dict(str(mot))
|
||||
|
||||
tab_tokens4 = ls.Ela_stemmize(tab_tokens3)
|
||||
|
||||
print(" VERIF APRES STEMISATION : " + str(tab_tokens4))
|
||||
|
||||
tab_tokens4.sort()
|
||||
|
||||
'''
|
||||
|
|
|
@ -1,39 +1,13 @@
|
|||
mots occurence moyenne id_formation source_field
|
||||
0 \n\n 1 0.02 mysy_ytubes_04 default
|
||||
1 analys 1 0.02 mysy_ytubes_04 default
|
||||
2 bt 1 0.02 mysy_ytubes_04 default
|
||||
3 licenc 1 0.02 mysy_ytubes_04 default
|
||||
4 charg 1 0.02 mysy_ytubes_04 default
|
||||
5 notion 1 0.02 mysy_ytubes_04 default
|
||||
6 cout 3 0.05 mysy_ytubes_04 default
|
||||
7 analyt 1 0.02 mysy_ytubes_04 default
|
||||
8 ge 3 0.05 mysy_ytubes_04 default
|
||||
9 gratuit 1 0.02 mysy_ytubes_04 default
|
||||
10 depart 1 0.02 mysy_ytubes_04 default
|
||||
11 iut 1 0.02 mysy_ytubes_04 default
|
||||
12 comptabilit 6 0.1 mysy_ytubes_04 default
|
||||
13 different 1 0.02 mysy_ytubes_04 default
|
||||
14 niveau 1 0.02 mysy_ytubes_04 default
|
||||
15 prix 1 0.02 mysy_ytubes_04 default
|
||||
16 professeur 1 0.02 mysy_ytubes_04 default
|
||||
17 sorbon 1 0.02 mysy_ytubes_04 default
|
||||
18 introduct 1 0.02 mysy_ytubes_04 default
|
||||
19 nord 1 0.02 mysy_ytubes_04 default
|
||||
20 cour 3 0.05 mysy_ytubes_04 default
|
||||
21 incorpore 1 0.02 mysy_ytubes_04 default
|
||||
22 lign 1 0.02 mysy_ytubes_04 default
|
||||
23 universit 1 0.02 mysy_ytubes_04 default
|
||||
24 stmg 1 0.02 mysy_ytubes_04 default
|
||||
25 villetan 1 0.02 mysy_ytubes_04 default
|
||||
26 dcg 1 0.02 mysy_ytubes_04 default
|
||||
27 marg 1 0.02 mysy_ytubes_04 default
|
||||
28 general 1 0.02 mysy_ytubes_04 default
|
||||
29 paris 1 0.02 mysy_ytubes_04 default
|
||||
30 corinn 1 0.02 mysy_ytubes_04 default
|
||||
31 debut 1 0.02 mysy_ytubes_04 default
|
||||
32 calcul 1 0.02 mysy_ytubes_04 default
|
||||
33 1 2 0.03 mysy_ytubes_04 default
|
||||
34 gestion 4 0.07 mysy_ytubes_04 default
|
||||
35 \n 8 0.13 mysy_ytubes_04 default
|
||||
36 zambotto 1 0.02 mysy_ytubes_04 default
|
||||
37 resultat 1 0.02 mysy_ytubes_04 default
|
||||
mots occurence moyenne id_formation source_field
|
||||
0 sommeil 1 0.07 8220 default
|
||||
1 stress 1 0.07 8220 default
|
||||
2 coaching 1 0.07 8220 default
|
||||
3 equilibre 1 0.07 8220 default
|
||||
4 relax 1 0.07 8220 default
|
||||
5 typ 1 0.07 8220 default
|
||||
6 trouv 1 0.07 8220 default
|
||||
7 accompagn 2 0.14 8220 default
|
||||
8 sante 2 0.14 8220 default
|
||||
9 travail 1 0.07 8220 default
|
||||
10 nutrit 1 0.07 8220 default
|
||||
11 respir 1 0.07 8220 default
|
|
@ -13,6 +13,7 @@ import csv
|
|||
import inspect
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pymongo import ReturnDocument
|
||||
|
||||
TOKEN_SIZE = 25
|
||||
CONNECTION_STRING = "mongodb://localhost/cherifdb"
|
||||
|
@ -295,7 +296,7 @@ si non, le mot est enregistré dans une table pour traitement utérieur
|
|||
|
||||
def check_word_in_fr_dict(mot=None):
|
||||
try:
|
||||
print("#### analuse du mot "+str(mot))
|
||||
print("#### analyse du mot "+str(mot))
|
||||
col_name = dbname["list_mots_fr"]
|
||||
col_name_not_fr = dbname["list_mots_not_fr"]
|
||||
mydata = {}
|
||||
|
@ -306,9 +307,14 @@ def check_word_in_fr_dict(mot=None):
|
|||
mydata['mot'] = mot
|
||||
mydata['treated'] = int("0")
|
||||
mydata['update_date'] = datetime.now()
|
||||
ret_val_tmp = col_name_not_fr.insert_one(mydata)
|
||||
|
||||
if (ret_val_tmp is False):
|
||||
|
||||
ret_val = col_name_not_fr.find_one_and_update(
|
||||
{'mot': str(mot) }, {"$set": mydata},upsert=True,
|
||||
return_document=ReturnDocument.AFTER
|
||||
)
|
||||
|
||||
if ( ret_val['_id'] is False):
|
||||
print(" Impossible d'enregistrer le '" + mot + "'")
|
||||
return False
|
||||
|
||||
|
|
Loading…
Reference in New Issue