29/03/22 - 22h30
parent
1cec3cfb81
commit
f9aaddb441
70
Ela_Spacy.py
70
Ela_Spacy.py
|
@ -8,10 +8,13 @@ import pandas as pd
|
|||
import numpy as np
|
||||
import pymongo
|
||||
from pymongo import MongoClient
|
||||
from unidecode import unidecode
|
||||
from collections import Counter
|
||||
import ela_spacy_common as lsc
|
||||
import prj_common as mycommon
|
||||
from unidecode import unidecode
|
||||
import inspect
|
||||
import sys, os
|
||||
from autocorrect import Speller
|
||||
|
||||
CONNECTION_STRING = "mongodb://localhost/cherifdb"
|
||||
|
||||
|
@ -31,6 +34,8 @@ lsc.update_stopWords(stopWords)
|
|||
#print(type(stopWords))
|
||||
|
||||
lsc.update_token_fr_pontuation(token_fr_pontuation)
|
||||
spell_fr = Speller(lang='fr')
|
||||
|
||||
|
||||
#print("token_fr_pontuation")
|
||||
#print(token_fr_pontuation)
|
||||
|
@ -73,6 +78,20 @@ def Ela_Normalize(sentence):
|
|||
|
||||
return sentence
|
||||
|
||||
'''
|
||||
Cette fonction prend un mot et retourne
|
||||
sa correction orthographique ne français
|
||||
'''
|
||||
def correct_fr_word(word):
|
||||
|
||||
try:
|
||||
print(" Fonction : correct_fr_word : '"+word+"' =======> "+spell_fr(word))
|
||||
return spell_fr(word)
|
||||
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
|
||||
return False
|
||||
|
||||
'''
|
||||
Suppression des ponctuations
|
||||
|
@ -120,11 +139,25 @@ def Ela_Remove_Noise_from_list(list):
|
|||
def Ela_Tokenize(sentence):
|
||||
# Tokeniser la phrase
|
||||
|
||||
sentence = Ela_Normalize(sentence)
|
||||
try:
|
||||
print(" Tokenaisee de du mot "+str(sentence))
|
||||
sentence = Ela_Normalize(sentence)
|
||||
|
||||
doc = nlp(str(sentence).lower())
|
||||
# Retourner le texte de chaque token
|
||||
return [unidecode(X.text) for X in doc]
|
||||
doc = nlp(str(sentence).lower())
|
||||
|
||||
print(" doc " + str(doc))
|
||||
|
||||
for val in doc:
|
||||
print(" VAL = "+str(val.text)+" unidecode(X.text) = "+str(unidecode(val.text)) )
|
||||
|
||||
|
||||
# Retourner le texte de chaque token
|
||||
return [unidecode(X.text) for X in doc]
|
||||
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e)+" - Line : "+ str(exc_tb.tb_lineno) )
|
||||
return False, " Impossible de recuperer la formation"
|
||||
|
||||
'''
|
||||
2. Enlever les mots les plus fréquents
|
||||
|
@ -171,20 +204,29 @@ def Ela_remove_pronoun(tab_tokens):
|
|||
Le stemming consiste à réduire un mot dans sa forme “racine”. Le but du stemming est de regrouper de nombreuses variantes
|
||||
d’un mot comme un seul et même mot. Par exemple, une fois que l’on applique un stemming sur “Chiens” ou “Chien”,
|
||||
le mot résultant est le même.
|
||||
|
||||
important :
|
||||
seuls les mots français sont stemmizé. les autres non.
|
||||
'''
|
||||
def Ela_stemmize(tab_tokens):
|
||||
|
||||
tab_ret_val = []
|
||||
print(" VERIFICATION SI LE MOT EST FR : " + str(tab_tokens))
|
||||
for mot in tab_tokens:
|
||||
if( mycommon.check_word_in_fr_dict(str(mot)) ):
|
||||
tab_ret_val.append(stemmer.stem(mot.text))
|
||||
else:
|
||||
tab_ret_val.append(mot.text)
|
||||
try:
|
||||
tab_ret_val = []
|
||||
print(" VERIFICATION SI LE MOT EST FR : " + str(tab_tokens))
|
||||
for mot in tab_tokens:
|
||||
if( mycommon.check_word_in_fr_dict(str(mot)) ):
|
||||
tab_ret_val.append(stemmer.stem(mot))
|
||||
else:
|
||||
tab_ret_val.append(mot)
|
||||
|
||||
print(" STMISATION TAB = "+str(tab_ret_val))
|
||||
print(" STMISATION TAB = "+str(tab_ret_val))
|
||||
|
||||
return tab_ret_val
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e)+" - Line : "+ str(exc_tb.tb_lineno) )
|
||||
return False, " Impossible de recuperer la formation"
|
||||
|
||||
return tab_ret_val
|
||||
#return [stemmer.stem(token.text) for token in tab_tokens]
|
||||
|
||||
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
,index,mots,occurence,moyenne,id_formation,source_field
|
||||
0,0,sommeil,1,0.07,8220,default
|
||||
1,1,stress,1,0.07,8220,default
|
||||
2,2,coaching,1,0.07,8220,default
|
||||
3,3,equilibre,1,0.07,8220,default
|
||||
4,4,relax,1,0.07,8220,default
|
||||
5,5,typ,1,0.07,8220,default
|
||||
6,6,trouv,1,0.07,8220,default
|
||||
7,7,accompagn,2,0.14,8220,default
|
||||
8,8,sante,2,0.14,8220,default
|
||||
9,9,travail,1,0.07,8220,default
|
||||
10,10,nutrit,1,0.07,8220,default
|
||||
11,11,respir,1,0.07,8220,default
|
||||
0,0,reglement,1,0.08,8866,objectif
|
||||
1,1,publiqu,1,0.08,8866,objectif
|
||||
2,2,const,1,0.08,8866,objectif
|
||||
3,3,format,1,0.08,8866,objectif
|
||||
4,4,appliqu,1,0.08,8866,objectif
|
||||
5,5,comptabilit,1,0.08,8866,objectif
|
||||
6,6,object,1,0.08,8866,objectif
|
||||
7,7,regl,1,0.08,8866,objectif
|
||||
8,8,evolu,1,0.08,8866,objectif
|
||||
9,9,princip,1,0.08,8866,objectif
|
||||
10,10,maitris,1,0.08,8866,objectif
|
||||
11,11,impose,1,0.08,8866,objectif
|
||||
|
|
|
|
@ -428,10 +428,17 @@ def ela_recherche_tokens(sentence):
|
|||
|
||||
'''
|
||||
print(" VERIF : "+str(tab_tokens3))
|
||||
tab_corrected_word = []
|
||||
for mot in tab_tokens3:
|
||||
mycommon.check_word_in_fr_dict(str(mot))
|
||||
mycommon.recherche_check_word_in_fr_dict(str(mot))
|
||||
val = ls.correct_fr_word(str(mot))
|
||||
if( val ):
|
||||
tab_corrected_word.append(str(val))
|
||||
|
||||
tab_tokens4 = ls.Ela_stemmize(tab_tokens3)
|
||||
|
||||
print("corrected word = "+str(tab_corrected_word))
|
||||
|
||||
tab_tokens4 = ls.Ela_stemmize(tab_corrected_word)
|
||||
|
||||
print(" VERIF APRES STEMISATION : " + str(tab_tokens4))
|
||||
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
mots occurence moyenne id_formation source_field
|
||||
0 sommeil 1 0.07 8220 default
|
||||
1 stress 1 0.07 8220 default
|
||||
2 coaching 1 0.07 8220 default
|
||||
3 equilibre 1 0.07 8220 default
|
||||
4 relax 1 0.07 8220 default
|
||||
5 typ 1 0.07 8220 default
|
||||
6 trouv 1 0.07 8220 default
|
||||
7 accompagn 2 0.14 8220 default
|
||||
8 sante 2 0.14 8220 default
|
||||
9 travail 1 0.07 8220 default
|
||||
10 nutrit 1 0.07 8220 default
|
||||
11 respir 1 0.07 8220 default
|
||||
mots occurence moyenne id_formation source_field
|
||||
0 reglement 1 0.08 8866 objectif
|
||||
1 publiqu 1 0.08 8866 objectif
|
||||
2 const 1 0.08 8866 objectif
|
||||
3 format 1 0.08 8866 objectif
|
||||
4 appliqu 1 0.08 8866 objectif
|
||||
5 comptabilit 1 0.08 8866 objectif
|
||||
6 object 1 0.08 8866 objectif
|
||||
7 regl 1 0.08 8866 objectif
|
||||
8 evolu 1 0.08 8866 objectif
|
||||
9 princip 1 0.08 8866 objectif
|
||||
10 maitris 1 0.08 8866 objectif
|
||||
11 impose 1 0.08 8866 objectif
|
|
@ -290,10 +290,9 @@ def tryInt(val):
|
|||
|
||||
'''
|
||||
Cette fonction verifie si un mot est dans le dictionnaire français - une table interne
|
||||
si non, le mot est enregistré dans une table pour traitement utérieur
|
||||
si non, le mot est enregistré dans une table pour traitement utérieur.
|
||||
Utilisé dans le cas de l'indexation d'une formation
|
||||
'''
|
||||
|
||||
|
||||
def check_word_in_fr_dict(mot=None):
|
||||
try:
|
||||
print("#### analyse du mot "+str(mot))
|
||||
|
@ -304,6 +303,7 @@ def check_word_in_fr_dict(mot=None):
|
|||
|
||||
if (val_tmp <= 0):
|
||||
myprint(" Le mot '" + mot + "' n'existe pas dans le dictionnaire")
|
||||
|
||||
mydata['mot'] = mot
|
||||
mydata['treated'] = int("0")
|
||||
mydata['update_date'] = datetime.now()
|
||||
|
@ -324,4 +324,25 @@ def check_word_in_fr_dict(mot=None):
|
|||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
'''
|
||||
Cette fonction recherche à savoir si un mot est dans le dictionnaire dans le cadre de la re
|
||||
recherche d'un utilisateur '''
|
||||
def recherche_check_word_in_fr_dict(mot=None):
|
||||
try:
|
||||
print("#### analyse du mot "+str(mot))
|
||||
col_name = dbname["list_mots_fr"]
|
||||
val_tmp = col_name.count_documents({'mot': str(mot)})
|
||||
|
||||
if (val_tmp <= 0):
|
||||
myprint(" Le mot '" + mot + "' n'existe pas dans le dictionnaire")
|
||||
|
||||
return False
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
|
||||
return False
|
||||
|
|
Loading…
Reference in New Issue