23/04/22 - 13h30

master
ChérifBALDE 2022-04-23 13:34:21 +02:00 committed by cherif
parent 8ce16ed6b0
commit 4480b8b45b
6 changed files with 155 additions and 46 deletions

View File

@ -214,7 +214,7 @@ def Ela_remove_pronoun(tab_tokens):
mytok = nlp(str(token).lower()) mytok = nlp(str(token).lower())
for token2 in mytok: for token2 in mytok:
if token2.pos_ != 'DET' and token2.pos_ != 'CCONJ' and token2.pos_ != 'ADP': if token2.pos_ != 'DET' and token2.pos_ != 'CCONJ' and token2.pos_ != 'ADP':
mywords.append(mytok) mywords.append(str(mytok))
return mywords return mywords
''' '''

View File

@ -1,2 +1,39 @@
,index,mots,occurence,moyenne,id_formation,source_field ,index,mots,occurence,moyenne,id_formation,source_field
0,0,formation4,0,0.0,formation4,default 0,0,"
",8,0.15,mysy_ytubes_04,description
1,1,depart,1,0.02,mysy_ytubes_04,description
2,2,resultat,1,0.02,mysy_ytubes_04,description
3,3,dcg,1,0.02,mysy_ytubes_04,description
4,4,bts,1,0.02,mysy_ytubes_04,description
5,5,"
",1,0.02,mysy_ytubes_04,description
6,6,gratuit,1,0.02,mysy_ytubes_04,description
7,7,marg,1,0.02,mysy_ytubes_04,description
8,8,analys,1,0.02,mysy_ytubes_04,description
9,9,notion,1,0.02,mysy_ytubes_04,description
10,10,paris,1,0.02,mysy_ytubes_04,description
11,11,villetaneuse,1,0.02,mysy_ytubes_04,description
12,12,iut,1,0.02,mysy_ytubes_04,description
13,13,zambotto,1,0.02,mysy_ytubes_04,description
14,14,cout,3,0.06,mysy_ytubes_04,description
15,15,cour,3,0.06,mysy_ytubes_04,description
16,16,different,1,0.02,mysy_ytubes_04,description
17,17,licenc,1,0.02,mysy_ytubes_04,description
18,18,calcul,1,0.02,mysy_ytubes_04,description
19,19,professeur,1,0.02,mysy_ytubes_04,description
20,20,corinne,1,0.02,mysy_ytubes_04,description
21,21,lign,1,0.02,mysy_ytubes_04,description
22,22,universit,1,0.02,mysy_ytubes_04,description
23,23,charg,1,0.02,mysy_ytubes_04,description
24,24,nord,1,0.02,mysy_ytubes_04,description
25,25,comptabilit,4,0.08,mysy_ytubes_04,description
26,26,sorbonne,1,0.02,mysy_ytubes_04,description
27,27,general,1,0.02,mysy_ytubes_04,description
28,28,debut,1,0.02,mysy_ytubes_04,description
29,29,stmg,1,0.02,mysy_ytubes_04,description
30,30,niveau,1,0.02,mysy_ytubes_04,description
31,31,prix,1,0.02,mysy_ytubes_04,description
32,32,incorporees,1,0.02,mysy_ytubes_04,description
33,33,gea,2,0.04,mysy_ytubes_04,description
34,34,gestion,3,0.06,mysy_ytubes_04,description

1 index mots occurence moyenne id_formation source_field
2 0 0 formation4 0 8 0.0 0.15 formation4 mysy_ytubes_04 default description
3 1 1 depart 1 0.02 mysy_ytubes_04 description
4 2 2 resultat 1 0.02 mysy_ytubes_04 description
5 3 3 dcg 1 0.02 mysy_ytubes_04 description
6 4 4 bts 1 0.02 mysy_ytubes_04 description
7 5 5 1 0.02 mysy_ytubes_04 description
8 6 6 gratuit 1 0.02 mysy_ytubes_04 description
9 7 7 marg 1 0.02 mysy_ytubes_04 description
10 8 8 analys 1 0.02 mysy_ytubes_04 description
11 9 9 notion 1 0.02 mysy_ytubes_04 description
12 10 10 paris 1 0.02 mysy_ytubes_04 description
13 11 11 villetaneuse 1 0.02 mysy_ytubes_04 description
14 12 12 iut 1 0.02 mysy_ytubes_04 description
15 13 13 zambotto 1 0.02 mysy_ytubes_04 description
16 14 14 cout 3 0.06 mysy_ytubes_04 description
17 15 15 cour 3 0.06 mysy_ytubes_04 description
18 16 16 different 1 0.02 mysy_ytubes_04 description
19 17 17 licenc 1 0.02 mysy_ytubes_04 description
20 18 18 calcul 1 0.02 mysy_ytubes_04 description
21 19 19 professeur 1 0.02 mysy_ytubes_04 description
22 20 20 corinne 1 0.02 mysy_ytubes_04 description
23 21 21 lign 1 0.02 mysy_ytubes_04 description
24 22 22 universit 1 0.02 mysy_ytubes_04 description
25 23 23 charg 1 0.02 mysy_ytubes_04 description
26 24 24 nord 1 0.02 mysy_ytubes_04 description
27 25 25 comptabilit 4 0.08 mysy_ytubes_04 description
28 26 26 sorbonne 1 0.02 mysy_ytubes_04 description
29 27 27 general 1 0.02 mysy_ytubes_04 description
30 28 28 debut 1 0.02 mysy_ytubes_04 description
31 29 29 stmg 1 0.02 mysy_ytubes_04 description
32 30 30 niveau 1 0.02 mysy_ytubes_04 description
33 31 31 prix 1 0.02 mysy_ytubes_04 description
34 32 32 incorporees 1 0.02 mysy_ytubes_04 description
35 33 33 gea 2 0.04 mysy_ytubes_04 description
36 34 34 gestion 3 0.06 mysy_ytubes_04 description
37
38
39

View File

@ -491,44 +491,50 @@ Cela revient a chercher "sentence" dans la table "ela_index", where "source_fiel
Elle retourne l'id_externe de la formation Elle retourne l'id_externe de la formation
''' '''
def ela_recherche_tokens_source_field(sentence, source_fied=""): def ela_recherche_tokens_source_field(sentence, source_fied=""):
try:
if( len(str(source_fied)) == 0):
mycommon.myprint(str(inspect.stack()[0][3])+" - source_fied est vide")
return False
if( len(str(source_fied)) == 0): # Verification que source_field correspond a la bonne colonne
mycommon.myprint(str(inspect.stack()[0][3])+" - source_fied est vide") field_list = ['default', 'title', 'description','objectif',]
return False if source_fied not in field_list:
mycommon.myprint(str(inspect.stack()[0][3])+" - Le champ '" + source_fied + "' n'existe pas, requete annulée")
return False
# Verification que source_field correspond a la bonne colonne print(" ici: sentence = "+sentence+", -- source_fied ="+source_fied)
field_list = ['default', 'title', 'description','objectif'] tab_training_id = []
if source_fied not in field_list: tab_tokens = ls.Ela_Tokenize(sentence)
mycommon.myprint(str(inspect.stack()[0][3])+" - Le champ '" + source_fied + "' n'existe pas, requete annulée") tab_tokens2 = ls.Ela_remove_stop_words(tab_tokens)
return False tab_tokens3 = ls.Ela_remove_pronoun(tab_tokens2)
tab_tokens4 = ls.Ela_stemmize(tab_tokens3)
tab_tokens4.sort()
tab_training_id = [] '''
tab_tokens = ls.Ela_Tokenize(sentence) tab_tokens4 contient la liste des mots à rechercher.
tab_tokens2 = ls.Ela_remove_stop_words(tab_tokens) En suite pour chaque mot de "tab_tokens4" on regarde dans la table "elaindex"
tab_tokens3 = ls.Ela_remove_pronoun(tab_tokens2) a quoi cela pourrait correspondre.
tab_tokens4 = ls.Ela_stemmize(tab_tokens3) '''
tab_tokens4.sort()
''' collection = db["elaindex"]
tab_tokens4 contient la liste des mots à rechercher.
En suite pour chaque mot de "tab_tokens4" on regarde dans la table "elaindex"
a quoi cela pourrait correspondre.
'''
collection = db["elaindex"]
for token in tab_tokens4: for token in tab_tokens4:
print(" #### Token rechercher dans l'index est : "+str(token)) print(" #### Token rechercher dans l'index est : '"+str(token)+"' et le source_field = '"+str(source_fied)+"' ")
for doc in collection.find({"mots":token, "source_field":source_fied}): for doc in collection.find({"mots":token, "source_field":source_fied}):
#print(" Trouvé dans le cours N° "+str(doc["id_formation"])) #print(" Trouvé dans le cours N° "+str(doc["id_formation"]))
if str(doc["id_formation"]) not in tab_training_id : if str(doc["id_formation"]) not in tab_training_id :
tab_training_id.append(str(doc["id_formation"])) tab_training_id.append(str(doc["id_formation"]))
#print(" Pour la phrase "+str(sentence)+", voici les formations correspondantes") #print(" Pour la phrase "+str(sentence)+", voici les formations correspondantes")
#print(tab_training_id) #print(tab_training_id)
return tab_training_id return tab_training_id
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "Traitement impossible "
'''' ''''
Cette fonction indexe les articles et avis. Cette fonction indexe les articles et avis.

View File

@ -1,2 +1,36 @@
mots occurence moyenne id_formation source_field mots occurence moyenne id_formation source_field
0 formation4 0 0.0 formation4 default 0 \n 8 0.15 mysy_ytubes_04 description
1 depart 1 0.02 mysy_ytubes_04 description
2 resultat 1 0.02 mysy_ytubes_04 description
3 dcg 1 0.02 mysy_ytubes_04 description
4 bts 1 0.02 mysy_ytubes_04 description
5 \n\n 1 0.02 mysy_ytubes_04 description
6 gratuit 1 0.02 mysy_ytubes_04 description
7 marg 1 0.02 mysy_ytubes_04 description
8 analys 1 0.02 mysy_ytubes_04 description
9 notion 1 0.02 mysy_ytubes_04 description
10 paris 1 0.02 mysy_ytubes_04 description
11 villetaneuse 1 0.02 mysy_ytubes_04 description
12 iut 1 0.02 mysy_ytubes_04 description
13 zambotto 1 0.02 mysy_ytubes_04 description
14 cout 3 0.06 mysy_ytubes_04 description
15 cour 3 0.06 mysy_ytubes_04 description
16 different 1 0.02 mysy_ytubes_04 description
17 licenc 1 0.02 mysy_ytubes_04 description
18 calcul 1 0.02 mysy_ytubes_04 description
19 professeur 1 0.02 mysy_ytubes_04 description
20 corinne 1 0.02 mysy_ytubes_04 description
21 lign 1 0.02 mysy_ytubes_04 description
22 universit 1 0.02 mysy_ytubes_04 description
23 charg 1 0.02 mysy_ytubes_04 description
24 nord 1 0.02 mysy_ytubes_04 description
25 comptabilit 4 0.08 mysy_ytubes_04 description
26 sorbonne 1 0.02 mysy_ytubes_04 description
27 general 1 0.02 mysy_ytubes_04 description
28 debut 1 0.02 mysy_ytubes_04 description
29 stmg 1 0.02 mysy_ytubes_04 description
30 niveau 1 0.02 mysy_ytubes_04 description
31 prix 1 0.02 mysy_ytubes_04 description
32 incorporees 1 0.02 mysy_ytubes_04 description
33 gea 2 0.04 mysy_ytubes_04 description
34 gestion 3 0.06 mysy_ytubes_04 description

View File

@ -197,7 +197,11 @@ def get_user_email_from_recid(recid = ""):
coll_token = dbname['user_account'] coll_token = dbname['user_account']
tmp_val = coll_token.find({'recid': str(recid), 'active': '1'}) tmp_val = coll_token.find({'recid': str(recid), 'active': '1'})
user_email = tmp_val[0]['email'] if( tmp_val and tmp_val[0] and tmp_val[0]['email']):
user_email = tmp_val[0]['email']
else:
return False, "Impossible de supprimer le compte utilisateur"
return user_email return user_email
except Exception as e: except Exception as e:
@ -205,11 +209,6 @@ def get_user_email_from_recid(recid = ""):
myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "Impossible de supprimer le compte utilisateur" return False, "Impossible de supprimer le compte utilisateur"
''' '''
recuperation du recid du partner recuperation du recid du partner
''' '''

View File

@ -217,6 +217,15 @@ def get_all_class(diction):
''' '''
Cette fonction recherche les formations correspondant à un text Cette fonction recherche les formations correspondant à un text
IMPORTANT : Dans la recherche par tips, on fait du && et non du ou.
exemple :
pour la recherche "description:"fichier" title:"Niveau 1""
le système va chercher toutes les formation qui on :
- "description:"fichier" ET ET ET ET ET
- title:"Niveau 1"
En gros, on fait une intersection entre les 2 listes.
''' '''
def recherche_text_simple(diction): def recherche_text_simple(diction):
@ -375,6 +384,7 @@ def recherche_text_simple(diction):
tips = re.findall(regexp, search_text, re.MULTILINE) tips = re.findall(regexp, search_text, re.MULTILINE)
nb_tips = len(tips) nb_tips = len(tips)
final_message3 = {}
if( nb_tips > 0 ): if( nb_tips > 0 ):
mycommon.myprint(" Une recherche par tips a été identifiée") mycommon.myprint(" Une recherche par tips a été identifiée")
@ -385,26 +395,35 @@ def recherche_text_simple(diction):
final_message = [] final_message = []
insertObject = [] insertObject = []
is_first_tip = True
for val in tips : for val in tips :
val2 = val.replace('"', '') val2 = val.replace('"', '')
new_diction['search_text'] = val2 new_diction['search_text'] = val2
print(" On va recherche : #####"+str(new_diction['search_text'] )) print(" On va recherche : #####"+str(new_diction['search_text'] ))
retval, message = recherche_tips_ret_ref(new_diction) retval, message = recherche_tips_ret_ref(new_diction)
#print(" pour la recherche de : "+str(new_diction)+" -- Voici le resultat "+str(message)) print(" pour la recherche de : "+str(new_diction)+" -- Voici le resultat "+str(message))
if (retval == True ): if (retval == True ):
final_retval = True final_retval = True
#print(" contact de message "+str(message)+ " # final_message "+str(final_message))
if( is_first_tip is True) :
final_message3 = message
else:
final_message3 = [x for x in message if x in final_message3]
print(" final_message3 = "+str(final_message3))
final_message.append(message) final_message.append(message)
is_first_tip = False
final_message = final_message3
#print(" liste defitive des ref "+str(final_message)) #print(" liste defitive des ref "+str(final_message))
final_message2 = [] final_message2 = []
for t in final_message: for t in final_message:
if( t is not False): if( t is not False):
for y in t: #print( " unique = "+str(t))
#print( " unique = "+str(y)) final_message2.append(t)
final_message2.append(y)
coll_name = dbname['myclass'] coll_name = dbname['myclass']
@ -845,6 +864,8 @@ def recherche_tips_ret_ref(diction):
mycommon.myprint(str(inspect.stack()[0][3])+" - le mot '"+val1+" ne correspond aucun tips, sorry") mycommon.myprint(str(inspect.stack()[0][3])+" - le mot '"+val1+" ne correspond aucun tips, sorry")
return False, " le mot '"+val1+" ne correspond aucun tips, sorry" return False, " le mot '"+val1+" ne correspond aucun tips, sorry"
if (str(my_tips) == "desc"):
my_tips = 'description'
mycommon.myprint(" Le tips recherché est "+my_tips+". Sa distance de Levenshtein ="+str(min)) mycommon.myprint(" Le tips recherché est "+my_tips+". Sa distance de Levenshtein ="+str(min))
@ -857,6 +878,9 @@ def recherche_tips_ret_ref(diction):
tab_training = [] tab_training = []
tab_training = ela_index.ela_recherche_tokens_source_field(str(chaine[1]).lower(), str(my_tips) ) tab_training = ela_index.ela_recherche_tokens_source_field(str(chaine[1]).lower(), str(my_tips) )
if( tab_training is False):
return False, tab_training
#mycommon.myprint(" pour phrase : #" + str(chaine[1]).lower() + "# , Pour le tips #"+ str(my_tips)+"#, voici la liste des formations") #mycommon.myprint(" pour phrase : #" + str(chaine[1]).lower() + "# , Pour le tips #"+ str(my_tips)+"#, voici la liste des formations")
#mycommon.myprint(tab_training) #mycommon.myprint(tab_training)
@ -978,8 +1002,14 @@ def recherche_tips(diction):
mycommon.myprint(str(inspect.stack()[0][3])+" - le mot '"+val1+" ne correspond aucun tips, sorry") mycommon.myprint(str(inspect.stack()[0][3])+" - le mot '"+val1+" ne correspond aucun tips, sorry")
return False, " le mot '"+val1+" ne correspond aucun tips, sorry" return False, " le mot '"+val1+" ne correspond aucun tips, sorry"
'''
Remplacement de l'abreviation 'desc' par description '''
mycommon.myprint(" Le tips recherché est "+my_tips+". Sa distance de Levenshtein ="+str(min)) if( str(my_tips) == "desc"):
my_tips = 'description'
mycommon.myprint(" Le tips recherché est '"+my_tips+"'. Sa distance de Levenshtein ="+str(min))
tab_ret = {} tab_ret = {}
tab_ret['tips'] = str(my_tips) tab_ret['tips'] = str(my_tips)
@ -990,6 +1020,9 @@ def recherche_tips(diction):
tab_training = [] tab_training = []
tab_training = ela_index.ela_recherche_tokens_source_field(str(chaine[1]).lower(), str(my_tips) ) tab_training = ela_index.ela_recherche_tokens_source_field(str(chaine[1]).lower(), str(my_tips) )
if (tab_training is False):
return False, tab_training
mycommon.myprint(" pour phrase : #" + str(chaine[1]).lower() + "# , Pour le tips #"+ str(my_tips)+"#, voici la liste des formations") mycommon.myprint(" pour phrase : #" + str(chaine[1]).lower() + "# , Pour le tips #"+ str(my_tips)+"#, voici la liste des formations")
mycommon.myprint(tab_training) mycommon.myprint(tab_training)