138 lines
5.3 KiB
Python
138 lines
5.3 KiB
Python
import os
|
|
import csv
|
|
import inspect
|
|
import sys
|
|
from datetime import datetime
|
|
from pymongo import ReturnDocument
|
|
from unidecode import unidecode
|
|
import GlobalVariable as MYSY_GV
|
|
import prj_common as mycommon
|
|
import re
|
|
from pymongo import MongoClient
|
|
import pymongo
|
|
|
|
|
|
'''
|
|
Cette fonction essayer de faire une recherche de leveinstein
|
|
elle prend une liste mot :
|
|
pr chaque mot,
|
|
1 - elle prends les 3 premier caractère
|
|
2 - recherche ceux qui correspondent
|
|
3 - pour chaque mot trouvé, faire un distance de levein et voir
|
|
|
|
puis elle retourne la liste des mots trouvé dans la table elaindex
|
|
|
|
Regles fonctionnelles :
|
|
Si 0 < len(mot) <= 3 ==> Recherche strict à faire
|
|
Si 3 < len(mot) <= 5 ==> Recherche avec une distance de 1
|
|
Si 5 < len(mot) <= 10 ==> Recherche avec une distance de 2
|
|
Si 10 < len(mot) ==> Recherche avec une distance de 3
|
|
'''
|
|
|
|
def mysy_recherche_levenshtein_orig():
|
|
try:
|
|
|
|
coll_name = MYSY_GV.dbname['elaindex']
|
|
mots = ['comptabilite', 'publique', 'fondamentau', 'programme', 'renforcement', 'defi', 'semaine', 'semaine']
|
|
training_mots = []
|
|
|
|
|
|
for mot in mots:
|
|
if( len( mot ) > 2):
|
|
mot_first_caract = mot[0:3]
|
|
#print(" Traitement de " + str(mot) + " = 3 first => " + str(mot_first_caract))
|
|
|
|
regx = re.compile("^" + mot_first_caract, re.IGNORECASE)
|
|
print("regx = " + str(regx))
|
|
v = {'$regex': "^" + mot_first_caract}
|
|
print("v = " + str(v))
|
|
|
|
for x in coll_name.find({'mots': v}):
|
|
cmp_levenshtein = int(mycommon.levenshtein(mot, str(x['mots'])))
|
|
|
|
|
|
if( len(mot) <= 5 and cmp_levenshtein <= 1):
|
|
if( str(x['mots']) not in training_mots):
|
|
training_mots.append( str(x['mots']) )
|
|
|
|
elif( len(mot) <= 10 and cmp_levenshtein <= 2):
|
|
if( str(x['mots']) not in training_mots):
|
|
training_mots.append( str(x['mots']) )
|
|
|
|
elif ( cmp_levenshtein <= 3):
|
|
if (str(x['mots']) not in training_mots):
|
|
training_mots.append(str(x['mots']))
|
|
|
|
elif (cmp_levenshtein > 3):
|
|
print(" KOO ?? KE cmp_levenshtein : "+mot+" CMP "+str(x['mots'])+" = "+str(cmp_levenshtein)+
|
|
" ==> id_formation"+str(x['id_formation'])+" ==> occurrence = "+str(x['occurence']))
|
|
|
|
return True, training_mots
|
|
|
|
except Exception as e:
|
|
exc_type, exc_obj, exc_tb = sys.exc_info()
|
|
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
|
|
return False, None
|
|
|
|
def mysy_recherche_levenshtein(tab_mot):
|
|
try:
|
|
|
|
coll_name = MYSY_GV.dbname['elaindex']
|
|
training_mots = []
|
|
|
|
for mot in tab_mot:
|
|
|
|
'''
|
|
Si le mot appartient à la collection "word_not_stem", alors il faut faire un recherche strict
|
|
pas de recherche elastique à faire.
|
|
'''
|
|
'''print("### + #### '"+str(mot)+"' mycommon.Word_Not_Stemmize(str(mot) = '"+
|
|
mycommon.Word_Not_Stemmize(str(mot)))
|
|
'''
|
|
if (mycommon.Word_Not_Stemmize(str(mot)) is False):
|
|
#"recherche strict"
|
|
if ( str(mot) not in training_mots):
|
|
#print(" Le mot "+str(mot)+" est dans la table 'word_not_stem', donc pas de recherche elastique")
|
|
training_mots.append(str(mot))
|
|
|
|
else:
|
|
|
|
'''
|
|
Si le mot N'APPARTIENT PAS à la collection "word_not_stem" alors c'est un mot classique
|
|
il faut faire une recherche elastique
|
|
'''
|
|
if( len( mot ) > 2):
|
|
mot_first_caract = mot[0:3]
|
|
#print(" Traitement de " + str(mot) + " = 3 first => " + str(mot_first_caract))
|
|
|
|
v = {'$regex': "^" + mot_first_caract}
|
|
#print("v = " + str(v))
|
|
|
|
for x in coll_name.find({'mots': v}):
|
|
cmp_levenshtein = int(mycommon.levenshtein(mot, str(x['mots'])))
|
|
|
|
|
|
if( len(mot) <= 5 and cmp_levenshtein <= 1):
|
|
if( str(x['mots']) not in training_mots):
|
|
training_mots.append( str(x['mots']) )
|
|
|
|
elif( len(mot) <= 10 and cmp_levenshtein <= 2):
|
|
if( str(x['mots']) not in training_mots):
|
|
training_mots.append( str(x['mots']) )
|
|
|
|
elif ( cmp_levenshtein <= 3):
|
|
if (str(x['mots']) not in training_mots):
|
|
training_mots.append(str(x['mots']))
|
|
|
|
'''elif (cmp_levenshtein > 3):
|
|
print(" KOO00 cmp_levenshtein : " + mot + " CMP " + str(x['mots']) + " = " + str(cmp_levenshtein) +
|
|
" ==> id_formation" + str(x['id_formation']) + " ==> occurrence = " + str(x['occurence']))
|
|
'''
|
|
|
|
return True, training_mots
|
|
|
|
except Exception as e:
|
|
exc_type, exc_obj, exc_tb = sys.exc_info()
|
|
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
|
|
return False, None
|