29/09/22 - 11h
parent
25b6c510e6
commit
02232da664
|
@ -644,7 +644,7 @@ def Ela_list_to_mongo(tab_tokens, traning_id, source_field):
|
|||
moyenne = round(int(occurrences[str(tmp)]) / size_tab, 2)
|
||||
final_lists.append([str(tmp), int(str(occurrences[str(tmp)])), str(moyenne), str(traning_id), str(source_field)])
|
||||
|
||||
print(final_lists)
|
||||
print("final_lists to record = "+str(final_lists))
|
||||
data = pd.DataFrame(final_lists, columns=('mots', "occurence", 'moyenne', 'id_formation', 'source_field'))
|
||||
my_file.write(str(data))
|
||||
my_file.close()
|
||||
|
|
14
class_mgt.py
14
class_mgt.py
|
@ -18,6 +18,7 @@ from pymongo import ReturnDocument
|
|||
import GlobalVariable as MYSY_GV
|
||||
from math import isnan
|
||||
import GlobalVariable as MYSY_GV
|
||||
import ela_index_bdd_classes as eibdd
|
||||
|
||||
|
||||
|
||||
|
@ -387,6 +388,12 @@ def add_class(diction):
|
|||
)
|
||||
|
||||
if ret_val and ret_val['_id']:
|
||||
#Indexation Title de la nouvelle formation ajoutée
|
||||
training_to_index_title = {}
|
||||
training_to_index_title['internal_url'] =mydata['internal_url']
|
||||
training_to_index_title['reindex'] = '1'
|
||||
eibdd.ela_index_given_classes_title(training_to_index_title)
|
||||
|
||||
return True, "La formation a bien été ajoutée"
|
||||
else:
|
||||
mycommon.myprint(" Impossible d'ajouter la formation "+str(str(mydata['external_code'])))
|
||||
|
@ -679,6 +686,13 @@ def update_class(diction):
|
|||
nb_doc = str(ret_val['_id'])
|
||||
mycommon.myprint(
|
||||
str(inspect.stack()[0][3]) + " - La formation a bin ete mise à jour =" + str(nb_doc))
|
||||
|
||||
# Indexation Title de la nouvelle formation ajoutée
|
||||
training_to_index_title = {}
|
||||
training_to_index_title['internal_url'] = ret_val['internal_url']
|
||||
training_to_index_title['reindex'] = '1'
|
||||
eibdd.ela_index_given_classes_title(training_to_index_title)
|
||||
|
||||
return True, " La formation a été mise à jour"
|
||||
|
||||
else:
|
||||
|
|
|
@ -1,10 +1,4 @@
|
|||
,index,mots,occurence,moyenne,id_formation,source_field
|
||||
0,0,startup,1,0.33,"
|
||||
lever-des-fonds-pour-ma-startup
|
||||
-2",title
|
||||
1,1,fond,1,0.33,"
|
||||
lever-des-fonds-pour-ma-startup
|
||||
-2",title
|
||||
2,2,lever,1,0.33,"
|
||||
lever-des-fonds-pour-ma-startup
|
||||
-2",title
|
||||
0,0,comprendre,1,0.33,CH_PA1201910,title
|
||||
1,1,develov,1,0.33,CH_PA1201910,title
|
||||
2,2,cherif,1,0.33,CH_PA1201910,title
|
||||
|
|
|
|
@ -320,6 +320,120 @@ def ela_index_all_classes_title():
|
|||
return False, "Impossible d'indexer la BDD des formations ==> KO"
|
||||
|
||||
|
||||
|
||||
"""
|
||||
Cette fonction va indexer une formation donnée ou toutes les formation d'un partenaire.
|
||||
Pour ce faire elle prend en argument :
|
||||
- internal_url
|
||||
- partenaire_recid
|
||||
|
||||
/!\ Important : La fonction peut reindexer les formation si l'argument
|
||||
reindexe = 1
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def ela_index_given_classes_title(diction) :
|
||||
# Making a Connection with MongoClient
|
||||
try:
|
||||
# Dictionnaire des champs utilisables
|
||||
field_list = ['internal_url', 'partner_owner_recid', 'reindex']
|
||||
incom_keys = diction.keys()
|
||||
|
||||
'''
|
||||
# Verification que les champs reçus dans l'API sont bien dans la liste des champs autorisés
|
||||
# Cela evite le cas ou une entité tierce ajouter les valeurs inconnu dans l'API
|
||||
# Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste
|
||||
# field_list.
|
||||
'''
|
||||
for val in incom_keys:
|
||||
if str(val).lower() not in str(field_list).lower():
|
||||
mycommon.myprint(str(inspect.stack()[0][3]) + " Le champ '" + val + "' n'existe pas, MAJ annulée")
|
||||
return False, " Impossible de se connecter, merci de contacter le support technique"
|
||||
|
||||
|
||||
|
||||
check = False
|
||||
internal_url = ""
|
||||
if ("internal_url" in diction.keys()):
|
||||
if diction['internal_url']:
|
||||
check = True
|
||||
internal_url = diction['internal_url']
|
||||
|
||||
partner_owner_recid = ""
|
||||
if ("partner_owner_recid" in diction.keys()):
|
||||
if diction['partner_owner_recid']:
|
||||
check = True
|
||||
partner_owner_recid = diction['partner_owner_recid']
|
||||
|
||||
reindex = ""
|
||||
if ("reindex" in diction.keys()):
|
||||
if diction['reindex']:
|
||||
check = True
|
||||
reindex = diction['reindex']
|
||||
|
||||
if( check is False ):
|
||||
mycommon.myprint(str(inspect.stack()[0][3]) + " - Aucune formation à indexer")
|
||||
return True, " Aucune formation à indexer"
|
||||
|
||||
|
||||
collection = MYSY_GV.dbname["myclass"]
|
||||
'''
|
||||
ATTENTION ATTENTION
|
||||
/!\ : avant l'indexation, on vide la table des indexes.
|
||||
cette logique est tres gourmande.
|
||||
ce quil faudrait faire :
|
||||
- si une formation est indexée, mettre un fag pour dire : indexé => OK
|
||||
- si on met à jour une formation ==> Mettre le flag à KO
|
||||
- si on ajoute une formation, mettre le flag à KO
|
||||
|
||||
|
||||
ex : indexation du champ 'title' ou du champ 'objectif', je vais introduitre la notion de 'source_fied'
|
||||
pour l'indexation title, 'source_fied' = 'title'
|
||||
pour l'indexation objectif, 'source_fied' = 'objectif'
|
||||
|
||||
apres pour l'indexation, il ne faudra indexer que les formation dont le flag est KO.
|
||||
aussi, avant d'indexer une formation, supprimer tous les index de la dite formation pour remplacer tous les anciens et eviter un doublon
|
||||
'''
|
||||
|
||||
nb_class_indexed = 0
|
||||
myquery = {'valide':'1'}
|
||||
if( internal_url and len(str(internal_url)) > 0 ):
|
||||
myquery['internal_url'] = internal_url
|
||||
|
||||
if (partner_owner_recid and len(str(partner_owner_recid)) > 0):
|
||||
myquery['partner_owner_recid'] = partner_owner_recid
|
||||
|
||||
if (reindex and len(str(reindex)) > 0):
|
||||
if(reindex == "0" ):
|
||||
myquery['indexed_title'] = "0"
|
||||
|
||||
#print(" QUERY INDEX TITLE ="+str(myquery))
|
||||
|
||||
for doc in collection.find(myquery):
|
||||
if ("title" in doc.keys()):
|
||||
chaine = doc["title"]
|
||||
mycommon.myprint(" external_code = " + str(doc["external_code"]) + " ==> " + str(chaine))
|
||||
retval = ela_index_class(str(doc["external_code"]), "title")
|
||||
print(" apres : ela_index_given_classes_title retval = " + str(retval))
|
||||
|
||||
if (retval is False):
|
||||
mycommon.myprint(
|
||||
str(inspect.stack()[0][3]) + "IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"]))
|
||||
else:
|
||||
nb_class_indexed = nb_class_indexed + 1
|
||||
|
||||
# retval = ls.ela_index_record_field(chaine, str(doc["external_code"]))
|
||||
|
||||
mycommon.myprint(str(nb_class_indexed) + " ont été indexes")
|
||||
return True, str(nb_class_indexed) + " ont été indexes "
|
||||
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
|
||||
return False, "Impossible d'indexer la BDD des formations ==> KO"
|
||||
|
||||
|
||||
'''
|
||||
La fonction "ela_index_all_classes_desc" va aller chercher chaque cours dans la BDD
|
||||
et créer les mots clées des "description" qui sont ensuite enregister dans la table "elaindex"
|
||||
|
@ -589,6 +703,8 @@ def mysy_recherche_levenshtein(tab_mot):
|
|||
coll_name = MYSY_GV.dbname['elaindex']
|
||||
training_mots = []
|
||||
|
||||
print("mysy_recherche_levenshtein debut "+str(tab_mot))
|
||||
|
||||
for mot in tab_mot:
|
||||
|
||||
'''
|
||||
|
@ -606,8 +722,14 @@ def mysy_recherche_levenshtein(tab_mot):
|
|||
|
||||
else:
|
||||
|
||||
if( len( mot ) > 2):
|
||||
mot_first_caract = mot[0:3]
|
||||
if( len( mot ) > 3):
|
||||
|
||||
"""
|
||||
/!\ Update du 29/09/22 :
|
||||
J'ai rendu le mysy_recherche_levenshtein un peu souple en
|
||||
en disant que si taille du mot > 5 alors j'accepte une distance de 2 à condition que 4 première lettre soient identiques
|
||||
"""
|
||||
mot_first_caract = mot[0:4]
|
||||
#print(" Traitement de " + str(mot) + " = 3 first => " + str(mot_first_caract))
|
||||
|
||||
v = {'$regex': "^" + mot_first_caract}
|
||||
|
@ -621,11 +743,11 @@ def mysy_recherche_levenshtein(tab_mot):
|
|||
if( str(x['mots']) not in training_mots):
|
||||
training_mots.append( str(x['mots']) )
|
||||
|
||||
elif( len(mot) <= 10 and cmp_levenshtein <= 0):
|
||||
elif( len(mot) <= 10 and cmp_levenshtein <= 1):
|
||||
if( str(x['mots']) not in training_mots):
|
||||
training_mots.append( str(x['mots']) )
|
||||
|
||||
elif ( cmp_levenshtein <= 0):
|
||||
elif ( cmp_levenshtein <= 1):
|
||||
if (str(x['mots']) not in training_mots):
|
||||
training_mots.append(str(x['mots']))
|
||||
|
||||
|
@ -633,7 +755,7 @@ def mysy_recherche_levenshtein(tab_mot):
|
|||
print(" KOOFF cmp_levenshtein : " + mot + " CMP " + str(x['mots']) + " = " + str(cmp_levenshtein) +
|
||||
" ==> id_formation" + str(x['id_formation']) + " ==> occurrence = " + str(x['occurence']))
|
||||
'''
|
||||
|
||||
print("mysy_recherche_levenshtein FINN " + str(training_mots))
|
||||
return True, training_mots
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
@ -1,6 +1,4 @@
|
|||
mots occurence ... id_formation source_field
|
||||
0 startup 1 ... \nlever-des-fonds-pour-ma-startup\n-2 title
|
||||
1 fond 1 ... \nlever-des-fonds-pour-ma-startup\n-2 title
|
||||
2 lever 1 ... \nlever-des-fonds-pour-ma-startup\n-2 title
|
||||
|
||||
[3 rows x 5 columns]
|
||||
mots occurence moyenne id_formation source_field
|
||||
0 comprendre 1 0.33 CH_PA1201910 title
|
||||
1 develov 1 0.33 CH_PA1201910 title
|
||||
2 cherif 1 0.33 CH_PA1201910 title
|
16
main.py
16
main.py
|
@ -1561,6 +1561,22 @@ def recordClassImage():
|
|||
return jsonify(status=status, message=message)
|
||||
|
||||
|
||||
"""
|
||||
Cette API va indexer une formation donnée ou toutes les formation d'un partenaire.
|
||||
Pour ce faire elle prend en argument :
|
||||
- internal_url
|
||||
- partenaire_recid
|
||||
"""
|
||||
@app.route('/myclass/api/ela_index_given_classes_title/', methods=['POST','GET'])
|
||||
@crossdomain(origin='*')
|
||||
def ela_index_given_classes_title():
|
||||
# On recupere le corps (payload) de la requete
|
||||
payload = request.form.to_dict()
|
||||
print(" ### payload = ",payload)
|
||||
status, retval = ela_index_class.ela_index_given_classes_title(payload)
|
||||
return jsonify(status=status, message=retval)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(" debut api")
|
||||
|
|
Loading…
Reference in New Issue