Validation 13/03/22 - 12h21

master
ChérifBALDE 2022-03-13 13:41:42 +01:00 committed by cherif
parent e9009638ea
commit e3cc35a035
7 changed files with 111 additions and 22 deletions

View File

@ -241,7 +241,6 @@ def ela_index_record_field(lines, class_id, source_field = ""):
if len(str(source_field)) == 0 : if len(str(source_field)) == 0 :
source_field = 'default' source_field = 'default'
myquery = {"id_formation": class_id, "source_field":source_field} myquery = {"id_formation": class_id, "source_field":source_field}
delete_row = coll_name.delete_many(myquery) delete_row = coll_name.delete_many(myquery)
mycommon.myprint(" elaindex - "+str(delete_row.deleted_count)+" documents deleted. Training ==> "+str(class_id)+" ") mycommon.myprint(" elaindex - "+str(delete_row.deleted_count)+" documents deleted. Training ==> "+str(class_id)+" ")

View File

@ -27,8 +27,7 @@ client = MongoClient(CONNECTION_STRING)
dbname = client['cherifdb'] dbname = client['cherifdb']
MAX_KEYWORD = 3
''' '''
Cette fonction ajoute une formation Cette fonction ajoute une formation
@ -152,6 +151,15 @@ def add_class(diction):
if ("mots_cle" in diction.keys()): if ("mots_cle" in diction.keys()):
if diction['mots_cle']: if diction['mots_cle']:
mydata['mots_cle'] = diction['mots_cle'] mydata['mots_cle'] = diction['mots_cle']
'''
Verification du nombre de mots clée : limite MAX_KEYWORD (3)
'''
nb_keyword = mydata['mots_cle'].split(";")
if( len(nb_keyword) > MAX_KEYWORD ):
mycommon.myprint(
str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés")
return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés"
if ("domaine" in diction.keys()): if ("domaine" in diction.keys()):
@ -1164,6 +1172,16 @@ def add_class_mass(file=None, Folder=None, diction=None):
mydata['duree_formation'] = str(df['duree_formation'].values[n]) mydata['duree_formation'] = str(df['duree_formation'].values[n])
mydata['plus_produit'] = str(df['plus_produit'].values[n]) mydata['plus_produit'] = str(df['plus_produit'].values[n])
mydata['mots_cle'] = str(df['mots_cle'].values[n]) mydata['mots_cle'] = str(df['mots_cle'].values[n])
'''
Verification du nombre de mots clée : limite MAX_KEYWORD (3)
'''
nb_keyword = mydata['mots_cle'].split(";")
if( len(nb_keyword) > MAX_KEYWORD ):
mycommon.myprint(
str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés")
return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés"
mydata['distantiel'] = str(df['distantiel'].values[n]) mydata['distantiel'] = str(df['distantiel'].values[n])
mydata['presentiel'] = str(df['presentiel'].values[n]) mydata['presentiel'] = str(df['presentiel'].values[n])

View File

@ -1,5 +1,2 @@
,index,mots,occurence,moyenne,id_formation,source_field ,index,mots,occurence,moyenne,id_formation,source_field
0,0,non-financi,1,0.25,25230,title 0,0,mot1,1,1.0,code01,keyword
1,1,1,1,0.25,25230,title
2,2,financ,1,0.25,25230,title
3,3,niveau,1,0.25,25230,title

1 index mots occurence moyenne id_formation source_field
2 0 0 non-financi mot1 1 0.25 1.0 25230 code01 title keyword
1 1 1 1 0.25 25230 title
2 2 financ 1 0.25 25230 title
3 3 niveau 1 0.25 25230 title

View File

@ -2,7 +2,9 @@
from pymongo import MongoClient from pymongo import MongoClient
import Ela_Spacy as ls import Ela_Spacy as ls
import prj_common as mycommon import prj_common as mycommon
from pymongo import ReturnDocument
import inspect import inspect
import sys, os
client = MongoClient("mongodb://localhost:27017/") client = MongoClient("mongodb://localhost:27017/")
# database # database
@ -24,8 +26,9 @@ def ela_index_class(external_code="", source_field = ""):
if( len(str(source_field)) == 0): if( len(str(source_field)) == 0):
source_field = 'default' source_field = 'default'
for doc in collection.find({"external_code":external_code}): for doc in collection.find({"external_code":external_code}):
# Indexation par defaut : champs TITRE + DESCRIPTION
if ( str(source_field) == "default" ): if ( str(source_field) == "default" ):
chaine = doc["title"]+" - "+doc["description"] chaine = doc["title"]+" - "+doc["description"]
#mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine)) #mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
@ -50,6 +53,7 @@ def ela_index_class(external_code="", source_field = ""):
else: else:
return False return False
# Indexation du champ : title
elif ( str(source_field) == "title" ): elif ( str(source_field) == "title" ):
chaine = doc["title"] chaine = doc["title"]
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine)) # mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
@ -74,6 +78,7 @@ def ela_index_class(external_code="", source_field = ""):
else: else:
return False return False
# Indexation du champ : description
elif ( str(source_field) == "description" ): elif ( str(source_field) == "description" ):
chaine = doc["description"] chaine = doc["description"]
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine)) # mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
@ -98,6 +103,8 @@ def ela_index_class(external_code="", source_field = ""):
else: else:
return False return False
# Indexation du champ : Objectif
elif ( str(source_field) == "objectif" ): elif ( str(source_field) == "objectif" ):
chaine = doc["objectif"] chaine = doc["objectif"]
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine)) # mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
@ -123,6 +130,32 @@ def ela_index_class(external_code="", source_field = ""):
return False return False
# Indexation du champ : mot_cle
elif (str(source_field) == "keyword"):
chaine = doc["mots_cle"]
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
retval = ls.ela_index_record_field(chaine, str(doc["external_code"]), source_field)
'''
# Apres indexation, on met a jour la formation pour
# mettre le champ "indexe" a 1.
'''
if (retval):
# mise à jour du champ "indexed"
myobjectif = {}
myobjectif['indexed_keyword'] = '1'
ret_val2 = collection.find_and_modify(query={"external_code": external_code},
update={"$set": myobjectif}
)
if (ret_val2 and ret_val2['_id']):
mycommon.myprint("La formation indexée (champs objectif) =" + str(ret_val2['_id']))
return True
else:
return False
return True return True
except Exception as e: except Exception as e:
mycommon.myprint(e) mycommon.myprint(e)
@ -179,20 +212,48 @@ def ela_index_all_classes():
mycommon.myprint(e) mycommon.myprint(e)
return False, "Impossible d'indexer la BDD des formations ==> KO" return False, "Impossible d'indexer la BDD des formations ==> KO"
'''
Cette fonction va indexer les mots clés ajoutés à une formation.
Par defaut, seul 3 mots clés sont acceptés par formation.
les mots clés sont separés par des ";".
ex : mot1;mot2;mot3
'''
def ela_index_all_classes_keywords():
try:
collection = db["myclass"]
nb_class_indexed = 0
for doc in collection.find({'indexed_keyword': '0'}):
if ("mots_cle" in doc.keys()):
chaine = doc["mots_cle"]
mycommon.myprint(" external_code = " + str(doc["external_code"]) + " ==> " + str(chaine))
retval = ela_index_class(str(doc["external_code"]), "keyword")
print(" apres : ela_index_all_classes_title retval = " + str(retval))
if (retval is False):
mycommon.myprint(
str(inspect.stack()[0][3]) + "IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"]))
else:
nb_class_indexed = nb_class_indexed + 1
# retval = ls.ela_index_record_field(chaine, str(doc["external_code"]))
mycommon.myprint(str(nb_class_indexed) + " ont été indexes")
return True, str(nb_class_indexed) + " ont été indexes "
return True
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "Impossible d'indexer la BDD des formations ==> KO"
''' '''
La fonction "ela_index_all_classes_title" va aller chercher chaque cours dans la BDD La fonction "ela_index_all_classes_title" va aller chercher chaque cours dans la BDD
et créer les mots clées des title qui sont ensuite enregister dans la table "elaindex" et créer les mots clées des title qui sont ensuite enregister dans la table "elaindex"
''' '''
def ela_index_all_classes_title(): def ela_index_all_classes_title():
# Making a Connection with MongoClient # Making a Connection with MongoClient
try: try:
client = MongoClient("mongodb://localhost:27017/")
# database
db = client["cherifdb"]
# collection
collection = db["myclass"] collection = db["myclass"]
''' '''
ATTENTION ATTENTION ATTENTION ATTENTION
@ -220,7 +281,7 @@ def ela_index_all_classes_title():
print(" apres : ela_index_all_classes_title retval = " + str(retval)) print(" apres : ela_index_all_classes_title retval = " + str(retval))
if (retval is False): if (retval is False):
mycommon.myprint("IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"])) mycommon.myprint(str(inspect.stack()[0][3]) + "IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"]))
else: else:
nb_class_indexed = nb_class_indexed + 1 nb_class_indexed = nb_class_indexed + 1
@ -230,7 +291,8 @@ def ela_index_all_classes_title():
return True, str(nb_class_indexed) + " ont été indexes " return True, str(nb_class_indexed) + " ont été indexes "
except Exception as e: except Exception as e:
mycommon.myprint(e) exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "Impossible d'indexer la BDD des formations ==> KO" return False, "Impossible d'indexer la BDD des formations ==> KO"

View File

@ -1,5 +1,2 @@
mots occurence moyenne id_formation source_field mots occurence moyenne id_formation source_field
0 non-financi 1 0.25 25230 title 0 mot1 1 1.0 code01 keyword
1 1 1 0.25 25230 title
2 financ 1 0.25 25230 title
3 niveau 1 0.25 25230 title

15
main.py
View File

@ -606,6 +606,21 @@ def ela_index_all_classes_obj():
return jsonify(status=status, message=retval) return jsonify(status=status, message=retval)
'''
Cette API index la base donnée en se basant sur les mots clées (keyword).
La fonction "ela_index_all_classes" va aller chercher chaque cours dans la BDD
et créer les mots clées qui sont ensuite enregister dans la table "elaindex"
'''
@app.route('/myclass/api/ela_index_all_classes_keywords/', methods=['POST','GET'])
@crossdomain(origin='*')
def ela_index_all_classes_keywords():
# On recupere le corps (payload) de la requete
payload = request.form.to_dict()
print(" ### payload = ",payload)
status, retval = ela_index_class.ela_index_all_classes_keywords()
return jsonify(status=status, message=retval)

View File

@ -188,7 +188,8 @@ def get_all_class(diction):
#mycommon.myprint(" insertObject = ", insertObject) #mycommon.myprint(" insertObject = ", insertObject)
return True, insertObject return True, insertObject
except Exception as e: except Exception as e:
mycommon.myprint(e) exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, " Impossible de recuperer les formations" return False, " Impossible de recuperer les formations"