Validation 13/03/22 - 12h21

master
ChérifBALDE 2022-03-13 13:41:42 +01:00 committed by cherif
parent e9009638ea
commit e3cc35a035
7 changed files with 111 additions and 22 deletions

View File

@ -241,7 +241,6 @@ def ela_index_record_field(lines, class_id, source_field = ""):
if len(str(source_field)) == 0 :
source_field = 'default'
myquery = {"id_formation": class_id, "source_field":source_field}
delete_row = coll_name.delete_many(myquery)
mycommon.myprint(" elaindex - "+str(delete_row.deleted_count)+" documents deleted. Training ==> "+str(class_id)+" ")

View File

@ -27,8 +27,7 @@ client = MongoClient(CONNECTION_STRING)
dbname = client['cherifdb']
MAX_KEYWORD = 3
'''
Cette fonction ajoute une formation
@ -152,6 +151,15 @@ def add_class(diction):
if ("mots_cle" in diction.keys()):
if diction['mots_cle']:
mydata['mots_cle'] = diction['mots_cle']
'''
Verification du nombre de mots clée : limite MAX_KEYWORD (3)
'''
nb_keyword = mydata['mots_cle'].split(";")
if( len(nb_keyword) > MAX_KEYWORD ):
mycommon.myprint(
str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés")
return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés"
if ("domaine" in diction.keys()):
@ -1164,6 +1172,16 @@ def add_class_mass(file=None, Folder=None, diction=None):
mydata['duree_formation'] = str(df['duree_formation'].values[n])
mydata['plus_produit'] = str(df['plus_produit'].values[n])
mydata['mots_cle'] = str(df['mots_cle'].values[n])
'''
Verification du nombre de mots clée : limite MAX_KEYWORD (3)
'''
nb_keyword = mydata['mots_cle'].split(";")
if( len(nb_keyword) > MAX_KEYWORD ):
mycommon.myprint(
str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés")
return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés"
mydata['distantiel'] = str(df['distantiel'].values[n])
mydata['presentiel'] = str(df['presentiel'].values[n])

View File

@ -1,5 +1,2 @@
,index,mots,occurence,moyenne,id_formation,source_field
0,0,non-financi,1,0.25,25230,title
1,1,1,1,0.25,25230,title
2,2,financ,1,0.25,25230,title
3,3,niveau,1,0.25,25230,title
0,0,mot1,1,1.0,code01,keyword

1 index mots occurence moyenne id_formation source_field
2 0 0 non-financi mot1 1 0.25 1.0 25230 code01 title keyword
1 1 1 1 0.25 25230 title
2 2 financ 1 0.25 25230 title
3 3 niveau 1 0.25 25230 title

View File

@ -2,7 +2,9 @@
from pymongo import MongoClient
import Ela_Spacy as ls
import prj_common as mycommon
from pymongo import ReturnDocument
import inspect
import sys, os
client = MongoClient("mongodb://localhost:27017/")
# database
@ -24,8 +26,9 @@ def ela_index_class(external_code="", source_field = ""):
if( len(str(source_field)) == 0):
source_field = 'default'
for doc in collection.find({"external_code":external_code}):
# Indexation par defaut : champs TITRE + DESCRIPTION
if ( str(source_field) == "default" ):
chaine = doc["title"]+" - "+doc["description"]
#mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
@ -50,6 +53,7 @@ def ela_index_class(external_code="", source_field = ""):
else:
return False
# Indexation du champ : title
elif ( str(source_field) == "title" ):
chaine = doc["title"]
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
@ -74,6 +78,7 @@ def ela_index_class(external_code="", source_field = ""):
else:
return False
# Indexation du champ : description
elif ( str(source_field) == "description" ):
chaine = doc["description"]
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
@ -98,6 +103,8 @@ def ela_index_class(external_code="", source_field = ""):
else:
return False
# Indexation du champ : Objectif
elif ( str(source_field) == "objectif" ):
chaine = doc["objectif"]
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
@ -123,6 +130,32 @@ def ela_index_class(external_code="", source_field = ""):
return False
# Indexation du champ : mot_cle
elif (str(source_field) == "keyword"):
chaine = doc["mots_cle"]
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
retval = ls.ela_index_record_field(chaine, str(doc["external_code"]), source_field)
'''
# Apres indexation, on met a jour la formation pour
# mettre le champ "indexe" a 1.
'''
if (retval):
# mise à jour du champ "indexed"
myobjectif = {}
myobjectif['indexed_keyword'] = '1'
ret_val2 = collection.find_and_modify(query={"external_code": external_code},
update={"$set": myobjectif}
)
if (ret_val2 and ret_val2['_id']):
mycommon.myprint("La formation indexée (champs objectif) =" + str(ret_val2['_id']))
return True
else:
return False
return True
except Exception as e:
mycommon.myprint(e)
@ -179,20 +212,48 @@ def ela_index_all_classes():
mycommon.myprint(e)
return False, "Impossible d'indexer la BDD des formations ==> KO"
'''
Cette fonction va indexer les mots clés ajoutés à une formation.
Par defaut, seul 3 mots clés sont acceptés par formation.
les mots clés sont separés par des ";".
ex : mot1;mot2;mot3
'''
def ela_index_all_classes_keywords():
try:
collection = db["myclass"]
nb_class_indexed = 0
for doc in collection.find({'indexed_keyword': '0'}):
if ("mots_cle" in doc.keys()):
chaine = doc["mots_cle"]
mycommon.myprint(" external_code = " + str(doc["external_code"]) + " ==> " + str(chaine))
retval = ela_index_class(str(doc["external_code"]), "keyword")
print(" apres : ela_index_all_classes_title retval = " + str(retval))
if (retval is False):
mycommon.myprint(
str(inspect.stack()[0][3]) + "IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"]))
else:
nb_class_indexed = nb_class_indexed + 1
# retval = ls.ela_index_record_field(chaine, str(doc["external_code"]))
mycommon.myprint(str(nb_class_indexed) + " ont été indexes")
return True, str(nb_class_indexed) + " ont été indexes "
return True
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "Impossible d'indexer la BDD des formations ==> KO"
'''
La fonction "ela_index_all_classes_title" va aller chercher chaque cours dans la BDD
et créer les mots clées des title qui sont ensuite enregister dans la table "elaindex"
'''
def ela_index_all_classes_title():
# Making a Connection with MongoClient
try:
client = MongoClient("mongodb://localhost:27017/")
# database
db = client["cherifdb"]
# collection
collection = db["myclass"]
'''
ATTENTION ATTENTION
@ -220,7 +281,7 @@ def ela_index_all_classes_title():
print(" apres : ela_index_all_classes_title retval = " + str(retval))
if (retval is False):
mycommon.myprint("IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"]))
mycommon.myprint(str(inspect.stack()[0][3]) + "IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"]))
else:
nb_class_indexed = nb_class_indexed + 1
@ -230,7 +291,8 @@ def ela_index_all_classes_title():
return True, str(nb_class_indexed) + " ont été indexes "
except Exception as e:
mycommon.myprint(e)
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "Impossible d'indexer la BDD des formations ==> KO"

View File

@ -1,5 +1,2 @@
mots occurence moyenne id_formation source_field
0 non-financi 1 0.25 25230 title
1 1 1 0.25 25230 title
2 financ 1 0.25 25230 title
3 niveau 1 0.25 25230 title
mots occurence moyenne id_formation source_field
0 mot1 1 1.0 code01 keyword

15
main.py
View File

@ -606,6 +606,21 @@ def ela_index_all_classes_obj():
return jsonify(status=status, message=retval)
'''
Cette API index la base donnée en se basant sur les mots clées (keyword).
La fonction "ela_index_all_classes" va aller chercher chaque cours dans la BDD
et créer les mots clées qui sont ensuite enregister dans la table "elaindex"
'''
@app.route('/myclass/api/ela_index_all_classes_keywords/', methods=['POST','GET'])
@crossdomain(origin='*')
def ela_index_all_classes_keywords():
# On recupere le corps (payload) de la requete
payload = request.form.to_dict()
print(" ### payload = ",payload)
status, retval = ela_index_class.ela_index_all_classes_keywords()
return jsonify(status=status, message=retval)

View File

@ -188,7 +188,8 @@ def get_all_class(diction):
#mycommon.myprint(" insertObject = ", insertObject)
return True, insertObject
except Exception as e:
mycommon.myprint(e)
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, " Impossible de recuperer les formations"