diff --git a/Ela_Spacy.py b/Ela_Spacy.py index cd98ebd..17fcf50 100644 --- a/Ela_Spacy.py +++ b/Ela_Spacy.py @@ -241,7 +241,6 @@ def ela_index_record_field(lines, class_id, source_field = ""): if len(str(source_field)) == 0 : source_field = 'default' - myquery = {"id_formation": class_id, "source_field":source_field} delete_row = coll_name.delete_many(myquery) mycommon.myprint(" elaindex - "+str(delete_row.deleted_count)+" documents deleted. Training ==> "+str(class_id)+" ") diff --git a/class_mgt.py b/class_mgt.py index 596b241..c00c39f 100644 --- a/class_mgt.py +++ b/class_mgt.py @@ -27,8 +27,7 @@ client = MongoClient(CONNECTION_STRING) dbname = client['cherifdb'] - - +MAX_KEYWORD = 3 ''' Cette fonction ajoute une formation @@ -152,6 +151,15 @@ def add_class(diction): if ("mots_cle" in diction.keys()): if diction['mots_cle']: mydata['mots_cle'] = diction['mots_cle'] + ''' + Verification du nombre de mots clée : limite MAX_KEYWORD (3) + ''' + nb_keyword = mydata['mots_cle'].split(";") + if( len(nb_keyword) > MAX_KEYWORD ): + mycommon.myprint( + str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés") + return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés" + if ("domaine" in diction.keys()): @@ -1164,6 +1172,16 @@ def add_class_mass(file=None, Folder=None, diction=None): mydata['duree_formation'] = str(df['duree_formation'].values[n]) mydata['plus_produit'] = str(df['plus_produit'].values[n]) mydata['mots_cle'] = str(df['mots_cle'].values[n]) + ''' + Verification du nombre de mots clée : limite MAX_KEYWORD (3) + ''' + nb_keyword = mydata['mots_cle'].split(";") + if( len(nb_keyword) > MAX_KEYWORD ): + mycommon.myprint( + str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés") + return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés" + + mydata['distantiel'] = str(df['distantiel'].values[n]) mydata['presentiel'] = str(df['presentiel'].values[n]) diff --git a/data_indexees.csv b/data_indexees.csv index d503337..d146f3c 100644 --- a/data_indexees.csv +++ b/data_indexees.csv @@ -1,5 +1,2 @@ ,index,mots,occurence,moyenne,id_formation,source_field -0,0,non-financi,1,0.25,25230,title -1,1,1,1,0.25,25230,title -2,2,financ,1,0.25,25230,title -3,3,niveau,1,0.25,25230,title +0,0,mot1,1,1.0,code01,keyword diff --git a/ela_index_bdd_classes.py b/ela_index_bdd_classes.py index 53dbdcb..e387cc5 100644 --- a/ela_index_bdd_classes.py +++ b/ela_index_bdd_classes.py @@ -2,7 +2,9 @@ from pymongo import MongoClient import Ela_Spacy as ls import prj_common as mycommon +from pymongo import ReturnDocument import inspect +import sys, os client = MongoClient("mongodb://localhost:27017/") # database @@ -24,8 +26,9 @@ def ela_index_class(external_code="", source_field = ""): if( len(str(source_field)) == 0): source_field = 'default' - for doc in collection.find({"external_code":external_code}): + + # Indexation par defaut : champs TITRE + DESCRIPTION if ( str(source_field) == "default" ): chaine = doc["title"]+" - "+doc["description"] #mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine)) @@ -50,6 +53,7 @@ def ela_index_class(external_code="", source_field = ""): else: return False + # Indexation du champ : title elif ( str(source_field) == "title" ): chaine = doc["title"] # mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine)) @@ -74,6 +78,7 @@ def ela_index_class(external_code="", source_field = ""): else: return False + # Indexation du champ : description elif ( str(source_field) == "description" ): chaine = doc["description"] # mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine)) @@ -98,6 +103,8 @@ def ela_index_class(external_code="", source_field = ""): else: return False + + # Indexation du champ : Objectif elif ( str(source_field) == "objectif" ): chaine = doc["objectif"] # mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine)) @@ -123,6 +130,32 @@ def ela_index_class(external_code="", source_field = ""): return False + # Indexation du champ : mot_cle + elif (str(source_field) == "keyword"): + chaine = doc["mots_cle"] + # mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine)) + retval = ls.ela_index_record_field(chaine, str(doc["external_code"]), source_field) + + ''' + # Apres indexation, on met a jour la formation pour + # mettre le champ "indexe" a 1. + ''' + if (retval): + # mise à jour du champ "indexed" + myobjectif = {} + myobjectif['indexed_keyword'] = '1' + + ret_val2 = collection.find_and_modify(query={"external_code": external_code}, + update={"$set": myobjectif} + ) + + if (ret_val2 and ret_val2['_id']): + mycommon.myprint("La formation indexée (champs objectif) =" + str(ret_val2['_id'])) + return True + else: + return False + + return True except Exception as e: mycommon.myprint(e) @@ -179,20 +212,48 @@ def ela_index_all_classes(): mycommon.myprint(e) return False, "Impossible d'indexer la BDD des formations ==> KO" +''' +Cette fonction va indexer les mots clés ajoutés à une formation. +Par defaut, seul 3 mots clés sont acceptés par formation. +les mots clés sont separés par des ";". +ex : mot1;mot2;mot3 +''' +def ela_index_all_classes_keywords(): + try: + collection = db["myclass"] + nb_class_indexed = 0 + for doc in collection.find({'indexed_keyword': '0'}): + if ("mots_cle" in doc.keys()): + chaine = doc["mots_cle"] + mycommon.myprint(" external_code = " + str(doc["external_code"]) + " ==> " + str(chaine)) + retval = ela_index_class(str(doc["external_code"]), "keyword") + print(" apres : ela_index_all_classes_title retval = " + str(retval)) + + if (retval is False): + mycommon.myprint( + str(inspect.stack()[0][3]) + "IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"])) + else: + nb_class_indexed = nb_class_indexed + 1 + + # retval = ls.ela_index_record_field(chaine, str(doc["external_code"])) + + mycommon.myprint(str(nb_class_indexed) + " ont été indexes") + return True, str(nb_class_indexed) + " ont été indexes " + + return True + except Exception as e: + exc_type, exc_obj, exc_tb = sys.exc_info() + mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) + return False, "Impossible d'indexer la BDD des formations ==> KO" ''' La fonction "ela_index_all_classes_title" va aller chercher chaque cours dans la BDD et créer les mots clées des title qui sont ensuite enregister dans la table "elaindex" ''' - - def ela_index_all_classes_title(): # Making a Connection with MongoClient try: - client = MongoClient("mongodb://localhost:27017/") - # database - db = client["cherifdb"] - # collection + collection = db["myclass"] ''' ATTENTION ATTENTION @@ -220,7 +281,7 @@ def ela_index_all_classes_title(): print(" apres : ela_index_all_classes_title retval = " + str(retval)) if (retval is False): - mycommon.myprint("IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"])) + mycommon.myprint(str(inspect.stack()[0][3]) + "IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"])) else: nb_class_indexed = nb_class_indexed + 1 @@ -230,7 +291,8 @@ def ela_index_all_classes_title(): return True, str(nb_class_indexed) + " ont été indexes " except Exception as e: - mycommon.myprint(e) + exc_type, exc_obj, exc_tb = sys.exc_info() + mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) return False, "Impossible d'indexer la BDD des formations ==> KO" diff --git a/ela_output_test_file_pandas_2.txt b/ela_output_test_file_pandas_2.txt index a75229b..16b4647 100644 --- a/ela_output_test_file_pandas_2.txt +++ b/ela_output_test_file_pandas_2.txt @@ -1,5 +1,2 @@ - mots occurence moyenne id_formation source_field -0 non-financi 1 0.25 25230 title -1 1 1 0.25 25230 title -2 financ 1 0.25 25230 title -3 niveau 1 0.25 25230 title \ No newline at end of file + mots occurence moyenne id_formation source_field +0 mot1 1 1.0 code01 keyword \ No newline at end of file diff --git a/main.py b/main.py index 2291bf1..264bf69 100644 --- a/main.py +++ b/main.py @@ -606,6 +606,21 @@ def ela_index_all_classes_obj(): return jsonify(status=status, message=retval) +''' +Cette API index la base donnée en se basant sur les mots clées (keyword). +La fonction "ela_index_all_classes" va aller chercher chaque cours dans la BDD +et créer les mots clées qui sont ensuite enregister dans la table "elaindex" + +''' +@app.route('/myclass/api/ela_index_all_classes_keywords/', methods=['POST','GET']) +@crossdomain(origin='*') +def ela_index_all_classes_keywords(): + # On recupere le corps (payload) de la requete + payload = request.form.to_dict() + print(" ### payload = ",payload) + status, retval = ela_index_class.ela_index_all_classes_keywords() + return jsonify(status=status, message=retval) + diff --git a/wrapper.py b/wrapper.py index 3c4539e..3b12af5 100644 --- a/wrapper.py +++ b/wrapper.py @@ -188,7 +188,8 @@ def get_all_class(diction): #mycommon.myprint(" insertObject = ", insertObject) return True, insertObject except Exception as e: - mycommon.myprint(e) + exc_type, exc_obj, exc_tb = sys.exc_info() + mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) return False, " Impossible de recuperer les formations"