From f982466549712180834357ac7f83843173401153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ch=C3=A9rifBALDE?= Date: Fri, 27 May 2022 16:05:09 +0200 Subject: [PATCH] 27/05/22 - 16h30 --- Ela_Spacy.py | 2 +- GlobalVariable.py | 2 +- articles_avis.py | 34 +++++++---- class_mgt.py | 53 +++++++++-------- data_indexees.csv | 22 ++++---- ela_output_test_file_pandas_2.txt | 22 ++++---- main.py | 20 ++++++- prj_common.py | 94 +++++++++++++++++++++++++++++-- 8 files changed, 183 insertions(+), 66 deletions(-) diff --git a/Ela_Spacy.py b/Ela_Spacy.py index caae5e6..5599e67 100644 --- a/Ela_Spacy.py +++ b/Ela_Spacy.py @@ -188,7 +188,7 @@ def Ela_Remove_Ponct_Special_Caractere(sentence): text = re.sub(r"\-", " ", text) list_noises = ['...', '.', ';', ',', ':', '!', '?', ')', '(', '[', ']', '\'', '"', '’', '`','©', '–', - '{', '}', '-', '=', '°', '#', '-', '/', '~', '&', '\\', '.', '^', '$', '*', '+','\\n', + '{', '}', '-', '=', '°', '#', '-', '/', '~', '&', '\\', '.', '^', '$', '*', '+','\\n','\n' '?', '{', '}', '[', ']', '|', '(', ')', '-', '>', '<', '@','®', '™', '«', '»'] sentence = text diff --git a/GlobalVariable.py b/GlobalVariable.py index 6f83363..ef66cfb 100644 --- a/GlobalVariable.py +++ b/GlobalVariable.py @@ -97,4 +97,4 @@ QUERY_LIMIT_ROW = 50 ''' Cette varibal definit le nombre max retour à faire sur page d'acceuil ''' -MAINPAGE_QUERY_LIMIT_ROW = 100 \ No newline at end of file +MAINPAGE_QUERY_LIMIT_ROW = 50 \ No newline at end of file diff --git a/articles_avis.py b/articles_avis.py index db16c30..fc99afa 100644 --- a/articles_avis.py +++ b/articles_avis.py @@ -5,9 +5,10 @@ Cette class gere les avis et articles sur une formation. ''' from pymongo import MongoClient import pymongo - +import hashlib import json from bson import ObjectId +from unidecode import unidecode import ela_index_bdd_classes as ela_index import logging @@ -19,6 +20,7 @@ import inspect import sys, os import GlobalVariable as MYSY_GV import user_message_mgt as umm +import Ela_Spacy as ES @@ -105,7 +107,7 @@ def get_articles_avis(diction): try: # Dictionnaire des champs utilisables - field_list = ['_id', 'token', 'user_ip', 'user_country_code', 'user_country_name', + field_list = ['internal_url', 'token', 'user_ip', 'user_country_code', 'user_country_name', 'user_city','user_postal', 'user_latitude', 'user_longitude', 'user_state'] incom_keys = diction.keys() @@ -125,7 +127,7 @@ def get_articles_avis(diction): Une fois qu'on a controlé que toutes les clés mise dans l'API sont correcte. etape precedente, On controle que les champs obligatoires sont presents dans la liste ''' - field_list_obligatoire = ['_id', 'token'] + field_list_obligatoire = ['internal_url', 'token'] for val in field_list_obligatoire: if val not in diction: @@ -175,17 +177,17 @@ def get_articles_avis(diction): my_title_formation = diction['title_formation'] my_internal_id = '' - if ("_id" in diction.keys()): - if diction['_id']: - my_internal_id = diction['_id'] + if ("internal_url" in diction.keys()): + if diction['internal_url']: + my_internal_id = diction['internal_url'] - print(" My Data =" + str(mydata)) + #print(" My Data =" + str(mydata)) coll_name = dbname['articles_avis'] insertObject = [] - for x in coll_name.find({'valide': '1', 'locked': '0', '_id':ObjectId(my_internal_id)}, + for x in coll_name.find({'valide': '1', 'locked': '0', 'internal_url':my_internal_id}, { "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0, "valide": 0, "locked": 0, }).sort([("title_formation",pymongo.ASCENDING), ("date_avis",pymongo.ASCENDING), @@ -290,6 +292,17 @@ def add_articles_avis(diction): if ("title_formation" in diction.keys()): if diction['title_formation']: mydata['title_formation'] = diction['title_formation'] + ''' + Gestion de l'url interne. Pr un meilleur SEO, le champ 'internal_url' = titre avec des tiret + ''' + my_internal_url = str(diction['title_formation']).replace("-","").replace(",", "").replace(" ", "").strip() + + local_status, my_internal_url = ES.Ela_Remove_Ponct_Special_Caractere(my_internal_url) + my_internal_url = unidecode(my_internal_url.lower()) + my_internal_url = my_internal_url.replace(" ","-") + suffix = hashlib.md5(my_internal_url.encode()).hexdigest() + mydata['internal_url'] = str(my_internal_url)+"-"+str(suffix[-3:]) + if ("user_ip" in diction.keys()): if diction['user_ip']: @@ -364,6 +377,7 @@ def add_articles_avis(diction): mydata['indexed'] = '0' mydata['indexed'] = '0' mydata['date_update'] = str(datetime.now()) + mydata['date_avis'] = str(datetime.now()) mydata['isalaune'] = "0" print(mydata) @@ -711,7 +725,7 @@ def get_article_avis_alaune(diction): for x in coll_name.find({'valide': '1', 'locked': '0', 'isalaune': '1'}, {"indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "isalaune": 0, "valide": 0, "locked": 0, }).sort( - [ ("note", pymongo.DESCENDING), ("date_avis", pymongo.DESCENDING),]): + [ ("date_avis", pymongo.DESCENDING), ("note", pymongo.DESCENDING)]): val = x['qualite'] if (len(x['qualite']) > MYSY_GV.MAX_CARACT_DEDUIT): @@ -730,7 +744,7 @@ def get_article_avis_alaune(diction): user = x insertObject.append(JSONEncoder().encode(user)) - #print(" getObject = ", str(insertObject)) + print(" getObject = ", str(insertObject)) return True, insertObject diff --git a/class_mgt.py b/class_mgt.py index ee19451..09bfe54 100644 --- a/class_mgt.py +++ b/class_mgt.py @@ -784,7 +784,7 @@ def get_class(diction): # Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste # field_list. ''' - field_list = ['external_code', 'token', 'title', 'valide', 'locked', 'user_ip', 'user_country_code', 'user_country_name', 'user_city', + field_list = ['internal_url', 'token', 'title', 'valide', 'locked', 'user_ip', 'user_country_code', 'user_country_name', 'user_city', 'user_postal', 'user_latitude', 'user_longitude', 'user_state'] incom_keys = diction.keys() @@ -806,13 +806,13 @@ def get_class(diction): # recuperation des paramettre mydata = {} - my_external_code = "" + my_internal_url = "" my_token = "" - if ("external_code" in diction.keys()): - if diction['external_code']: - my_external_code = diction['external_code'] + if ("internal_url" in diction.keys()): + if diction['internal_url']: + my_internal_url = diction['internal_url'] if ("token" in diction.keys()): if diction['token']: @@ -823,10 +823,10 @@ def get_class(diction): Gestion des filters. ''' - external_code_crit = {} - if ("external_code" in diction.keys()): - if diction['external_code']: - external_code_crit['external_code'] = diction['external_code'] + internal_url_crit = {} + if ("internal_url" in diction.keys()): + if diction['internal_url']: + internal_url_crit['internal_url'] = diction['internal_url'] title_crit = {} if ("title" in diction.keys()): @@ -866,9 +866,10 @@ def get_class(diction): RetObject = [] filt_external_code = {} - if ("external_code" in diction.keys()): - filt_external_code = {'external_code':str(diction['external_code'])} - external_code = str(diction['external_code']) + internal_url = "" + if ("internal_url" in diction.keys()): + filt_external_code = {'internal_url':str(diction['internal_url'])} + internal_url = str(diction['internal_url']) #print(' ICICICIC '+str(filt_external_code)) @@ -878,10 +879,10 @@ def get_class(diction): print(" #### avant requete get partner_owner_recid ="+str(user_recid)+ - " filt_external_code = "+str(filt_external_code)+ + " internal_url = "+str(my_internal_url)+ " filt_title = "+str(filt_title)) - for retVal in coll_name.find({'valide':'1','locked':'0','external_code':external_code}, + for retVal in coll_name.find({'valide':'1','locked':'0','internal_url':internal_url}, {"_id": 0, "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0, "valide": 0, "locked": 0, "partner_owner_recid": 0, } ): @@ -931,7 +932,7 @@ def get_class(diction): retVal['pedagogie'] = tmp_str[:MYSY_GV.MAX_CARACT_DETAIL] + " ..." - mycommon.myprint(str(retVal)) + #mycommon.myprint(str(retVal)) user = retVal RetObject.append(JSONEncoder().encode(user)) @@ -960,7 +961,7 @@ def get_partner_class(diction): # Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste # field_list. ''' - field_list = ['external_code', 'token', 'title', 'valide', 'locked'] + field_list = ['internal_url', 'token', 'title', 'valide', 'locked'] incom_keys = diction.keys() for val in incom_keys: if val not in field_list: @@ -984,9 +985,9 @@ def get_partner_class(diction): my_token = "" - if ("external_code" in diction.keys()): - if diction['external_code']: - my_external_code = diction['external_code'] + if ("internal_url" in diction.keys()): + if diction['internal_url']: + my_internal_code = diction['internal_url'] if ("token" in diction.keys()): if diction['token']: @@ -997,9 +998,9 @@ def get_partner_class(diction): ''' external_code_crit = {} - if ("external_code" in diction.keys()): - if diction['external_code']: - external_code_crit['external_code'] = diction['external_code'] + if ("internal_url" in diction.keys()): + if diction['internal_url']: + external_code_crit['internal_url'] = diction['internal_url'] title_crit = {} if ("title" in diction.keys()): @@ -1043,7 +1044,7 @@ def get_partner_class(diction): RetObject = [] filt_external_code = {} if ("external_code" in diction.keys()): - filt_external_code = {'external_code':{'$regex':str(diction['external_code'])}} + filt_external_code = {'internal_url':{'$regex':str(diction['internal_url'])}} filt_title = {} if ("title" in diction.keys()): @@ -1056,8 +1057,10 @@ def get_partner_class(diction): " filt_title = "+str(filt_title)) val_tmp = 1 - for retVal in coll_name.find(({"$and": [ {'valide':'1'},{'locked':'0'}, - {'partner_owner_recid':user_recid} ,filt_external_code, filt_title ]})): + for retVal in coll_name.find( {'valide':'1'},{'locked':'0'}, + {'partner_owner_recid':user_recid} , + filt_external_code, + filt_title): mycommon.myprint(str(retVal)) user = retVal user['id'] = str(val_tmp) diff --git a/data_indexees.csv b/data_indexees.csv index 829cf73..7a99374 100644 --- a/data_indexees.csv +++ b/data_indexees.csv @@ -1,34 +1,34 @@ ,index,mots,occurence,moyenne,id_formation,source_field -0,0,infrastructure,2,0.12," +0,0,ressource,1,0.06," concevoir-une-infrastructure-de-reseau-dans-azure -4",default -1,1,concevoir,2,0.12," +1,1,locau,1,0.06," concevoir-une-infrastructure-de-reseau-dans-azure -4",default -2,2,decouvrez,1,0.06," +2,2,azure,3,0.18," concevoir-une-infrastructure-de-reseau-dans-azure -4",default -3,3,ressource,1,0.06," +3,3,comment,1,0.06," concevoir-une-infrastructure-de-reseau-dans-azure -4",default -4,4,reseau,3,0.18," +4,4,decouvrez,1,0.06," concevoir-une-infrastructure-de-reseau-dans-azure -4",default -5,5,connecter,1,0.06," +5,5,securisee,1,0.06," concevoir-une-infrastructure-de-reseau-dans-azure -4",default -6,6,azure,3,0.18," +6,6,connecter,1,0.06," concevoir-une-infrastructure-de-reseau-dans-azure -4",default -7,7,flexible,1,0.06," +7,7,concevoir,2,0.12," concevoir-une-infrastructure-de-reseau-dans-azure -4",default -8,8,comment,1,0.06," +8,8,flexible,1,0.06," concevoir-une-infrastructure-de-reseau-dans-azure -4",default -9,9,locau,1,0.06," +9,9,reseau,3,0.18," concevoir-une-infrastructure-de-reseau-dans-azure -4",default -10,10,securisee,1,0.06," +10,10,infrastructure,2,0.12," concevoir-une-infrastructure-de-reseau-dans-azure -4",default diff --git a/ela_output_test_file_pandas_2.txt b/ela_output_test_file_pandas_2.txt index 8fc1d74..d27641a 100644 --- a/ela_output_test_file_pandas_2.txt +++ b/ela_output_test_file_pandas_2.txt @@ -1,14 +1,14 @@ mots ... source_field -0 infrastructure ... default -1 concevoir ... default -2 decouvrez ... default -3 ressource ... default -4 reseau ... default -5 connecter ... default -6 azure ... default -7 flexible ... default -8 comment ... default -9 locau ... default -10 securisee ... default +0 ressource ... default +1 locau ... default +2 azure ... default +3 comment ... default +4 decouvrez ... default +5 securisee ... default +6 connecter ... default +7 concevoir ... default +8 flexible ... default +9 reseau ... default +10 infrastructure ... default [11 rows x 5 columns] \ No newline at end of file diff --git a/main.py b/main.py index a8039c8..b4a50f6 100644 --- a/main.py +++ b/main.py @@ -1036,15 +1036,29 @@ def OrderClassRandomly(): return jsonify(status=status, message=result) -@app.route('/myclass/api/testch/', methods=['GET','POST']) +@app.route('/myclass/api/Migration_internal_url/', methods=['GET','POST']) @crossdomain(origin='*') -def testch(): +def Migration_internal_url(): # On recupere le corps (payload) de la requete payload = request.form.to_dict() print(" ### payload = ", payload) - status, result = mycommon.FormationAlaUne() + status, result = mycommon.Migration_internal_url() return jsonify(status=status, message=result) + +''' +Cette API va aller chercher dans la collection "elaindex" +les mots qui contiennent le mot "mot"''' +@app.route('/myclass/api/GetMotFromElaIndex/', methods=['GET','POST']) +@crossdomain(origin='*') +def GetMotFromElaIndex(): + # On recupere le corps (payload) de la requete + payload = request.form.to_dict() + print(" ### payload = ", payload) + status, result = mycommon.GetMotFromElaIndex(payload) + return jsonify(status=status, message=result) + + if __name__ == '__main__': print(" debut api") diff --git a/prj_common.py b/prj_common.py index 560f26b..affad8b 100644 --- a/prj_common.py +++ b/prj_common.py @@ -1,4 +1,5 @@ - +from pymongo import MongoClient +import pymongo from difflib import SequenceMatcher import textdistance from datetime import datetime @@ -845,11 +846,96 @@ def FormationAlaUne(): return_document=ReturnDocument.AFTER ) - - - return True, "ok" except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() print(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) return False, "KO" + + +def Migration_internal_url(): + try: + coll_name = MYSY_GV.dbname['myclass'] + for retVal in coll_name.find({}): + user = retVal + + + result = coll_name.update_many( + {'_id':ObjectId(str(user['_id']))}, + { + "$set": {"internal_url": str(user['_id'])} + }) + '''print("raw:", result.raw_result) + print("acknowledged:", result.acknowledged) + print("matched_count:", result.matched_count)''' + + return True, "ok" + + except Exception as e: + exc_type, exc_obj, exc_tb = sys.exc_info() + print(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) + return False, "KO" + + +''' +Cette fonction prends un mot de X digit +et retourne le top X des mot ayant le meme digit +/!\ cette fonction est glement utilisee dans le car des +des recherches vides +''' +def GetMotFromElaIndex(diction): + try : + + ''' + # Verification que les champs reçus dans l'API sont bien dans la liste des champs autorisés + # Cela evite le cas ou une entité tierce ajouter les valeurs inconnu dans l'API + # Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste + # field_list. + ''' + field_list = ['mot', 'domaine'] + + incom_keys = diction.keys() + for val in incom_keys: + if val not in field_list: + mycommon.myprint(str(inspect.stack()[0][ + 3]) + " - Le champ '" + val + "' n'existe pas, Creation formation annulée") + return False, [] + + ''' + Une fois qu'on a controlé que toutes les clés mise dans l'API sont correcte. etape precedente, + On controle que les champs obligatoires sont presents dans la liste + ''' + field_list_obligatoire = ['mot'] + + for val in field_list_obligatoire: + if val not in diction: + mycommon.myprint( + str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ") + return False, [] + + + coll_name = MYSY_GV.dbname['elaindex'] + training_mots = [] + + mot = "" + if ("mot" in diction.keys()): + if diction['mot']: + mot = diction['mot'] + + if( len(mot) < 3 ): + return True, [] + + search = str(mot) + search_expr = re.compile(f".*{search}.*", re.I) + + print(" #### mot recu "+mot+" search_expr = "+str(search_expr)) + for x in coll_name.find({'mots': {'$regex': search_expr}}).sort([("occurence", pymongo.DESCENDING)]).limit(5): + if( x['mots'] not in training_mots ): + training_mots.append(x['mots']) + + return True, training_mots + + except Exception as e: + exc_type, exc_obj, exc_tb = sys.exc_info() + print(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) + return False, [] \ No newline at end of file