From 271279400a496f3dab9dcc48ba18ff2ac905d9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ch=C3=A9rifBALDE?= Date: Sat, 17 Sep 2022 14:22:25 +0200 Subject: [PATCH] 17/09/22 - 14h00 --- GlobalVariable.py | 4 +++ class_mgt.py | 86 +++++++++++++++++++++++++++++++++++------------ prj_common.py | 19 ++++++++++- statistics.py | 50 +++++++++++++-------------- 4 files changed, 111 insertions(+), 48 deletions(-) diff --git a/GlobalVariable.py b/GlobalVariable.py index ec361b3..8c8ebc1 100644 --- a/GlobalVariable.py +++ b/GlobalVariable.py @@ -145,6 +145,10 @@ Taux de TVA TVA_TAUX = 0.2 +""" +Variable defini le mombre max de mots clés dans une formation +""" +MAX_KEYWORD = 3 """ Connexion SFTP diff --git a/class_mgt.py b/class_mgt.py index b4d0945..2bc9c30 100644 --- a/class_mgt.py +++ b/class_mgt.py @@ -29,9 +29,6 @@ class JSONEncoder(json.JSONEncoder): - -MAX_KEYWORD = 3 - ''' Cette fonction ajoute une formation elle verifie le token de l'entité qui ajoute la formation. @@ -47,7 +44,8 @@ def add_class(diction): ''' field_list = ['external_code', 'title', 'description', 'trainer', 'institut_formation', 'distantiel', 'presentiel', 'price', 'url','duration', 'duration_unit', 'token', 'plus_produit', 'mots_cle','domaine', - 'internal_url', 'zone_diffusion', 'metier', 'date_lieu', 'published'] + 'internal_url', 'zone_diffusion', 'metier', 'date_lieu', 'published', 'img_url'] + incom_keys = diction.keys() for val in incom_keys: if val not in field_list: @@ -201,13 +199,13 @@ def add_class(diction): if diction['mots_cle']: mydata['mots_cle'] = diction['mots_cle'] ''' - Verification du nombre de mots clée : limite MAX_KEYWORD (3) + Verification du nombre de mots clée : limite MYSY_GV.MAX_KEYWORD (3) ''' nb_keyword = mydata['mots_cle'].split(";") - if( len(nb_keyword) > MAX_KEYWORD ): + if( len(nb_keyword) > MYSY_GV.MAX_KEYWORD ): mycommon.myprint( - str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés") - return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés" + str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MYSY_GV.MAX_KEYWORD)+" mots clés") + return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MYSY_GV.MAX_KEYWORD)+" mots clés" @@ -215,6 +213,19 @@ def add_class(diction): if diction['domaine']: mydata['domaine'] = diction['domaine'] + # Traitement de l'url imag + if ("img_url" in diction.keys()): + if diction['img_url']: + # Verifier si l'image existe + status_tmp, img = mycommon.TryUrlImage(str(diction['img_url'])) + if( status_tmp is False): + mycommon.myprint( + str(inspect.stack()[0][3]) + " : l'url de l'image de la formation " + str( + mydata['external_code']) + " est incorrecte ") + return False, " l'url de l'image de la formation " + str(mydata['external_code']) + " est incorrecte " + + mydata['img_url'] = diction['img_url'] + if ("duration_unit" in diction.keys()): if diction['duration_unit']: @@ -242,7 +253,7 @@ def add_class(diction): tab_country = [] tab_city = [] for val in tmp_str: - print(" val = "+str(val)) + #print(" val = "+str(val)) if( len(str(val)) <= 0 ): continue @@ -392,7 +403,7 @@ def update_class(diction): field_list = ['external_code', 'title', 'description', 'trainer', 'institut_formation', 'distantiel', 'presentiel','price', 'url', 'duration', 'token','plus_produit', 'mots_cle', 'domaine', 'internal_code', 'internal_url','zone_diffusion', 'metier', - 'date_lieu', 'published'] + 'date_lieu', 'published', 'img_url'] incom_keys = diction.keys() for val in incom_keys: @@ -457,6 +468,19 @@ def update_class(diction): if ("title" in diction.keys()): mydata['title'] = diction['title'] + if ("img_url" in diction.keys()): + if diction['img_url']: + # Verifier si l'image existe + status_tmp, img = mycommon.TryUrlImage(str(diction['img_url'])) + if (status_tmp is False): + mycommon.myprint( + str(inspect.stack()[0][3]) + " : l'url de l'image de la formation " + str(my_internal_code) + " est incorrecte ") + return False, " l'url de l'image de la formation " + str(my_internal_code) + " est incorrecte " + + + mydata['img_url'] = diction['img_url'] + + if ("description" in diction.keys()): mydata['description'] = diction['description'] @@ -525,9 +549,7 @@ def update_class(diction): tab_country = [] tab_city = [] for val in tmp_str: - print(" val = "+str(val)) - print(" val = " + str(val)) if (len(str(val)) <= 0): continue @@ -1963,9 +1985,9 @@ def add_class_mass(file=None, Folder=None, diction=None): ''' # Verification que les noms des colonne sont bien corrects" ''' - field_list = ['external_code', 'title', 'description', 'trainer', 'institut_formation', - 'distantiel', 'presentiel', 'prix', 'domaine', 'url','duration', 'plus_produit', - 'mots_cle', 'zone_diffusion', 'metier', 'date_lieu', 'publie'] + field_list = ['external_code', 'titre', 'description', 'formateur', 'institut_formation', + 'distantiel', 'presentiel', 'prix', 'domaine', 'url','duree', 'plus_produit', + 'mots_cle', 'zone_diffusion', 'metier', 'date_lieu', 'publie', 'img_url'] total_rows = len(df) @@ -2011,14 +2033,14 @@ def add_class_mass(file=None, Folder=None, diction=None): for n in x: mydata = {} mydata['external_code'] = str(df['external_code'].values[n]) - mydata['title'] = str(df['title'].values[n]) + mydata['title'] = str(df['titre'].values[n]) mydata['domaine'] = str(df['domaine'].values[n]) mydata['description'] = str(df['description'].values[n]) - mydata['trainer'] = str(df['trainer'].values[n]) + mydata['trainer'] = str(df['formateur'].values[n]) mydata['institut_formation'] = str(df['institut_formation'].values[n]) #mydata['distantiel'] = str(df['presentiel'].values[n]) mydata['url'] = str(df['url'].values[n]) - mydata['duration'] = float(str(df['duration'].values[n])) + mydata['duration'] = float(str(df['duree'].values[n])) mydata['plus_produit'] = str(df['plus_produit'].values[n]) mydata['mots_cle'] = str(df['mots_cle'].values[n]) mydata['presentiel'] = str(df['presentiel'].values[n]) @@ -2028,13 +2050,33 @@ def add_class_mass(file=None, Folder=None, diction=None): mydata['published'] = str(df['publie'].values[n]) ''' - Verification du nombre de mots clée : limite MAX_KEYWORD (3) + Verification de l'image + ''' + if ("img_url" in df.keys()): + mydata['img_url'] = str(df['img_url'].values[n]) + if (str(df['img_url'].values[n]) == 'nan'): + mydata['img_url'] = "" + + + print(" ### mydata['img_url'] = '"+str(mydata['img_url'])+"' ") + if( len(str(mydata['img_url'])) > 0 ): + # Verifier si l'image existe + status_tmp, img = mycommon.TryUrlImage(str(mydata['img_url'])) + if (status_tmp is False): + mycommon.myprint( + str(inspect.stack()[0][3]) + " : l'url de l'image de la formation " + str( + mydata['external_code']) + " est incorrecte ") + return False, " l'url de l'image de la formation " + str(mydata['external_code']) + " est incorrecte " + + + ''' + Verification du nombre de mots clée : limite MYSY_GV.MAX_KEYWORD (3) ''' nb_keyword = mydata['mots_cle'].split(";") - if( len(nb_keyword) > MAX_KEYWORD ): + if( len(nb_keyword) > MYSY_GV.MAX_KEYWORD ): mycommon.myprint( - str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés") - return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés" + str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MYSY_GV.MAX_KEYWORD)+" mots clés") + return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MYSY_GV.MAX_KEYWORD)+" mots clés" ''' diff --git a/prj_common.py b/prj_common.py index 7fd6586..ccdb6c4 100644 --- a/prj_common.py +++ b/prj_common.py @@ -1,7 +1,8 @@ import hashlib import _pickle as cPickle import pickle - +from PIL import Image +import requests import bson from pymongo import MongoClient import pymongo @@ -1759,3 +1760,19 @@ def removeLSaccount(): exc_type, exc_obj, exc_tb = sys.exc_info() print(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno)) return False + +""" +Cette fonction verifier si une url est une image. +elle permet si besoin de verifier la taille de l'image +Cette fonction sera dans un premier temps utilisée +dans la gestion des images associées à une formation. +""" +def TryUrlImage(url=None): + try: + img = Image.open(requests.get(url, stream=True).raw) + return True, img + + except Exception as e: + exc_type, exc_obj, exc_tb = sys.exc_info() + print(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno)) + return False, False diff --git a/statistics.py b/statistics.py index dba899a..edaef53 100644 --- a/statistics.py +++ b/statistics.py @@ -76,22 +76,22 @@ def GetStat_class_view_old(): if ("Date_view" in result['_id'].keys()): if result['_id']['Date_view']: - print( "Date_view = "+str( result['_id']['Date_view']) ) + tmp_val['Date_view'] = str( result['_id']['Date_view']) if ("OWNER" in result['_id'].keys()): if result['_id']['OWNER']: - print( "Owner = "+str( result['_id']['OWNER']) ) + #print( "Owner = "+str( result['_id']['OWNER']) ) tmp_val['owner'] = str( result['_id']['OWNER']) if ("INTERNAL URL" in result['_id'].keys()): if result['_id']['INTERNAL URL']: - print( "INTERNAL URL = "+str( result['_id']['INTERNAL URL']) ) + #print( "INTERNAL URL = "+str( result['_id']['INTERNAL URL']) ) tmp_val['internal_url'] = str(result['_id']['INTERNAL URL']) - print("Nombre = "+str(result['count'])) + #print("Nombre = "+str(result['count'])) tmp_val['nb_view'] = str(result['count']) insertObject.append(JSONEncoder().encode(tmp_val)) @@ -221,7 +221,7 @@ def GetStat_class_view(diction): } ] - print(" PIP 2 ="+str(pipe2)) + #print(" PIP 2 ="+str(pipe2)) insertObject = [] for result in collection.aggregate(pipe2): @@ -232,20 +232,20 @@ def GetStat_class_view(diction): if ("Date_view" in result['_id'].keys()): if result['_id']['Date_view']: - print("Date_view = " + str(result['_id']['Date_view'])) + #print("Date_view = " + str(result['_id']['Date_view'])) tmp_val['Date_view'] = str(result['_id']['Date_view']) if ("OWNER" in result['_id'].keys()): if result['_id']['OWNER']: - print("Owner = " + str(result['_id']['OWNER'])) + #print("Owner = " + str(result['_id']['OWNER'])) tmp_val['owner'] = str(result['_id']['OWNER']) if ("INTERNAL URL" in result['_id'].keys()): if result['_id']['INTERNAL URL']: - print("INTERNAL URL = " + str(result['_id']['INTERNAL URL'])) + #print("INTERNAL URL = " + str(result['_id']['INTERNAL URL'])) tmp_val['internal_url'] = str(result['_id']['INTERNAL URL']) - print("Nombre = " + str(result['count'])) + #print("Nombre = " + str(result['count'])) tmp_val['nb_view'] = str(result['count']) insertObject.append(JSONEncoder().encode(tmp_val)) @@ -442,13 +442,13 @@ def GetStat_class_view_topX(diction): name = tmp tab_valeur = [] for result in collection.aggregate(pipe2): - print(str(result['_id'])+" ===> "+str(result['count'])) + if( name in result['_id']['INTERNAL URL']): - print(" OKKKKKKKKKKKKKKKKKKKK ") + tab_valeur.append(str(result['count'])) ret_val['data'+str(i)] = tab_valeur - print(" valll = "+str(ret_val)) + insertObject.append(JSONEncoder().encode(ret_val)) insertObject.append(JSONEncoder().encode(categories)) @@ -524,7 +524,7 @@ def Get_X_Best_Class_On_Given_period(diction): } ] - print(" PIP 222 =" + str(pipe2)) + #print(" PIP 222 =" + str(pipe2)) insertObject = [] for result in collection.aggregate(pipe2): @@ -536,15 +536,15 @@ def Get_X_Best_Class_On_Given_period(diction): if ("OWNER" in result['_id'].keys()): if result['_id']['OWNER']: - print("Owner = " + str(result['_id']['OWNER'])) + #print("Owner = " + str(result['_id']['OWNER'])) tmp_val['owner'] = str(result['_id']['OWNER']) if ("INTERNAL URL" in result['_id'].keys()): if result['_id']['INTERNAL URL']: - print("INTERNAL URL = " + str(result['_id']['INTERNAL URL'])) + #print("INTERNAL URL = " + str(result['_id']['INTERNAL URL'])) tmp_val['internal_url'] = str(result['_id']['INTERNAL URL']) - print("Nombre = " + str(result['count'])) + #print("Nombre = " + str(result['count'])) tmp_val['nb_view'] = str(result['count']) insertObject.append(JSONEncoder().encode(tmp_val)) @@ -672,12 +672,12 @@ def GetStat_class_by_internal_url(diction): tmp = tmp - 1 day_before = new_end_date + timedelta(days=tmp) categories_val = categories_val+str(day_before)+"," - print(" day_before DATE = " + str(day_before)+ " VS "+str(new_from_date)) + #print(" day_before DATE = " + str(day_before)+ " VS "+str(new_from_date)) if(categories_val.endswith(',') ): categories_val = categories_val[:-1] - print(" categories_val = "+categories_val) + #print(" categories_val = "+categories_val) categories['categories'] = str(categories_val) # Verification de la validité du token @@ -756,28 +756,28 @@ def GetStat_class_by_internal_url(diction): ] - print(" PIP 2 ="+str(pipe2)) - for result in collection.aggregate(pipe2): - print(str(result['_id']) + " ===> " + str(result['count'])) + #print(" PIP 2 ="+str(pipe2)) + #for result in collection.aggregate(pipe2): + # print(str(result['_id']) + " ===> " + str(result['count'])) insertObject = [] i = 0 for tmp in liste_formation : i = i +1 - print(" Traitement de "+str(tmp)) + #print(" Traitement de "+str(tmp)) ret_val = {} ret_val['name'+str(i)] = tmp name = tmp tab_valeur = [] for result in collection.aggregate(pipe2): - print(str(result['_id'])+" ===> "+str(result['count'])) + if( name in result['_id']['INTERNAL URL']): - print(" OKKKKKKKKKKKKKKKKKKKK ") + tab_valeur.append(str(result['count'])) ret_val['data'+str(i)] = tab_valeur - print(" valll = "+str(ret_val)) + insertObject.append(JSONEncoder().encode(ret_val)) insertObject.append(JSONEncoder().encode(categories))