17/09/22 - 14h00

master
ChérifBALDE 2022-09-17 14:22:25 +02:00 committed by cherif
parent 64f404125b
commit 271279400a
4 changed files with 111 additions and 48 deletions

View File

@ -145,6 +145,10 @@ Taux de TVA
TVA_TAUX = 0.2
"""
Variable defini le mombre max de mots clés dans une formation
"""
MAX_KEYWORD = 3
"""
Connexion SFTP

View File

@ -29,9 +29,6 @@ class JSONEncoder(json.JSONEncoder):
MAX_KEYWORD = 3
'''
Cette fonction ajoute une formation
elle verifie le token de l'entité qui ajoute la formation.
@ -47,7 +44,8 @@ def add_class(diction):
'''
field_list = ['external_code', 'title', 'description', 'trainer', 'institut_formation', 'distantiel', 'presentiel',
'price', 'url','duration', 'duration_unit', 'token', 'plus_produit', 'mots_cle','domaine',
'internal_url', 'zone_diffusion', 'metier', 'date_lieu', 'published']
'internal_url', 'zone_diffusion', 'metier', 'date_lieu', 'published', 'img_url']
incom_keys = diction.keys()
for val in incom_keys:
if val not in field_list:
@ -201,13 +199,13 @@ def add_class(diction):
if diction['mots_cle']:
mydata['mots_cle'] = diction['mots_cle']
'''
Verification du nombre de mots clée : limite MAX_KEYWORD (3)
Verification du nombre de mots clée : limite MYSY_GV.MAX_KEYWORD (3)
'''
nb_keyword = mydata['mots_cle'].split(";")
if( len(nb_keyword) > MAX_KEYWORD ):
if( len(nb_keyword) > MYSY_GV.MAX_KEYWORD ):
mycommon.myprint(
str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés")
return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés"
str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MYSY_GV.MAX_KEYWORD)+" mots clés")
return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MYSY_GV.MAX_KEYWORD)+" mots clés"
@ -215,6 +213,19 @@ def add_class(diction):
if diction['domaine']:
mydata['domaine'] = diction['domaine']
# Traitement de l'url imag
if ("img_url" in diction.keys()):
if diction['img_url']:
# Verifier si l'image existe
status_tmp, img = mycommon.TryUrlImage(str(diction['img_url']))
if( status_tmp is False):
mycommon.myprint(
str(inspect.stack()[0][3]) + " : l'url de l'image de la formation " + str(
mydata['external_code']) + " est incorrecte ")
return False, " l'url de l'image de la formation " + str(mydata['external_code']) + " est incorrecte "
mydata['img_url'] = diction['img_url']
if ("duration_unit" in diction.keys()):
if diction['duration_unit']:
@ -242,7 +253,7 @@ def add_class(diction):
tab_country = []
tab_city = []
for val in tmp_str:
print(" val = "+str(val))
#print(" val = "+str(val))
if( len(str(val)) <= 0 ):
continue
@ -392,7 +403,7 @@ def update_class(diction):
field_list = ['external_code', 'title', 'description', 'trainer', 'institut_formation', 'distantiel',
'presentiel','price', 'url', 'duration', 'token','plus_produit', 'mots_cle',
'domaine', 'internal_code', 'internal_url','zone_diffusion', 'metier',
'date_lieu', 'published']
'date_lieu', 'published', 'img_url']
incom_keys = diction.keys()
for val in incom_keys:
@ -457,6 +468,19 @@ def update_class(diction):
if ("title" in diction.keys()):
mydata['title'] = diction['title']
if ("img_url" in diction.keys()):
if diction['img_url']:
# Verifier si l'image existe
status_tmp, img = mycommon.TryUrlImage(str(diction['img_url']))
if (status_tmp is False):
mycommon.myprint(
str(inspect.stack()[0][3]) + " : l'url de l'image de la formation " + str(my_internal_code) + " est incorrecte ")
return False, " l'url de l'image de la formation " + str(my_internal_code) + " est incorrecte "
mydata['img_url'] = diction['img_url']
if ("description" in diction.keys()):
mydata['description'] = diction['description']
@ -525,9 +549,7 @@ def update_class(diction):
tab_country = []
tab_city = []
for val in tmp_str:
print(" val = "+str(val))
print(" val = " + str(val))
if (len(str(val)) <= 0):
continue
@ -1963,9 +1985,9 @@ def add_class_mass(file=None, Folder=None, diction=None):
'''
# Verification que les noms des colonne sont bien corrects"
'''
field_list = ['external_code', 'title', 'description', 'trainer', 'institut_formation',
'distantiel', 'presentiel', 'prix', 'domaine', 'url','duration', 'plus_produit',
'mots_cle', 'zone_diffusion', 'metier', 'date_lieu', 'publie']
field_list = ['external_code', 'titre', 'description', 'formateur', 'institut_formation',
'distantiel', 'presentiel', 'prix', 'domaine', 'url','duree', 'plus_produit',
'mots_cle', 'zone_diffusion', 'metier', 'date_lieu', 'publie', 'img_url']
total_rows = len(df)
@ -2011,14 +2033,14 @@ def add_class_mass(file=None, Folder=None, diction=None):
for n in x:
mydata = {}
mydata['external_code'] = str(df['external_code'].values[n])
mydata['title'] = str(df['title'].values[n])
mydata['title'] = str(df['titre'].values[n])
mydata['domaine'] = str(df['domaine'].values[n])
mydata['description'] = str(df['description'].values[n])
mydata['trainer'] = str(df['trainer'].values[n])
mydata['trainer'] = str(df['formateur'].values[n])
mydata['institut_formation'] = str(df['institut_formation'].values[n])
#mydata['distantiel'] = str(df['presentiel'].values[n])
mydata['url'] = str(df['url'].values[n])
mydata['duration'] = float(str(df['duration'].values[n]))
mydata['duration'] = float(str(df['duree'].values[n]))
mydata['plus_produit'] = str(df['plus_produit'].values[n])
mydata['mots_cle'] = str(df['mots_cle'].values[n])
mydata['presentiel'] = str(df['presentiel'].values[n])
@ -2028,13 +2050,33 @@ def add_class_mass(file=None, Folder=None, diction=None):
mydata['published'] = str(df['publie'].values[n])
'''
Verification du nombre de mots clée : limite MAX_KEYWORD (3)
Verification de l'image
'''
if ("img_url" in df.keys()):
mydata['img_url'] = str(df['img_url'].values[n])
if (str(df['img_url'].values[n]) == 'nan'):
mydata['img_url'] = ""
print(" ### mydata['img_url'] = '"+str(mydata['img_url'])+"' ")
if( len(str(mydata['img_url'])) > 0 ):
# Verifier si l'image existe
status_tmp, img = mycommon.TryUrlImage(str(mydata['img_url']))
if (status_tmp is False):
mycommon.myprint(
str(inspect.stack()[0][3]) + " : l'url de l'image de la formation " + str(
mydata['external_code']) + " est incorrecte ")
return False, " l'url de l'image de la formation " + str(mydata['external_code']) + " est incorrecte "
'''
Verification du nombre de mots clée : limite MYSY_GV.MAX_KEYWORD (3)
'''
nb_keyword = mydata['mots_cle'].split(";")
if( len(nb_keyword) > MAX_KEYWORD ):
if( len(nb_keyword) > MYSY_GV.MAX_KEYWORD ):
mycommon.myprint(
str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés")
return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MAX_KEYWORD)+" mots clés"
str(inspect.stack()[0][3]) + " : La formation "+str(mydata['external_code'])+" a plus de "+ str(MYSY_GV.MAX_KEYWORD)+" mots clés")
return False, " La formation "+str(mydata['external_code'])+" a plus de "+ str(MYSY_GV.MAX_KEYWORD)+" mots clés"
'''

View File

@ -1,7 +1,8 @@
import hashlib
import _pickle as cPickle
import pickle
from PIL import Image
import requests
import bson
from pymongo import MongoClient
import pymongo
@ -1759,3 +1760,19 @@ def removeLSaccount():
exc_type, exc_obj, exc_tb = sys.exc_info()
print(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
return False
"""
Cette fonction verifier si une url est une image.
elle permet si besoin de verifier la taille de l'image
Cette fonction sera dans un premier temps utilisée
dans la gestion des images associées à une formation.
"""
def TryUrlImage(url=None):
try:
img = Image.open(requests.get(url, stream=True).raw)
return True, img
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
return False, False

View File

@ -76,22 +76,22 @@ def GetStat_class_view_old():
if ("Date_view" in result['_id'].keys()):
if result['_id']['Date_view']:
print( "Date_view = "+str( result['_id']['Date_view']) )
tmp_val['Date_view'] = str( result['_id']['Date_view'])
if ("OWNER" in result['_id'].keys()):
if result['_id']['OWNER']:
print( "Owner = "+str( result['_id']['OWNER']) )
#print( "Owner = "+str( result['_id']['OWNER']) )
tmp_val['owner'] = str( result['_id']['OWNER'])
if ("INTERNAL URL" in result['_id'].keys()):
if result['_id']['INTERNAL URL']:
print( "INTERNAL URL = "+str( result['_id']['INTERNAL URL']) )
#print( "INTERNAL URL = "+str( result['_id']['INTERNAL URL']) )
tmp_val['internal_url'] = str(result['_id']['INTERNAL URL'])
print("Nombre = "+str(result['count']))
#print("Nombre = "+str(result['count']))
tmp_val['nb_view'] = str(result['count'])
insertObject.append(JSONEncoder().encode(tmp_val))
@ -221,7 +221,7 @@ def GetStat_class_view(diction):
}
]
print(" PIP 2 ="+str(pipe2))
#print(" PIP 2 ="+str(pipe2))
insertObject = []
for result in collection.aggregate(pipe2):
@ -232,20 +232,20 @@ def GetStat_class_view(diction):
if ("Date_view" in result['_id'].keys()):
if result['_id']['Date_view']:
print("Date_view = " + str(result['_id']['Date_view']))
#print("Date_view = " + str(result['_id']['Date_view']))
tmp_val['Date_view'] = str(result['_id']['Date_view'])
if ("OWNER" in result['_id'].keys()):
if result['_id']['OWNER']:
print("Owner = " + str(result['_id']['OWNER']))
#print("Owner = " + str(result['_id']['OWNER']))
tmp_val['owner'] = str(result['_id']['OWNER'])
if ("INTERNAL URL" in result['_id'].keys()):
if result['_id']['INTERNAL URL']:
print("INTERNAL URL = " + str(result['_id']['INTERNAL URL']))
#print("INTERNAL URL = " + str(result['_id']['INTERNAL URL']))
tmp_val['internal_url'] = str(result['_id']['INTERNAL URL'])
print("Nombre = " + str(result['count']))
#print("Nombre = " + str(result['count']))
tmp_val['nb_view'] = str(result['count'])
insertObject.append(JSONEncoder().encode(tmp_val))
@ -442,13 +442,13 @@ def GetStat_class_view_topX(diction):
name = tmp
tab_valeur = []
for result in collection.aggregate(pipe2):
print(str(result['_id'])+" ===> "+str(result['count']))
if( name in result['_id']['INTERNAL URL']):
print(" OKKKKKKKKKKKKKKKKKKKK ")
tab_valeur.append(str(result['count']))
ret_val['data'+str(i)] = tab_valeur
print(" valll = "+str(ret_val))
insertObject.append(JSONEncoder().encode(ret_val))
insertObject.append(JSONEncoder().encode(categories))
@ -524,7 +524,7 @@ def Get_X_Best_Class_On_Given_period(diction):
}
]
print(" PIP 222 =" + str(pipe2))
#print(" PIP 222 =" + str(pipe2))
insertObject = []
for result in collection.aggregate(pipe2):
@ -536,15 +536,15 @@ def Get_X_Best_Class_On_Given_period(diction):
if ("OWNER" in result['_id'].keys()):
if result['_id']['OWNER']:
print("Owner = " + str(result['_id']['OWNER']))
#print("Owner = " + str(result['_id']['OWNER']))
tmp_val['owner'] = str(result['_id']['OWNER'])
if ("INTERNAL URL" in result['_id'].keys()):
if result['_id']['INTERNAL URL']:
print("INTERNAL URL = " + str(result['_id']['INTERNAL URL']))
#print("INTERNAL URL = " + str(result['_id']['INTERNAL URL']))
tmp_val['internal_url'] = str(result['_id']['INTERNAL URL'])
print("Nombre = " + str(result['count']))
#print("Nombre = " + str(result['count']))
tmp_val['nb_view'] = str(result['count'])
insertObject.append(JSONEncoder().encode(tmp_val))
@ -672,12 +672,12 @@ def GetStat_class_by_internal_url(diction):
tmp = tmp - 1
day_before = new_end_date + timedelta(days=tmp)
categories_val = categories_val+str(day_before)+","
print(" day_before DATE = " + str(day_before)+ " VS "+str(new_from_date))
#print(" day_before DATE = " + str(day_before)+ " VS "+str(new_from_date))
if(categories_val.endswith(',') ):
categories_val = categories_val[:-1]
print(" categories_val = "+categories_val)
#print(" categories_val = "+categories_val)
categories['categories'] = str(categories_val)
# Verification de la validité du token
@ -756,28 +756,28 @@ def GetStat_class_by_internal_url(diction):
]
print(" PIP 2 ="+str(pipe2))
for result in collection.aggregate(pipe2):
print(str(result['_id']) + " ===> " + str(result['count']))
#print(" PIP 2 ="+str(pipe2))
#for result in collection.aggregate(pipe2):
# print(str(result['_id']) + " ===> " + str(result['count']))
insertObject = []
i = 0
for tmp in liste_formation :
i = i +1
print(" Traitement de "+str(tmp))
#print(" Traitement de "+str(tmp))
ret_val = {}
ret_val['name'+str(i)] = tmp
name = tmp
tab_valeur = []
for result in collection.aggregate(pipe2):
print(str(result['_id'])+" ===> "+str(result['count']))
if( name in result['_id']['INTERNAL URL']):
print(" OKKKKKKKKKKKKKKKKKKKK ")
tab_valeur.append(str(result['count']))
ret_val['data'+str(i)] = tab_valeur
print(" valll = "+str(ret_val))
insertObject.append(JSONEncoder().encode(ret_val))
insertObject.append(JSONEncoder().encode(categories))