537 lines
18 KiB
Python
537 lines
18 KiB
Python
'''
|
|
Ce fichier traite tout ce qui est liée aux requetes de statistique
|
|
|
|
'''
|
|
import pymongo
|
|
from pymongo import MongoClient
|
|
import json
|
|
from bson import ObjectId
|
|
import re
|
|
from datetime import datetime
|
|
import prj_common as mycommon
|
|
import secrets
|
|
import inspect
|
|
import sys, os
|
|
import csv
|
|
import pandas as pd
|
|
from pymongo import ReturnDocument
|
|
import GlobalVariable as MYSY_GV
|
|
from math import isnan
|
|
import GlobalVariable as MYSY_GV
|
|
from datetime import timedelta
|
|
|
|
|
|
class JSONEncoder(json.JSONEncoder):
|
|
def default(self, o):
|
|
if isinstance(o, ObjectId):
|
|
return str(o)
|
|
return json.JSONEncoder.default(self, o)
|
|
|
|
|
|
def GetStat_class_view_old():
|
|
try:
|
|
# collection
|
|
collection = MYSY_GV.dbname["user_recherche_result"]
|
|
|
|
"""
|
|
Pour avoir une requete qui groupe par jour, ou jour-heure
|
|
on doit jouer avec le
|
|
"date" : { $substr: [ "$date_update", 0, 25 ] } ==> ou 25 et la largeur de la coupure,
|
|
"""
|
|
|
|
QUERY_BY = "jour"
|
|
SUBSTR = 0
|
|
|
|
if (QUERY_BY == 'heure'):
|
|
SUBSTR = 13
|
|
elif( QUERY_BY == 'jour'):
|
|
SUBSTR = 10
|
|
elif (QUERY_BY == 'mois'):
|
|
SUBSTR = 7
|
|
|
|
|
|
|
|
pipe2 = [{
|
|
'$group': {
|
|
'_id': {
|
|
"INTERNAL URL":"$internal_url",
|
|
"OWNER":"$owner",
|
|
"Date_view": { "$substr": ["$date_update", 0, SUBSTR]},
|
|
},
|
|
'count': { '$count': { }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"$sort" : { "count": -1 }
|
|
}
|
|
]
|
|
|
|
insertObject = []
|
|
for result in collection.aggregate(pipe2):
|
|
tmp_val = {}
|
|
|
|
#tab_training_id.append(str(result["_id"]))
|
|
#print(result['_id'])
|
|
|
|
if ("Date_view" in result['_id'].keys()):
|
|
if result['_id']['Date_view']:
|
|
print( "Date_view = "+str( result['_id']['Date_view']) )
|
|
tmp_val['Date_view'] = str( result['_id']['Date_view'])
|
|
|
|
|
|
if ("OWNER" in result['_id'].keys()):
|
|
if result['_id']['OWNER']:
|
|
print( "Owner = "+str( result['_id']['OWNER']) )
|
|
tmp_val['owner'] = str( result['_id']['OWNER'])
|
|
|
|
if ("INTERNAL URL" in result['_id'].keys()):
|
|
if result['_id']['INTERNAL URL']:
|
|
print( "INTERNAL URL = "+str( result['_id']['INTERNAL URL']) )
|
|
tmp_val['internal_url'] = str(result['_id']['INTERNAL URL'])
|
|
|
|
|
|
print("Nombre = "+str(result['count']))
|
|
tmp_val['nb_view'] = str(result['count'])
|
|
insertObject.append(JSONEncoder().encode(tmp_val))
|
|
|
|
|
|
#print(result)
|
|
|
|
|
|
return True, insertObject
|
|
|
|
except Exception as e:
|
|
exc_type, exc_obj, exc_tb = sys.exc_info()
|
|
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
|
|
return False, "Impossible de recuperer les stat"
|
|
|
|
|
|
def GetStat_class_view(diction):
|
|
try:
|
|
'''
|
|
# Verification que les champs reçus dans l'API sont bien dans la liste des champs autorisés
|
|
# Cela evite le cas ou une entité tierce ajouter les valeurs inconnu dans l'API
|
|
# Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste
|
|
# field_list.
|
|
'''
|
|
field_list = ['external_code', 'internal_url', 'zone_diffusion', 'metier', 'date_lieu',
|
|
'published', 'token', 'date_start', 'date_end']
|
|
incom_keys = diction.keys()
|
|
for val in incom_keys:
|
|
if val not in field_list:
|
|
mycommon.myprint(str(inspect.stack()[0][
|
|
3]) + " - Le champ '" + val + "' n'est pas autorisé, Creation formation annulée")
|
|
return False, " Verifier votre API"
|
|
|
|
'''
|
|
Une fois qu'on a controlé que toutes les clés mise dans l'API sont correcte. etape precedente,
|
|
On controle que les champs obligatoires sont presents dans la liste
|
|
'''
|
|
field_list_obligatoire = [ 'token']
|
|
|
|
for val in field_list_obligatoire:
|
|
if val not in diction:
|
|
mycommon.myprint(str(inspect.stack()[0][3])+" - La valeur '" + val + "' n'est pas presente dans liste ")
|
|
return False, " Verifier votre API"
|
|
|
|
'''
|
|
Verification si le token et l'email sont valident
|
|
'''
|
|
|
|
# recuperation des paramettre
|
|
mydata = {}
|
|
|
|
|
|
if ("token" in diction.keys()):
|
|
if diction['token']:
|
|
mydata['token'] = diction['token']
|
|
|
|
date_start = ""
|
|
if ("date_start" in diction.keys()):
|
|
if diction['date_start']:
|
|
date_start = diction['date_start']
|
|
|
|
date_end = ""
|
|
if ("date_end" in diction.keys()):
|
|
if diction['date_end']:
|
|
date_end = diction['date_end']
|
|
|
|
|
|
|
|
# Verification de la validité du token
|
|
'''
|
|
Important : pour créer une formation, il faut obligatoirement avoir un token.
|
|
PAS DE CREATION DE FORMATION EN MODE NON CONNECTE.
|
|
|
|
CONCERNANT LES CREEES PAR NOS SYSTEME AUTOMATIQUE, IL FAUDRA LEUR PASSER UNE VALEUR MALGRE TOUT
|
|
|
|
'''
|
|
retval = mycommon.check_partner_token_validity("", str(mydata['token']))
|
|
|
|
if retval is False:
|
|
mycommon.myprint(str(inspect.stack()[0][3]) + " - Le token n'est pas valide")
|
|
return False, "L'email ou le token ne sont pas valident"
|
|
|
|
# Recuperation du recid du partenaire
|
|
user_recid = mycommon.get_parnter_recid_from_token(str(mydata['token']))
|
|
if user_recid is False:
|
|
mycommon.myprint(str(inspect.stack()[0][3]) + " - Impossible de recuperer le recid du partenaire")
|
|
return False, " Les informations d'identification sont incorrectes"
|
|
|
|
# collection
|
|
collection = MYSY_GV.dbname["user_recherche_result"]
|
|
|
|
"""
|
|
Pour avoir une requete qui groupe par jour, ou jour-heure
|
|
on doit jouer avec le
|
|
"date" : { $substr: [ "$date_update", 0, 25 ] } ==> ou 25 et la largeur de la coupure,
|
|
"""
|
|
|
|
thisweek = datetime.today() - timedelta(days=0)
|
|
thisweek_format = thisweek.strftime('%Y-%m-%d')
|
|
|
|
QUERY_BY = "jour"
|
|
SUBSTR = 0
|
|
|
|
if (QUERY_BY == 'heure'):
|
|
SUBSTR = 13
|
|
elif (QUERY_BY == 'jour'):
|
|
SUBSTR = 10
|
|
elif (QUERY_BY == 'mois'):
|
|
SUBSTR = 7
|
|
|
|
pipe2 = [
|
|
{'$match': {'date_update' : { '$gte' : str(date_start), '$lte' : str(date_end)},
|
|
'token':str(mydata['token'])
|
|
}},
|
|
{
|
|
'$group': {
|
|
'_id': {
|
|
|
|
"OWNER": "$owner",
|
|
"Date_view": {"$substr": ["$date_update", 0, SUBSTR]},
|
|
},
|
|
'count': {'$count': {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"$sort": {"count": -1}
|
|
}
|
|
]
|
|
|
|
print(" PIP 2 ="+str(pipe2))
|
|
|
|
insertObject = []
|
|
for result in collection.aggregate(pipe2):
|
|
tmp_val = {}
|
|
|
|
# tab_training_id.append(str(result["_id"]))
|
|
# print(result['_id'])
|
|
|
|
if ("Date_view" in result['_id'].keys()):
|
|
if result['_id']['Date_view']:
|
|
print("Date_view = " + str(result['_id']['Date_view']))
|
|
tmp_val['Date_view'] = str(result['_id']['Date_view'])
|
|
|
|
if ("OWNER" in result['_id'].keys()):
|
|
if result['_id']['OWNER']:
|
|
print("Owner = " + str(result['_id']['OWNER']))
|
|
tmp_val['owner'] = str(result['_id']['OWNER'])
|
|
|
|
if ("INTERNAL URL" in result['_id'].keys()):
|
|
if result['_id']['INTERNAL URL']:
|
|
print("INTERNAL URL = " + str(result['_id']['INTERNAL URL']))
|
|
tmp_val['internal_url'] = str(result['_id']['INTERNAL URL'])
|
|
|
|
print("Nombre = " + str(result['count']))
|
|
tmp_val['nb_view'] = str(result['count'])
|
|
insertObject.append(JSONEncoder().encode(tmp_val))
|
|
|
|
# print(result)
|
|
|
|
return True, insertObject
|
|
|
|
except Exception as e:
|
|
exc_type, exc_obj, exc_tb = sys.exc_info()
|
|
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
|
|
return False, "Impossible de recuperer les stat"
|
|
|
|
|
|
"""
|
|
Cette fonction retourne le top 5 sur une periode donnée
|
|
"""
|
|
def GetStat_class_view_topX(diction):
|
|
try:
|
|
'''
|
|
# Verification que les champs reçus dans l'API sont bien dans la liste des champs autorisés
|
|
# Cela evite le cas ou une entité tierce ajouter les valeurs inconnu dans l'API
|
|
# Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste
|
|
# field_list.
|
|
'''
|
|
field_list = ['external_code', 'internal_url', 'zone_diffusion', 'metier', 'date_lieu',
|
|
'published', 'token', 'date_start', 'date_end', 'topX']
|
|
incom_keys = diction.keys()
|
|
for val in incom_keys:
|
|
if val not in field_list:
|
|
mycommon.myprint(str(inspect.stack()[0][
|
|
3]) + " - Le champ '" + val + "' n'est pas autorisé, Creation formation annulée")
|
|
return False, " Verifier votre API"
|
|
|
|
'''
|
|
Une fois qu'on a controlé que toutes les clés mise dans l'API sont correcte. etape precedente,
|
|
On controle que les champs obligatoires sont presents dans la liste
|
|
'''
|
|
field_list_obligatoire = [ 'token', 'topX']
|
|
|
|
for val in field_list_obligatoire:
|
|
if val not in diction:
|
|
mycommon.myprint(str(inspect.stack()[0][3])+" - La valeur '" + val + "' n'est pas presente dans liste ")
|
|
return False, " Verifier votre API"
|
|
|
|
'''
|
|
Verification si le token et l'email sont valident
|
|
'''
|
|
|
|
# recuperation des paramettre
|
|
mydata = {}
|
|
|
|
|
|
if ("token" in diction.keys()):
|
|
if diction['token']:
|
|
mydata['token'] = diction['token']
|
|
|
|
date_start = ""
|
|
if ("date_start" in diction.keys()):
|
|
if diction['date_start']:
|
|
date_start = diction['date_start']
|
|
|
|
date_end = ""
|
|
if ("date_end" in diction.keys()):
|
|
if diction['date_end']:
|
|
date_end = diction['date_end']
|
|
|
|
topX = 0
|
|
if ("topX" in diction.keys()):
|
|
if diction['topX']:
|
|
topX = mycommon.tryInt(diction['topX'])
|
|
|
|
|
|
|
|
|
|
# Verification de la validité du token
|
|
'''
|
|
Important : pour créer une formation, il faut obligatoirement avoir un token.
|
|
PAS DE CREATION DE FORMATION EN MODE NON CONNECTE.
|
|
|
|
CONCERNANT LES CREEES PAR NOS SYSTEME AUTOMATIQUE, IL FAUDRA LEUR PASSER UNE VALEUR MALGRE TOUT
|
|
|
|
'''
|
|
retval = mycommon.check_partner_token_validity("", str(mydata['token']))
|
|
|
|
if retval is False:
|
|
mycommon.myprint(str(inspect.stack()[0][3]) + " - Le token n'est pas valide")
|
|
return False, "L'email ou le token ne sont pas valident"
|
|
|
|
# Recuperation du recid du partenaire
|
|
user_recid = mycommon.get_parnter_recid_from_token(str(mydata['token']))
|
|
if user_recid is False:
|
|
mycommon.myprint(str(inspect.stack()[0][3]) + " - Impossible de recuperer le recid du partenaire")
|
|
return False, " Les informations d'identification sont incorrectes"
|
|
|
|
|
|
"""
|
|
Recuperation des id (internal_url) des formations
|
|
"""
|
|
status_tmp, diction_topx = Get_X_Best_Class_On_Given_period(diction)
|
|
|
|
print(" pour cette stat, le voix le top " + str(topX) + " des ref concernées ")
|
|
print(diction_topx)
|
|
|
|
print(" ========= print(diction_topx)")
|
|
liste_formation = []
|
|
for val_tmp in diction_topx:
|
|
json_object = json.loads(val_tmp)
|
|
if ("internal_url" in json_object.keys()):
|
|
if json_object['internal_url']:
|
|
liste_formation.append(str(json_object['internal_url']))
|
|
|
|
# collection
|
|
collection = MYSY_GV.dbname["user_recherche_result"]
|
|
|
|
"""
|
|
Pour avoir une requete qui groupe par jour, ou jour-heure
|
|
on doit jouer avec le
|
|
"date" : { $substr: [ "$date_update", 0, 25 ] } ==> ou 25 et la largeur de la coupure,
|
|
"""
|
|
|
|
thisweek = datetime.today() - timedelta(days=0)
|
|
thisweek_format = thisweek.strftime('%Y-%m-%d')
|
|
|
|
QUERY_BY = "jour"
|
|
SUBSTR = 0
|
|
|
|
if (QUERY_BY == 'heure'):
|
|
SUBSTR = 13
|
|
elif (QUERY_BY == 'jour'):
|
|
SUBSTR = 10
|
|
elif (QUERY_BY == 'mois'):
|
|
SUBSTR = 7
|
|
|
|
pipe2 = [
|
|
{'$match': {'date_update' : { '$gte' : str(date_start), '$lte' : str(date_end)},
|
|
'token':str(mydata['token']),
|
|
'internal_url': {'$in': liste_formation},
|
|
}},
|
|
{
|
|
'$group': {
|
|
'_id': {
|
|
"INTERNAL URL": "$internal_url",
|
|
"OWNER": "$owner",
|
|
"Date_view": {"$substr": ["$date_update", 0, SUBSTR]},
|
|
},
|
|
'count': {'$count': {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"$sort": {"count": -1}
|
|
},
|
|
|
|
]
|
|
|
|
print(" PIP 2 ="+str(pipe2))
|
|
|
|
insertObject = []
|
|
for result in collection.aggregate(pipe2):
|
|
tmp_val = {}
|
|
|
|
# tab_training_id.append(str(result["_id"]))
|
|
# print(result['_id'])
|
|
|
|
if ("Date_view" in result['_id'].keys()):
|
|
if result['_id']['Date_view']:
|
|
print("Date_view = " + str(result['_id']['Date_view']))
|
|
tmp_val['Date_view'] = str(result['_id']['Date_view'])
|
|
|
|
if ("OWNER" in result['_id'].keys()):
|
|
if result['_id']['OWNER']:
|
|
print("Owner = " + str(result['_id']['OWNER']))
|
|
tmp_val['owner'] = str(result['_id']['OWNER'])
|
|
|
|
if ("INTERNAL URL" in result['_id'].keys()):
|
|
if result['_id']['INTERNAL URL']:
|
|
print("INTERNAL URL = " + str(result['_id']['INTERNAL URL']))
|
|
tmp_val['internal_url'] = str(result['_id']['INTERNAL URL'])
|
|
|
|
print("Nombre = " + str(result['count']))
|
|
tmp_val['nb_view'] = str(result['count'])
|
|
insertObject.append(JSONEncoder().encode(tmp_val))
|
|
|
|
# print(result)
|
|
|
|
return True, insertObject
|
|
|
|
except Exception as e:
|
|
exc_type, exc_obj, exc_tb = sys.exc_info()
|
|
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
|
|
return False, "Impossible de recuperer les stat"
|
|
|
|
|
|
"""
|
|
Cette fonction recuprer la liste des X formations d'un client donné,
|
|
les plus vue sur une periode donnée
|
|
"""
|
|
def Get_X_Best_Class_On_Given_period(diction):
|
|
try:
|
|
|
|
field_list = ['token', 'date_start', 'date_end', 'topX']
|
|
incom_keys = diction.keys()
|
|
for val in incom_keys:
|
|
if val not in field_list:
|
|
mycommon.myprint(str(inspect.stack()[0][
|
|
3]) + " - Le champ '" + val + "' n'est pas autorisé, action annulée")
|
|
return False, " Verifier votre API"
|
|
|
|
# collection
|
|
collection = MYSY_GV.dbname["user_recherche_result"]
|
|
|
|
mytoken = ""
|
|
if ("token" in diction.keys()):
|
|
if diction['token']:
|
|
mytoken = diction['token']
|
|
|
|
date_start = ""
|
|
if ("date_start" in diction.keys()):
|
|
if diction['date_start']:
|
|
date_start = diction['date_start']
|
|
|
|
date_end = ""
|
|
if ("date_end" in diction.keys()):
|
|
if diction['date_end']:
|
|
date_end = diction['date_end']
|
|
|
|
topX = 0
|
|
if ("topX" in diction.keys()):
|
|
if diction['topX']:
|
|
topX = mycommon.tryInt(diction['topX'])
|
|
|
|
pipe2 = [
|
|
{'$match': {'date_update': {'$gte': str(date_start), '$lte': str(date_end)},
|
|
'token': str(mytoken)
|
|
}},
|
|
{
|
|
'$group': {
|
|
'_id': {
|
|
"INTERNAL URL": "$internal_url",
|
|
"OWNER": "$owner",
|
|
|
|
},
|
|
'count': {'$count': {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"$sort": {"count": -1}
|
|
},
|
|
{
|
|
"$limit": topX
|
|
}
|
|
]
|
|
|
|
print(" PIP 222 =" + str(pipe2))
|
|
|
|
insertObject = []
|
|
for result in collection.aggregate(pipe2):
|
|
tmp_val = {}
|
|
|
|
# tab_training_id.append(str(result["_id"]))
|
|
# print(result['_id'])
|
|
|
|
|
|
if ("OWNER" in result['_id'].keys()):
|
|
if result['_id']['OWNER']:
|
|
print("Owner = " + str(result['_id']['OWNER']))
|
|
tmp_val['owner'] = str(result['_id']['OWNER'])
|
|
|
|
if ("INTERNAL URL" in result['_id'].keys()):
|
|
if result['_id']['INTERNAL URL']:
|
|
print("INTERNAL URL = " + str(result['_id']['INTERNAL URL']))
|
|
tmp_val['internal_url'] = str(result['_id']['INTERNAL URL'])
|
|
|
|
print("Nombre = " + str(result['count']))
|
|
tmp_val['nb_view'] = str(result['count'])
|
|
insertObject.append(JSONEncoder().encode(tmp_val))
|
|
|
|
# print(result)
|
|
|
|
|
|
return True, insertObject
|
|
|
|
except Exception as e:
|
|
exc_type, exc_obj, exc_tb = sys.exc_info()
|
|
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
|
|
return False, "Impossible de X les formations les plus vue"
|
|
|