""" Ce fichier contient les requetes utilisées dans les tableaux de bord des Inscriptions. Liste des requete à date : - TDB des inscriptions par session sur une période - TDB des inscriptions par formation sur une période """ import ast import dateutil import pymongo from dateutil.relativedelta import relativedelta from flask import send_file from pymongo import MongoClient import json from bson import ObjectId import re from datetime import datetime, timezone, date import prj_common as mycommon import secrets import inspect import sys, os import csv import pandas as pd from pymongo import ReturnDocument import GlobalVariable as MYSY_GV from math import isnan import GlobalVariable as MYSY_GV from datetime import timedelta from datetime import timedelta from operator import itemgetter """ Recuperation du nombre d'inscrit par par session sur une période """ def Get_Qery_Inscription_By_Session_By_Periode(diction): try: diction = mycommon.strip_dictionary(diction) """ Verification des input acceptés """ field_list = ['token', 'session_start_date', 'session_end_date', 'filter_value' ] incom_keys = diction.keys() for val in incom_keys: if val not in field_list and val.startswith('my_') is False: mycommon.myprint(str( inspect.stack()[0][3]) + " Le champ '" + val + "' n'existe pas") return False, " Les informations fournies sont incorrectes", """ Verification des champs obligatoires """ field_list_obligatoire = ['token', 'session_start_date', 'session_end_date' ] for val in field_list_obligatoire: if val not in diction: mycommon.myprint( str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ") return False, " Les informations fournies sont incorrectes", """ Verification de l'identité et autorisation de l'entité qui appelle cette API """ token = "" if ("token" in diction.keys()): if diction['token']: token = diction['token'] local_status, my_partner = mycommon.Check_Connexion_And_Return_Partner_Data(diction) if (local_status is not True): return local_status, my_partner filt_session_start_date = "" if ("session_start_date" in diction.keys() and diction['session_start_date']): filt_session_start_date = str(diction['session_start_date'])[0:10] local_status = mycommon.CheckisDate(filt_session_start_date) if (local_status is False): mycommon.myprint(str( inspect.stack()[0][3]) + " Le filtre : 'date de debut de session' n'est pas au format jj/mm/aaaa.") return False, " Le filtre : 'date de debut de session' n'est pas au format jj/mm/aaaa." filt_session_end_date = "" if ("session_end_date" in diction.keys() and diction['session_end_date']): filt_session_end_date = str(diction['session_end_date'])[0:10] local_status = mycommon.CheckisDate(filt_session_end_date) if (local_status is False): mycommon.myprint(str( inspect.stack()[0][3]) + " Le filtre : 'date de fin de session' n'est pas au format jj/mm/aaaa.") return False, " Le filtre : 'date de fin de session' n'est pas au format jj/mm/aaaa." """ Si la valeur de 'filter_value' est m0 ou m1, on va aller recuperer les date du mois correspondant. On ecrase les valeur de filt_session_start_date et filt_session_end_date """ if( 'filter_value' in diction.keys() ): if( str(diction['filter_value']) == "m0"): # On recupere les date du mois en cours local_status, start_current_month_date, end_current_month_date = mycommon.Get_Current_Month_Start_End_Date() if (local_status is False): return local_status, start_current_month_date filt_session_start_date = start_current_month_date filt_session_end_date = end_current_month_date elif (str(diction['filter_value']) == "m1"): # On recupere les date du mois en cours local_status, start_current_month_date, end_current_month_date = mycommon.Get_Previous_Month_Start_End_Date() if (local_status is False): return local_status, start_current_month_date filt_session_start_date = start_current_month_date filt_session_end_date = end_current_month_date RetObject = [] val_tmp = 1 filt_session_start_date_ISODATE = datetime.strptime(str(filt_session_start_date), '%d/%m/%Y') filt_session_end_date_ISODATE = datetime.strptime(str(filt_session_end_date), '%d/%m/%Y') filt_session_start_date_ISODATE_work = filt_session_start_date_ISODATE filt_session_end_date_ISODATE_work = filt_session_end_date_ISODATE """ qery_match = {'$and': [ {"partner_owner_recid":str(my_partner['recid']) }, {"valide":'1'}, {"apprenant_id": { "$exists": True } }, { 'mysy_date_debut_session': { '$gte': filt_session_start_date_ISODATE, '$lte': filt_session_end_date_ISODATE } } , ]} """ qery_match = {'$and': [{"partner_owner_recid": str(my_partner['recid'])}, {"valide": '1'}, {"apprenant_id": {"$exists": True}}, ]} pipe_qry = ([ {"$addFields": { "mysy_date_debut_session": { '$dateFromString': { 'dateString': '$date_debut', 'format': "%d/%m/%Y" } } } }, {'$match': qery_match}, #---- {'$lookup': { 'from': 'session_formation', 'let': {'session_id': "$session_id", 'partner_owner_recid': '$partner_owner_recid'}, 'pipeline': [ {'$match': {'$expr': {'$and': [ {'$eq': ["$_id", {'$convert': { 'input': "$$session_id", 'to': "objectId", 'onError': {'error': 'true'}, 'onNull': {'isnull': 'true'} }}]}, {'$eq': ["$valide", "1"]}, {'$eq': ["$partner_owner_recid", '$$partner_owner_recid']} ] } } }, ], 'as': 'session_formation_collection' } }, { '$unwind': '$session_formation_collection' }, # --- { "$lookup": { 'from': 'myclass', 'localField': 'session_formation_collection.class_internal_url', 'foreignField': 'internal_url', "pipeline": [{'$project': {'title': 1, 'internal_url': 1, 'external_code': 1, 'published': 1}} ], "as": "myclass_collection" } }, { "$unwind": "$myclass_collection" }, # -- # --- {'$lookup': { 'from': 'apprenant', "let": {'apprenant_id': "$apprenant_id", 'partner_owner_recid': '$partner_owner_recid'}, 'pipeline': [ {'$match': {'$expr': {'$and': [ {'$eq': ["$_id", {'$convert': { 'input': "$$apprenant_id", 'to': "objectId", 'onError': {'error': 'true'}, 'onNull': {'isnull': 'true'} }}]}, {'$eq': ["$valide", "1"]}, {'$eq': ["$partner_owner_recid", '$$partner_owner_recid']} ] } } }, ], 'as': 'apprenant_collection' } }, { '$unwind': '$apprenant_collection' }, ]) """ Cette requete donne le tableau complete des apprenants avec les sessions si on veut faire des group by, on ajoute ceci : {'$group': { '_id': { "apprenant_collection_nom":"$apprenant_collection.nom", "apprenant_collection_prenom":"$apprenant_collection.prenom", "apprenant_collection_session":"$session_formation_collection.code_session", }, 'count': {'$count': {} } } }, """ #print(" ### Get_Qery_Inscription_By_Session_By_Periode ici pipe_qry = ", pipe_qry) cpt = 0 tab_lines_inscription_data = [] for retval in MYSY_GV.dbname['inscription'].aggregate(pipe_qry): del retval['_id'] del retval['session_formation_collection']['_id'] del retval['apprenant_collection']['_id'] del retval['myclass_collection']['_id'] tab_lines_inscription_data.append(retval) #print(str(cpt)+" ### retval = ", retval) #json_formatted_str = json.dumps(retval, indent=2) #print(" ### Final Line : ", json_formatted_str) cpt = cpt + 1 local_status, local_retval = Format_Inscription_Data(tab_lines_inscription_data) if (local_status is False): return local_status, local_retval RetObject = [] cpt = 1 for val in local_retval: val['id'] = str(cpt) RetObject.append(mycommon.JSONEncoder().encode(val)) cpt = cpt + 1 tab_lines_employe_by_contract_json_formatted_str = json.dumps(val, indent=2) # print(" ### Final Line " + str(cpt) + " : ", tab_lines_employe_by_contract_json_formatted_str) print(" RetObject = ", RetObject) print(' on a ' + str(cpt) + ' lignes ') return True, RetObject except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) return False, " Impossible de récupérer les données " """ Cette fonction permet de formatter le tableau des apprenants sous forme de tableau plus exploitable """ def Format_Inscription_Data(tab_diction): try: return_tab = [] for val in tab_diction: node_inscrit = {} node_inscrit['inscription_status'] = val['status'] # Recuperation des données d'un apprenant (donc apprenant_id) if( "apprenant_id" in val.keys() and str(val['apprenant_id']).strip() != "" and "apprenant_collection" in val.keys()): if( "civilite" in val['apprenant_collection'].keys() and val['apprenant_collection']['civilite']): node_inscrit['civilite'] = val['apprenant_collection']['civilite'] else: node_inscrit['civilite'] = "" if ("nom" in val['apprenant_collection'].keys() and val['apprenant_collection']['nom']): node_inscrit['nom'] = val['apprenant_collection']['nom'] else: node_inscrit['nom'] = "" if ("prenom" in val['apprenant_collection'].keys() and val['apprenant_collection']['prenom']): node_inscrit['prenom'] = val['apprenant_collection']['prenom'] else: node_inscrit['prenom'] = "" if ("email" in val['apprenant_collection'].keys() and val['apprenant_collection']['email']): node_inscrit['email'] = val['apprenant_collection']['email'] else: node_inscrit['email'] = "" if ("telephone" in val['apprenant_collection'].keys() and val['apprenant_collection']['telephone']): node_inscrit['telephone'] = val['apprenant_collection']['telephone'] else: node_inscrit['telephone'] = "" # Recuperation des données d'une inscription n'ayant pas de dossier 'apprenant' elif ("apprenant_id" not in val.keys() or str(val['apprenant_id']).strip() != ""): if ("civilite" in val.keys() and val['civilite']): node_inscrit['civilite'] = val['civilite'] else: node_inscrit['civilite'] = "" if ("nom" in val.keys() and val['nom']): node_inscrit['nom'] = val['nom'] else: node_inscrit['nom'] = "" if ("prenom" in val.keys() and val['prenom']): node_inscrit['prenom'] = val['prenom'] else: node_inscrit['prenom'] = "" if ("email" in val.keys() and val['email']): node_inscrit['email'] = val['email'] else: node_inscrit['email'] = "" if ("telephone" in val.keys() and val['telephone']): node_inscrit['telephone'] = val['telephone'] else: node_inscrit['telephone'] = "" # Récuperation des données de la session à la quelle la personne est inscrite if( "session_formation_collection" in val.keys() ): if ("code_session" in val['session_formation_collection'].keys() and val['session_formation_collection']['code_session']): node_inscrit['session_code_session'] = val['session_formation_collection']['code_session'] else: node_inscrit['session_code_session'] = "" if ("class_internal_url" in val['session_formation_collection'].keys() and val['session_formation_collection']['class_internal_url']): node_inscrit['session_class_internal_url'] = val['session_formation_collection']['class_internal_url'] if ("date_debut" in val['session_formation_collection'].keys() and val['session_formation_collection']['date_debut']): node_inscrit['session_date_debut'] = val['session_formation_collection']['date_debut'] else: node_inscrit['session_date_debut'] = "" if ("date_fin" in val['session_formation_collection'].keys() and val['session_formation_collection']['date_fin']): node_inscrit['session_date_fin'] = val['session_formation_collection']['date_debut'] else: node_inscrit['session_date_fin'] = "" if ("distantiel" in val['session_formation_collection'].keys() and val['session_formation_collection']['distantiel']): node_inscrit['session_distantiel'] = val['session_formation_collection']['distantiel'] else: node_inscrit['session_distantiel'] = "" if ("presentiel" in val['session_formation_collection'].keys() and val['session_formation_collection']['presentiel']): node_inscrit['session_presentiel'] = val['session_formation_collection']['presentiel'] else: node_inscrit['session_presentiel'] = "" # Récuperation des données de la formation attachée a la session if ("myclass_collection" in val.keys()): if ("external_code" in val['myclass_collection'].keys() and val['myclass_collection']['external_code']): node_inscrit['session_class_external_code'] = val['myclass_collection']['external_code'] else: node_inscrit['session_class_external_code'] = "" if ("external_code" in val['myclass_collection'].keys() and val['myclass_collection']['external_code']): node_inscrit['session_class_external_code'] = val['myclass_collection']['external_code'] else: node_inscrit['session_class_external_code'] = "" if ("title" in val['myclass_collection'].keys() and val['myclass_collection']['title']): node_inscrit['session_class_title'] = val['myclass_collection']['title'] else: node_inscrit['session_class_title'] = "" if ("internal_url" in val['myclass_collection'].keys() and val['myclass_collection']['internal_url']): node_inscrit['session_class_internal_url'] = val['myclass_collection']['internal_url'] else: node_inscrit['session_class_internal_url'] = "" return_tab.append(node_inscrit) return True, return_tab except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) return False, " Impossible de reformater et de normer le tableau des inscrits " """ Cette fonction permet d'experter un dashbord en csv """ def TBD_Inscription_Export_Dashbord_To_Csv(diction): try: diction = mycommon.strip_dictionary(diction) """ Verification des input acceptés """ field_list = ['token', 'user_dashbord_id'] incom_keys = diction.keys() for val in incom_keys: if val not in field_list and val.startswith('my_') is False: mycommon.myprint(str( inspect.stack()[0][3]) + " Le champ '" + val + "' n'existe pas") return False, " Les informations fournies sont incorrectes", """ Verification des champs obligatoires """ field_list_obligatoire = ['token', 'user_dashbord_id'] for val in field_list_obligatoire: if val not in diction: mycommon.myprint( str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ") return False, " Les informations fournies sont incorrectes", """ Verification de l'identité et autorisation de l'entité qui appelle cette API """ token = "" if ("token" in diction.keys()): if diction['token']: token = diction['token'] local_status, my_partner = mycommon.Check_Connexion_And_Return_Partner_Data(diction) if (local_status is not True): return local_status, my_partner # Recuperation des données du user_dashbord my_user_dashbord = MYSY_GV.dbname['user_dashbord'].find_one({'_id': ObjectId(str(diction['user_dashbord_id'])), 'valide': '1', 'locked': '0', 'partner_owner_recid': str(my_partner['recid'])}) if (my_user_dashbord is None): mycommon.myprint( str(inspect.stack()[0][3]) + " - L'identifiant du tableau de bord est invalide ") return False, " L'identifiant du tableau de bord est invalide", new_retval_data = {} my_new_diction = {} my_new_diction['token'] = diction['token'] session_start_date = "" session_end_date = "" local_default_filter = ast.literal_eval(str(my_user_dashbord['default_filter'])) if ("session_start_date" in local_default_filter.keys() and "session_end_date" in local_default_filter.keys()): session_start_date = local_default_filter['session_start_date'] session_end_date = local_default_filter['session_end_date'] elif ("periode" in local_default_filter.keys()): my_new_diction['filter_value'] = str(local_default_filter['periode']) if (str(local_default_filter['periode']) == "m0"): # On recupere les date du mois en cours local_status, start_current_month_date, end_current_month_date = mycommon.Get_Current_Month_Start_End_Date() if (local_status is False): return local_status, start_current_month_date session_start_date = start_current_month_date session_end_date = end_current_month_date elif (str(local_default_filter['periode']) == "m1"): # On recupere les date du mois en cours local_status, start_current_month_date, end_current_month_date = mycommon.Get_Previous_Month_Start_End_Date() if (local_status is False): return local_status, start_current_month_date session_start_date = start_current_month_date session_end_date = end_current_month_date my_new_diction['session_start_date'] = session_start_date my_new_diction['session_end_date'] = session_end_date if (my_user_dashbord['dashbord_internal_code'] == "tbd_inscription_01"): local_status, local_retval = Get_Qery_Inscription_By_Session_By_Periode(my_new_diction) if (local_status is False): return local_status, local_retval new_retval_data = local_retval else: mycommon.myprint( str(inspect.stack()[0][3]) + " Requête inconnue ") return False, " Requête inconnue ", # Recuperation des colonne à exporter base_config_dashbord_data = MYSY_GV.dbname['base_config_dashbord'].find_one( {'dashbord_internal_code': str(my_user_dashbord['dashbord_internal_code'])}) tab_exported_fields = [] if ("exported_fields" in base_config_dashbord_data): tab_exported_fields = base_config_dashbord_data['exported_fields'] # Remettres les dates de filtres en debut de liste if ("filtre_date_fin" in tab_exported_fields): tab_exported_fields.remove("filtre_date_fin") tab_exported_fields.insert(0, "filtre_date_fin") if ("filtre_date_debut" in tab_exported_fields): tab_exported_fields.remove("filtre_date_debut") tab_exported_fields.insert(0, "filtre_date_debut") tab_exported_fields.insert(0, "date_extraction") todays_date = str(date.today().strftime("%d/%m/%Y")) ts = datetime.now().timestamp() ts = str(ts).replace(".", "").replace(",", "")[-5:] orig_file_name = "Export_csv_" + str(my_partner['recid']) + "_" + str(ts) + ".csv" outputFilename = str(MYSY_GV.TEMPORARY_DIRECTORY) + "/" + str(orig_file_name) with open(outputFilename, 'w', newline='') as outfile: fields = tab_exported_fields write = csv.DictWriter(outfile, fieldnames=fields) write.writeheader() for answers_record in new_retval_data: # Here we are using 'cursor' as an iterator flattened_record = {} for local_fiels in tab_exported_fields: answers_record_JSON = ast.literal_eval(str(answers_record)) if (str(local_fiels) in answers_record_JSON.keys()): flattened_record[str(local_fiels)] = answers_record_JSON[str(local_fiels)] flattened_record['filtre_date_debut'] = str(session_start_date) flattened_record['filtre_date_fin'] = str(session_end_date) flattened_record['date_extraction'] = str(todays_date) write.writerow(flattened_record) if os.path.exists(outputFilename): # print(" ### ok os.path.exists(outputFilename) "+str(outputFilename)) return True, send_file(outputFilename, as_attachment=True) return False, "Impossible de générer l'export csv (2) " except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) return False, " Impossible d'exporter les données " """ TDB : Nombre d'inscription par session """ """ TDB : Nombre d'inscription par formation """