Elyos_FI_Back_Office/attached_file_mgt.py

723 lines
28 KiB
Python

'''
Ce fichier permet de gerer les pièces jointes.
toutes les fonctions necessaires à la gestion des pièces jointes)
'''
import hashlib
import _pickle as cPickle
import pickle
from PIL import Image
import requests
import bson
from pymongo import MongoClient
import pymongo
from difflib import SequenceMatcher
import textdistance
from datetime import datetime
import logging
import secrets
import base64
from bson import ObjectId
from pymongo import MongoClient
import inspect
from werkzeug.utils import secure_filename
import time
import os
import csv
import inspect
import sys
import pandas as pd
from datetime import datetime
from pymongo import ReturnDocument
from unidecode import unidecode
import GlobalVariable as MYSY_GV
from serpapi import GoogleSearch
import re
import random
import json
from colorama import Fore
from colorama import Style
from flask import Flask, Response, render_template, request, abort, jsonify, send_from_directory
from xhtml2pdf import pisa
import jinja2
import ftplib
import pysftp
import html
import mariadb
from flask import send_file
import GlobalVariable as MYSY_GV
import prj_common as mycommon
from pathlib import Path
"""
Cette fonction recuperer un fichier,
effectue les controle de taille et de securité,
le stock dans une emplacement et
retourne le path complet.
/!\ : les metadata sont enregistrés dans une collection.
metadata :
- file_business_object : ici on definit a quoi correspond le fichier (ex : diplome 1, pièce identité, etc).
/!\ : ce champ est la clé. si un utilisateur envoie 2 fois un fichier pr le meme ob, une mise à jour sera faite dans la collection, en ecrasant l'ancien.
- file_name
- full_path
- file_extention
- date_downeload
- object_owner_collection : Ce champ definit la collection a laquelle est rattachée la PJ. par exemple : "candidat", "employe"
- object_owner_id : id du proprietaire dans la collection (ex : le champ '_id' dans la collection "employee"
(les 2 champs 'object_owner' sont utilisé pour gerer à qui / quoi est rattacher à la pièce jointe).
- status :
0 : downloaded
1 : accepted
-1 : rejected
"""
def Store_User_Downloaded_File(file=None, Folder=None, diction=None):
try:
full_file_path = ""
diction = mycommon.strip_dictionary(diction)
"""
Verification des input acceptés
"""
field_list = ['token', 'file_business_object', 'file_name', 'status','object_owner_collection', 'object_owner_id']
incom_keys = diction.keys()
for val in incom_keys:
if val not in field_list and val.startswith('my_') is False:
mycommon.myprint(str(
inspect.stack()[0][3]) + " Le champ '" + val + "' n'existe pas")
return False, " Les informations fournies sont incorrectes",
"""
Verification des champs obligatoires
"""
field_list_obligatoire = ['token', 'file_business_object', 'object_owner_collection', 'object_owner_id' ]
for val in field_list_obligatoire:
if val not in diction:
mycommon.myprint(
str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ")
return False, " Les informations fournies sont incorrectes",
local_status, my_partner = mycommon.Check_Connexion_And_Return_Partner_Data(diction)
if (local_status is not True):
return local_status, my_partner
basename = os.path.basename(file.filename)
basename2 = basename.split(".")
if (len(basename2) != 2):
mycommon.myprint(str(inspect.stack()[0][3]) + " - : Le nom du fichier est incorrect")
return False, "Le nom du fichier est incorrect"
if (str(basename2[1]).lower() not in MYSY_GV.ALLOWED_EXTENSIONS):
mycommon.myprint(str(inspect.stack()[0][3]) + " - : le format de fichier '"+str(basename2[1])+"' n'est pas autorisé. Les extentions autorisées sont : "+str(MYSY_GV.ALLOWED_EXTENSIONS))
return False, "le format de fichier '"+str(basename2[1])+"' n'est pas autorisé. Les extentions autorisées sont : "+str(MYSY_GV.ALLOWED_EXTENSIONS)
file_extention = str(basename2[1]).lower().strip()
new_basename2 = re.sub(r'[^a-zA-Z0-9]', '', str(basename2[0]))
timestr = time.strftime("%Y%m%d_%H%M%S")
local_base_name = str(new_basename2).replace('(', '').replace(')', '').replace(' ', '')
new_file_name = str(local_base_name) + "_" + str(timestr) +"_"+str(my_partner['recid'])+ "." + str(basename2[1])
file.filename = new_file_name
file.save(os.path.join(str(MYSY_GV.upload_folder), secure_filename(file.filename))) # t
Global_file_name = MYSY_GV.upload_folder+ file.filename
file_business_object = ""
if ("file_business_object" in diction.keys()):
if diction['file_business_object']:
file_business_object = diction['file_business_object']
token = ""
if ("token" in diction.keys()):
if diction['token']:
token = diction['token']
object_owner_collection = ""
if ("object_owner_collection" in diction.keys()):
if diction['object_owner_collection']:
object_owner_collection = diction['object_owner_collection']
object_owner_id = ""
if ("object_owner_id" in diction.keys()):
if diction['object_owner_id']:
object_owner_id = diction['object_owner_id']
document_display_order = ""
if ("document_display_order" in diction.keys()):
if diction['document_display_order']:
document_display_order = diction['document_display_order']
if (len(str(file.filename)) > MYSY_GV.FILE_NAME_MAX_SIZE):
mycommon.myprint(
str(inspect.stack()[0][
3]) + " - Fichier invalide, trop long = " + str(
file.filename) + ". le nom doit faire moins de 100 caractère")
return False, " - Nom de fichier trop long. Il doit faire moins de 100 caractères"
mydata = {}
mydata['file_business_object'] = file_business_object
mydata['date_update'] = str(datetime.now())
mydata['file_name'] = file.filename
mydata['full_path'] = Global_file_name
mydata['file_extention'] = file_extention
mydata['object_owner_collection'] = object_owner_collection
mydata['object_owner_id'] = object_owner_id
mydata['document_display_order'] = document_display_order
mydata['partner_owner_recid'] = str(my_partner['recid'])
mydata['valide'] = "1"
mydata['status'] = "0"
ret_val = MYSY_GV.dbname['download_files'].find_one_and_update({'file_business_object': str(file_business_object),
'object_owner_collection':str(object_owner_collection),
'object_owner_id':str(object_owner_id),
'partner_owner_recid': str(my_partner['recid']),
'valide': '1'},
{"$set": mydata},
upsert=True,
return_document=ReturnDocument.AFTER
)
return True, Global_file_name
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
return False, "Error "
"""
Cette fonction est la copie de la première a l'exception qu'il s'agit d'une utilisation interne
"""
def Internal_Usage_Store_User_Downloaded_File(Folder=None, diction=None):
try:
full_file_path = ""
diction = mycommon.strip_dictionary(diction)
"""
Verification des input acceptés
"""
field_list = ['token', 'file_business_object', 'file_name', 'status','object_owner_collection', 'object_owner_id',
'file_name_to_store']
incom_keys = diction.keys()
for val in incom_keys:
if val not in field_list and val.startswith('my_') is False:
mycommon.myprint(str(
inspect.stack()[0][3]) + " Le champ '" + val + "' n'existe pas")
return False, " Les informations fournies sont incorrectes",
"""
Verification des champs obligatoires
"""
field_list_obligatoire = ['token', 'file_business_object', 'object_owner_collection', 'object_owner_id', 'file_name_to_store' ]
for val in field_list_obligatoire:
if val not in diction:
mycommon.myprint(
str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ")
return False, " Les informations fournies sont incorrectes",
local_status, my_partner = mycommon.Check_Connexion_And_Return_Partner_Data(diction)
if (local_status is not True):
return local_status, my_partner
basename = os.path.basename(str(diction['file_name_to_store']))
basename2 = basename.split(".")
if (len(basename2) != 2):
mycommon.myprint(str(inspect.stack()[0][3]) + " - : Le nom du fichier est incorrect")
return False, "Le nom du fichier est incorrect"
if (str(basename2[1]).lower() not in MYSY_GV.ALLOWED_EXTENSIONS):
mycommon.myprint(str(inspect.stack()[0][3]) + " - : le format de fichier '"+str(basename2[1])+"' n'est pas autorisé. Les extentions autorisées sont : "+str(MYSY_GV.ALLOWED_EXTENSIONS))
return False, "le format de fichier '"+str(basename2[1])+"' n'est pas autorisé. Les extentions autorisées sont : "+str(MYSY_GV.ALLOWED_EXTENSIONS)
file_extention = str(basename2[1]).lower().strip()
new_basename2 = str(basename2[0])
timestr = time.strftime("%Y%m%d_%H%M%S")
local_base_name = str(new_basename2).replace('(', '').replace(')', '').replace(' ', '')
partial_file_name_without_folder = str(local_base_name) + "_" + str(timestr) +"_"+str(my_partner['recid'])+ "." + str(basename2[1])
new_file_name = MYSY_GV.upload_folder+str(local_base_name) + "_" + str(timestr) +"_"+str(my_partner['recid'])+ "." + str(basename2[1])
os.rename(str(diction['file_name_to_store']), new_file_name)
Global_file_name = new_file_name
file_business_object = ""
if ("file_business_object" in diction.keys()):
if diction['file_business_object']:
file_business_object = diction['file_business_object']
token = ""
if ("token" in diction.keys()):
if diction['token']:
token = diction['token']
object_owner_collection = ""
if ("object_owner_collection" in diction.keys()):
if diction['object_owner_collection']:
object_owner_collection = diction['object_owner_collection']
object_owner_id = ""
if ("object_owner_id" in diction.keys()):
if diction['object_owner_id']:
object_owner_id = diction['object_owner_id']
document_display_order = ""
if ("document_display_order" in diction.keys()):
if diction['document_display_order']:
document_display_order = diction['document_display_order']
mydata = {}
mydata['file_business_object'] = file_business_object
mydata['date_update'] = str(datetime.now())
mydata['file_name'] = partial_file_name_without_folder
mydata['full_path'] = Global_file_name
mydata['file_extention'] = file_extention
mydata['object_owner_collection'] = object_owner_collection
mydata['object_owner_id'] = object_owner_id
mydata['document_display_order'] = document_display_order
mydata['partner_owner_recid'] = str(my_partner['recid'])
mydata['valide'] = "1"
mydata['status'] = "0"
ret_val = MYSY_GV.dbname['download_files'].find_one_and_update({'file_business_object': str(file_business_object),
'object_owner_collection':str(object_owner_collection),
'object_owner_id':str(object_owner_id),
'partner_owner_recid': str(my_partner['recid']),
'file_name':str(partial_file_name_without_folder),
'valide': '1'},
{"$set": mydata},
upsert=True,
return_document=ReturnDocument.AFTER
)
return True, Global_file_name
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
return False, "Error "
"""
Fonction de recuperation des pièces jointes d'objet
"""
def Get_Stored_Downloaded_File(diction):
try:
full_file_path = ""
diction = mycommon.strip_dictionary(diction)
"""
Verification des input acceptés
"""
field_list = ['token', 'file_name']
incom_keys = diction.keys()
for val in incom_keys:
if val not in field_list and val.startswith('my_') is False:
mycommon.myprint(str(
inspect.stack()[0][3]) + " Le champ '" + val + "' n'est pas autorisé")
return False, " Les informations fournies sont incorrectes",
"""
Verification des champs obligatoires
"""
field_list_obligatoire = ['token', 'file_name']
for val in field_list_obligatoire:
if val not in diction:
mycommon.myprint(
str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ")
return False, " Les informations fournies sont incorrectes",
"""
Verification de l'identité et autorisation de l'entité qui
appelle cette API
"""
token = ""
if ("token" in diction.keys()):
if diction['token']:
token = diction['token']
local_status, my_partner = mycommon.Check_Connexion_And_Return_Partner_Data(diction)
if (local_status is not True):
return local_status, my_partner
file_name = ""
if ("file_name" in diction.keys()):
if diction['file_name']:
file_name = diction['file_name']
if(len(str(file_name)) > MYSY_GV.FILE_NAME_MAX_SIZE ):
mycommon.myprint(
str(inspect.stack()[0][
3]) + " - Fichier invalide, trop long = "+ str(file_name)+ ". le nom doit faire moins de 100 caractère")
return False, " - Nom de fichier trop long. Il doit faire moins de 100 caractères"
if( not file_name ):
mycommon.myprint(
str(inspect.stack()[0][
3]) + " - Fichier invalide = ", str(file_name))
return False, " - nom du fichier invalide "
full_path = MYSY_GV.upload_folder+str(file_name)
"""
Recuperation de la list des fichier dans la collection
"""
tmp_count = MYSY_GV.dbname['download_files'].count_documents({'full_path': str(full_path), 'valide': '1',
'partner_owner_recid':my_partner['recid']})
if (tmp_count <= 0):
mycommon.myprint(
str(inspect.stack()[0][
3]) + " - Fichier invalide = ", str(full_path))
return False, " - Fichier invalide "
if os.path.exists(full_path):
print(" ### full_path = ", full_path)
return True, send_file(full_path, as_attachment=True)
else:
mycommon.myprint(
str(inspect.stack()[0][
3]) + " - Fichier invalide (2)= ", str(full_path))
return False, " - Fichier invalide (2) "
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
return False, False
"""
Fonction qui test la recuperation
d'un fichier stocké sur le serveur
"""
UPLOAD_DIRECTORY = "./user_download_files"
def get_file():
"""Download a file."""
path2 = "test_file.pdf"
print(" ### PATH2 = ", path2)
if os.path.exists(UPLOAD_DIRECTORY + "/" + str(path2) ):
path = UPLOAD_DIRECTORY + "/" + str(path2)
print(" ### PATH = ", path)
return True, send_file(path, as_attachment=True)
else:
return False, False
def test_get_stored_file():
try:
filename = "./user_download_files/"
return True, "true"
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
return False, False
"""
Recuperation de la liste des fichiers stocké
"""
def Get_List_object_owner_collection_Stored_Files(diction):
try:
diction = mycommon.strip_dictionary(diction)
"""
Verification des input acceptés
"""
field_list = ['token', 'object_owner_collection', 'object_owner_id']
incom_keys = diction.keys()
for val in incom_keys:
if val not in field_list and val.startswith('my_') is False:
mycommon.myprint(str(
inspect.stack()[0][3]) + " Le champ '" + val + "' n'existe pas")
return False, " Les informations fournies sont incorrectes",
"""
Verification des champs obligatoires
"""
field_list_obligatoire = ['token', 'object_owner_collection', 'object_owner_id']
for val in field_list_obligatoire:
if val not in diction:
mycommon.myprint(
str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ")
return False, " Les informations fournies sont incorrectes",
"""
Verification de l'identité et autorisation de l'entité qui
appelle cette API
"""
token = ""
if ("token" in diction.keys()):
if diction['token']:
token = diction['token']
local_status, my_partner = mycommon.Check_Connexion_And_Return_Partner_Data(diction)
if (local_status is not True):
return local_status, my_partner
object_owner_collection = ""
if ("object_owner_collection" in diction.keys()):
if diction['object_owner_collection']:
object_owner_collection = diction['object_owner_collection']
object_owner_id = ""
if ("object_owner_id" in diction.keys()):
if diction['object_owner_id']:
object_owner_id = diction['object_owner_id']
my_query = {}
my_query['object_owner_collection'] = object_owner_collection
my_query['object_owner_id'] = object_owner_id
my_query['valide'] = "1"
my_query['partner_owner_recid'] = str(my_partner['recid'])
#print(" ##### myquery = " + str(my_query))
RetObject = []
nb_val = 0
for retval in MYSY_GV.dbname['download_files'].find(my_query).sort([("document_display_order", pymongo.DESCENDING)]):
ret_file = {}
if ("file_business_object" in retval.keys()):
ret_file['file_business_object'] = retval['file_business_object']
if ("object_owner_collection" in retval.keys()):
ret_file['object_owner_collection'] = retval['object_owner_collection']
if ("object_owner_id" in retval.keys()):
ret_file['object_owner_id'] = retval['object_owner_id']
if ("file_name" in retval.keys()):
ret_file['file_name'] = retval['file_name']
if ("status" in retval.keys()):
ret_file['status'] = retval['status']
if ("document_display_order" in retval.keys()):
ret_file['document_display_order'] = retval['document_display_order']
if ("full_path" in retval.keys()):
ret_file['full_path'] = retval['full_path']
if ("file_cononical_name" in retval.keys()):
ret_file['file_cononical_name'] = retval['file_cononical_name']
else:
ret_file['file_cononical_name'] = "Anonyme"
RetObject.append(mycommon.JSONEncoder().encode(ret_file))
nb_val = nb_val + 1
#print(" ### Get_List_object_owner_collection_Stored_Files = ", RetObject)
return True, RetObject
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
return False, False
"""
Suppression d'un fichier stocké
"""
def Delete_Stored_Downloaded_File(diction):
try:
full_file_path = ""
diction = mycommon.strip_dictionary(diction)
"""
Verification des input acceptés
"""
field_list = ['token', 'file_name']
incom_keys = diction.keys()
for val in incom_keys:
if val not in field_list and val.startswith('my_') is False:
mycommon.myprint(str(
inspect.stack()[0][3]) + " Le champ '" + val + "' n'est pas autorisé")
return False, " Les informations fournies sont incorrectes",
"""
Verification des champs obligatoires
"""
field_list_obligatoire = ['token', 'file_name']
for val in field_list_obligatoire:
if val not in diction:
mycommon.myprint(
str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ")
return False, " Les informations fournies sont incorrectes",
"""
Verification de l'identité et autorisation de l'entité qui
appelle cette API
"""
token = ""
if ("token" in diction.keys()):
if diction['token']:
token = diction['token']
file_name = ""
if ("file_name" in diction.keys()):
if diction['file_name']:
file_name = diction['file_name']
if(len(str(file_name)) > MYSY_GV.FILE_NAME_MAX_SIZE ):
mycommon.myprint(
str(inspect.stack()[0][
3]) + " - Fichier invalide, trop long = "+ str(file_name)+ ". le nom doit faire moins de 100 caractère")
return False, " - Nom de fichier trop long. Il doit faire moins de 100 caractères"
if( not file_name ):
mycommon.myprint(
str(inspect.stack()[0][
3]) + " - Fichier invalide = ", str(file_name))
return False, " - nom du fichier invalide "
full_path = MYSY_GV.upload_folder+str(file_name)
"""
Recuperation de la list des fichier dans la collection
"""
tmp_count = MYSY_GV.dbname['download_files'].count_documents({'full_path': str(full_path), 'valide': '1'})
if (tmp_count <= 0):
mycommon.myprint(
str(inspect.stack()[0][
3]) + " - Fichier invalide = ", str(full_path))
return False, " - Fichier invalide "
delete_doc = MYSY_GV.dbname['download_files'].delete_one({'full_path': str(full_path), 'valide': '1'})
if os.path.exists(full_path):
print(" ### full_path = ", full_path)
os.remove(full_path)
return True, "Le fichier a été supprimé"
else:
mycommon.myprint(
str(inspect.stack()[0][
3]) + " - Fichier invalide (2)= ", str(full_path))
return False, " - Fichier invalide (2) "
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
return False, " Impossible de supprimer le fichier"
"""
Suppression de toutes les pièces jointes d'une entité données (object_owner_id)
par exemple lorqu'on supprime un apprenant, il faudrait supprimer toutes
les pièces jointes associé à cette personne
"""
def Delete_Entity_Stored_Downloaded_File(diction):
try:
full_file_path = ""
diction = mycommon.strip_dictionary(diction)
"""
Verification des input acceptés
"""
field_list = ['token', 'object_owner_id', 'object_owner_collection']
incom_keys = diction.keys()
for val in incom_keys:
if val not in field_list and val.startswith('my_') is False:
mycommon.myprint(str(
inspect.stack()[0][3]) + " Le champ '" + val + "' n'est pas autorisé")
return False, " Les informations fournies sont incorrectes",
"""
Verification des champs obligatoires
"""
field_list_obligatoire = ['token', 'object_owner_id', 'object_owner_collection']
for val in field_list_obligatoire:
if val not in diction:
mycommon.myprint(
str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ")
return False, " Les informations fournies sont incorrectes",
"""
Verification de l'identité et autorisation de l'entité qui
appelle cette API
"""
token = ""
if ("token" in diction.keys()):
if diction['token']:
token = diction['token']
# Verifier que le l'entité existe (ON NE verifie PAS le valide ou locked)
is_entity_valide = MYSY_GV.dbname[str(diction['object_owner_collection'])].count_documents({'_id':ObjectId(str(diction['object_owner_id']))})
if( is_entity_valide <= 0 ):
mycommon.myprint(
str(inspect.stack()[0][3]) + " L'identifiant de l'entité est invalide ")
return False, " L'identifiant de l'entité est invalide "
for val in MYSY_GV.dbname['download_files'].find({'object_owner_collection':str(diction['object_owner_collection']),
'object_owner_id':str(diction['object_owner_id'])}):
if( "full_path" in val.keys() ):
file_name_to_remove = val['full_path']
try:
os.unlink(file_name_to_remove)
except:
mycommon.myprint(
str(inspect.stack()[0][3]) + " WARNING : Impossible de supprimer physiquement le fichier "+str(file_name_to_remove)+" ")
"""
Suppression des données dans la collection
"""
delete_row = MYSY_GV.dbname['download_files'].delete_many({'object_owner_collection': str(diction['object_owner_collection']),
'object_owner_id': str(diction['object_owner_id'])})
return True, " ("+str(delete_row.deleted_count)+" document(s) supprimé(s) "
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno))
return False, " Impossible de supprimer les fichier"