27/05/22 - 16h30

master
ChérifBALDE 2022-05-27 16:05:09 +02:00 committed by cherif
parent bfca587bb7
commit f982466549
8 changed files with 183 additions and 66 deletions

View File

@ -188,7 +188,7 @@ def Ela_Remove_Ponct_Special_Caractere(sentence):
text = re.sub(r"\-", " ", text) text = re.sub(r"\-", " ", text)
list_noises = ['...', '.', ';', ',', ':', '!', '?', ')', '(', '[', ']', '\'', '"', '', '`','©', '', list_noises = ['...', '.', ';', ',', ':', '!', '?', ')', '(', '[', ']', '\'', '"', '', '`','©', '',
'{', '}', '-', '=', '°', '#', '-', '/', '~', '&', '\\', '.', '^', '$', '*', '+','\\n', '{', '}', '-', '=', '°', '#', '-', '/', '~', '&', '\\', '.', '^', '$', '*', '+','\\n','\n'
'?', '{', '}', '[', ']', '|', '(', ')', '-', '>', '<', '@','®', '', '«', '»'] '?', '{', '}', '[', ']', '|', '(', ')', '-', '>', '<', '@','®', '', '«', '»']
sentence = text sentence = text

View File

@ -97,4 +97,4 @@ QUERY_LIMIT_ROW = 50
''' '''
Cette varibal definit le nombre max Cette varibal definit le nombre max
retour à faire sur page d'acceuil ''' retour à faire sur page d'acceuil '''
MAINPAGE_QUERY_LIMIT_ROW = 100 MAINPAGE_QUERY_LIMIT_ROW = 50

View File

@ -5,9 +5,10 @@ Cette class gere les avis et articles sur une formation.
''' '''
from pymongo import MongoClient from pymongo import MongoClient
import pymongo import pymongo
import hashlib
import json import json
from bson import ObjectId from bson import ObjectId
from unidecode import unidecode
import ela_index_bdd_classes as ela_index import ela_index_bdd_classes as ela_index
import logging import logging
@ -19,6 +20,7 @@ import inspect
import sys, os import sys, os
import GlobalVariable as MYSY_GV import GlobalVariable as MYSY_GV
import user_message_mgt as umm import user_message_mgt as umm
import Ela_Spacy as ES
@ -105,7 +107,7 @@ def get_articles_avis(diction):
try: try:
# Dictionnaire des champs utilisables # Dictionnaire des champs utilisables
field_list = ['_id', 'token', 'user_ip', 'user_country_code', 'user_country_name', field_list = ['internal_url', 'token', 'user_ip', 'user_country_code', 'user_country_name',
'user_city','user_postal', 'user_latitude', 'user_longitude', 'user_state'] 'user_city','user_postal', 'user_latitude', 'user_longitude', 'user_state']
incom_keys = diction.keys() incom_keys = diction.keys()
@ -125,7 +127,7 @@ def get_articles_avis(diction):
Une fois qu'on a controlé que toutes les clés mise dans l'API sont correcte. etape precedente, Une fois qu'on a controlé que toutes les clés mise dans l'API sont correcte. etape precedente,
On controle que les champs obligatoires sont presents dans la liste On controle que les champs obligatoires sont presents dans la liste
''' '''
field_list_obligatoire = ['_id', 'token'] field_list_obligatoire = ['internal_url', 'token']
for val in field_list_obligatoire: for val in field_list_obligatoire:
if val not in diction: if val not in diction:
@ -175,17 +177,17 @@ def get_articles_avis(diction):
my_title_formation = diction['title_formation'] my_title_formation = diction['title_formation']
my_internal_id = '' my_internal_id = ''
if ("_id" in diction.keys()): if ("internal_url" in diction.keys()):
if diction['_id']: if diction['internal_url']:
my_internal_id = diction['_id'] my_internal_id = diction['internal_url']
print(" My Data =" + str(mydata)) #print(" My Data =" + str(mydata))
coll_name = dbname['articles_avis'] coll_name = dbname['articles_avis']
insertObject = [] insertObject = []
for x in coll_name.find({'valide': '1', 'locked': '0', '_id':ObjectId(my_internal_id)}, for x in coll_name.find({'valide': '1', 'locked': '0', 'internal_url':my_internal_id},
{ "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0, { "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0,
"valide": 0, "locked": 0, }).sort([("title_formation",pymongo.ASCENDING), "valide": 0, "locked": 0, }).sort([("title_formation",pymongo.ASCENDING),
("date_avis",pymongo.ASCENDING), ("date_avis",pymongo.ASCENDING),
@ -290,6 +292,17 @@ def add_articles_avis(diction):
if ("title_formation" in diction.keys()): if ("title_formation" in diction.keys()):
if diction['title_formation']: if diction['title_formation']:
mydata['title_formation'] = diction['title_formation'] mydata['title_formation'] = diction['title_formation']
'''
Gestion de l'url interne. Pr un meilleur SEO, le champ 'internal_url' = titre avec des tiret
'''
my_internal_url = str(diction['title_formation']).replace("-","").replace(",", "").replace(" ", "").strip()
local_status, my_internal_url = ES.Ela_Remove_Ponct_Special_Caractere(my_internal_url)
my_internal_url = unidecode(my_internal_url.lower())
my_internal_url = my_internal_url.replace(" ","-")
suffix = hashlib.md5(my_internal_url.encode()).hexdigest()
mydata['internal_url'] = str(my_internal_url)+"-"+str(suffix[-3:])
if ("user_ip" in diction.keys()): if ("user_ip" in diction.keys()):
if diction['user_ip']: if diction['user_ip']:
@ -364,6 +377,7 @@ def add_articles_avis(diction):
mydata['indexed'] = '0' mydata['indexed'] = '0'
mydata['indexed'] = '0' mydata['indexed'] = '0'
mydata['date_update'] = str(datetime.now()) mydata['date_update'] = str(datetime.now())
mydata['date_avis'] = str(datetime.now())
mydata['isalaune'] = "0" mydata['isalaune'] = "0"
print(mydata) print(mydata)
@ -711,7 +725,7 @@ def get_article_avis_alaune(diction):
for x in coll_name.find({'valide': '1', 'locked': '0', 'isalaune': '1'}, for x in coll_name.find({'valide': '1', 'locked': '0', 'isalaune': '1'},
{"indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "isalaune": 0, {"indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "isalaune": 0,
"valide": 0, "locked": 0, }).sort( "valide": 0, "locked": 0, }).sort(
[ ("note", pymongo.DESCENDING), ("date_avis", pymongo.DESCENDING),]): [ ("date_avis", pymongo.DESCENDING), ("note", pymongo.DESCENDING)]):
val = x['qualite'] val = x['qualite']
if (len(x['qualite']) > MYSY_GV.MAX_CARACT_DEDUIT): if (len(x['qualite']) > MYSY_GV.MAX_CARACT_DEDUIT):
@ -730,7 +744,7 @@ def get_article_avis_alaune(diction):
user = x user = x
insertObject.append(JSONEncoder().encode(user)) insertObject.append(JSONEncoder().encode(user))
#print(" getObject = ", str(insertObject)) print(" getObject = ", str(insertObject))
return True, insertObject return True, insertObject

View File

@ -784,7 +784,7 @@ def get_class(diction):
# Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste # Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste
# field_list. # field_list.
''' '''
field_list = ['external_code', 'token', 'title', 'valide', 'locked', 'user_ip', 'user_country_code', 'user_country_name', 'user_city', field_list = ['internal_url', 'token', 'title', 'valide', 'locked', 'user_ip', 'user_country_code', 'user_country_name', 'user_city',
'user_postal', 'user_latitude', 'user_longitude', 'user_state'] 'user_postal', 'user_latitude', 'user_longitude', 'user_state']
incom_keys = diction.keys() incom_keys = diction.keys()
@ -806,13 +806,13 @@ def get_class(diction):
# recuperation des paramettre # recuperation des paramettre
mydata = {} mydata = {}
my_external_code = "" my_internal_url = ""
my_token = "" my_token = ""
if ("external_code" in diction.keys()): if ("internal_url" in diction.keys()):
if diction['external_code']: if diction['internal_url']:
my_external_code = diction['external_code'] my_internal_url = diction['internal_url']
if ("token" in diction.keys()): if ("token" in diction.keys()):
if diction['token']: if diction['token']:
@ -823,10 +823,10 @@ def get_class(diction):
Gestion des filters. Gestion des filters.
''' '''
external_code_crit = {} internal_url_crit = {}
if ("external_code" in diction.keys()): if ("internal_url" in diction.keys()):
if diction['external_code']: if diction['internal_url']:
external_code_crit['external_code'] = diction['external_code'] internal_url_crit['internal_url'] = diction['internal_url']
title_crit = {} title_crit = {}
if ("title" in diction.keys()): if ("title" in diction.keys()):
@ -866,9 +866,10 @@ def get_class(diction):
RetObject = [] RetObject = []
filt_external_code = {} filt_external_code = {}
if ("external_code" in diction.keys()): internal_url = ""
filt_external_code = {'external_code':str(diction['external_code'])} if ("internal_url" in diction.keys()):
external_code = str(diction['external_code']) filt_external_code = {'internal_url':str(diction['internal_url'])}
internal_url = str(diction['internal_url'])
#print(' ICICICIC '+str(filt_external_code)) #print(' ICICICIC '+str(filt_external_code))
@ -878,10 +879,10 @@ def get_class(diction):
print(" #### avant requete get partner_owner_recid ="+str(user_recid)+ print(" #### avant requete get partner_owner_recid ="+str(user_recid)+
" filt_external_code = "+str(filt_external_code)+ " internal_url = "+str(my_internal_url)+
" filt_title = "+str(filt_title)) " filt_title = "+str(filt_title))
for retVal in coll_name.find({'valide':'1','locked':'0','external_code':external_code}, for retVal in coll_name.find({'valide':'1','locked':'0','internal_url':internal_url},
{"_id": 0, "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0, {"_id": 0, "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0,
"valide": 0, "locked": 0, "partner_owner_recid": 0, } "valide": 0, "locked": 0, "partner_owner_recid": 0, }
): ):
@ -931,7 +932,7 @@ def get_class(diction):
retVal['pedagogie'] = tmp_str[:MYSY_GV.MAX_CARACT_DETAIL] + " ..." retVal['pedagogie'] = tmp_str[:MYSY_GV.MAX_CARACT_DETAIL] + " ..."
mycommon.myprint(str(retVal)) #mycommon.myprint(str(retVal))
user = retVal user = retVal
RetObject.append(JSONEncoder().encode(user)) RetObject.append(JSONEncoder().encode(user))
@ -960,7 +961,7 @@ def get_partner_class(diction):
# Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste # Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste
# field_list. # field_list.
''' '''
field_list = ['external_code', 'token', 'title', 'valide', 'locked'] field_list = ['internal_url', 'token', 'title', 'valide', 'locked']
incom_keys = diction.keys() incom_keys = diction.keys()
for val in incom_keys: for val in incom_keys:
if val not in field_list: if val not in field_list:
@ -984,9 +985,9 @@ def get_partner_class(diction):
my_token = "" my_token = ""
if ("external_code" in diction.keys()): if ("internal_url" in diction.keys()):
if diction['external_code']: if diction['internal_url']:
my_external_code = diction['external_code'] my_internal_code = diction['internal_url']
if ("token" in diction.keys()): if ("token" in diction.keys()):
if diction['token']: if diction['token']:
@ -997,9 +998,9 @@ def get_partner_class(diction):
''' '''
external_code_crit = {} external_code_crit = {}
if ("external_code" in diction.keys()): if ("internal_url" in diction.keys()):
if diction['external_code']: if diction['internal_url']:
external_code_crit['external_code'] = diction['external_code'] external_code_crit['internal_url'] = diction['internal_url']
title_crit = {} title_crit = {}
if ("title" in diction.keys()): if ("title" in diction.keys()):
@ -1043,7 +1044,7 @@ def get_partner_class(diction):
RetObject = [] RetObject = []
filt_external_code = {} filt_external_code = {}
if ("external_code" in diction.keys()): if ("external_code" in diction.keys()):
filt_external_code = {'external_code':{'$regex':str(diction['external_code'])}} filt_external_code = {'internal_url':{'$regex':str(diction['internal_url'])}}
filt_title = {} filt_title = {}
if ("title" in diction.keys()): if ("title" in diction.keys()):
@ -1056,8 +1057,10 @@ def get_partner_class(diction):
" filt_title = "+str(filt_title)) " filt_title = "+str(filt_title))
val_tmp = 1 val_tmp = 1
for retVal in coll_name.find(({"$and": [ {'valide':'1'},{'locked':'0'}, for retVal in coll_name.find( {'valide':'1'},{'locked':'0'},
{'partner_owner_recid':user_recid} ,filt_external_code, filt_title ]})): {'partner_owner_recid':user_recid} ,
filt_external_code,
filt_title):
mycommon.myprint(str(retVal)) mycommon.myprint(str(retVal))
user = retVal user = retVal
user['id'] = str(val_tmp) user['id'] = str(val_tmp)

View File

@ -1,34 +1,34 @@
,index,mots,occurence,moyenne,id_formation,source_field ,index,mots,occurence,moyenne,id_formation,source_field
0,0,infrastructure,2,0.12," 0,0,ressource,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default
1,1,concevoir,2,0.12," 1,1,locau,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default
2,2,decouvrez,1,0.06," 2,2,azure,3,0.18,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default
3,3,ressource,1,0.06," 3,3,comment,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default
4,4,reseau,3,0.18," 4,4,decouvrez,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default
5,5,connecter,1,0.06," 5,5,securisee,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default
6,6,azure,3,0.18," 6,6,connecter,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default
7,7,flexible,1,0.06," 7,7,concevoir,2,0.12,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default
8,8,comment,1,0.06," 8,8,flexible,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default
9,9,locau,1,0.06," 9,9,reseau,3,0.18,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default
10,10,securisee,1,0.06," 10,10,infrastructure,2,0.12,"
concevoir-une-infrastructure-de-reseau-dans-azure concevoir-une-infrastructure-de-reseau-dans-azure
-4",default -4",default

1 index mots occurence moyenne id_formation source_field
2 0 0 infrastructure ressource 2 1 0.12 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
3 1 1 concevoir locau 2 1 0.12 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
4 2 2 decouvrez azure 1 3 0.06 0.18 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
5 3 3 ressource comment 1 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
6 4 4 reseau decouvrez 3 1 0.18 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
7 5 5 connecter securisee 1 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
8 6 6 azure connecter 3 1 0.18 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
9 7 7 flexible concevoir 1 2 0.06 0.12 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
10 8 8 comment flexible 1 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
11 9 9 locau reseau 1 3 0.06 0.18 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
12 10 10 securisee infrastructure 1 2 0.06 0.12 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

View File

@ -1,14 +1,14 @@
mots ... source_field mots ... source_field
0 infrastructure ... default 0 ressource ... default
1 concevoir ... default 1 locau ... default
2 decouvrez ... default 2 azure ... default
3 ressource ... default 3 comment ... default
4 reseau ... default 4 decouvrez ... default
5 connecter ... default 5 securisee ... default
6 azure ... default 6 connecter ... default
7 flexible ... default 7 concevoir ... default
8 comment ... default 8 flexible ... default
9 locau ... default 9 reseau ... default
10 securisee ... default 10 infrastructure ... default
[11 rows x 5 columns] [11 rows x 5 columns]

20
main.py
View File

@ -1036,15 +1036,29 @@ def OrderClassRandomly():
return jsonify(status=status, message=result) return jsonify(status=status, message=result)
@app.route('/myclass/api/testch/', methods=['GET','POST']) @app.route('/myclass/api/Migration_internal_url/', methods=['GET','POST'])
@crossdomain(origin='*') @crossdomain(origin='*')
def testch(): def Migration_internal_url():
# On recupere le corps (payload) de la requete # On recupere le corps (payload) de la requete
payload = request.form.to_dict() payload = request.form.to_dict()
print(" ### payload = ", payload) print(" ### payload = ", payload)
status, result = mycommon.FormationAlaUne() status, result = mycommon.Migration_internal_url()
return jsonify(status=status, message=result) return jsonify(status=status, message=result)
'''
Cette API va aller chercher dans la collection "elaindex"
les mots qui contiennent le mot "mot"'''
@app.route('/myclass/api/GetMotFromElaIndex/', methods=['GET','POST'])
@crossdomain(origin='*')
def GetMotFromElaIndex():
# On recupere le corps (payload) de la requete
payload = request.form.to_dict()
print(" ### payload = ", payload)
status, result = mycommon.GetMotFromElaIndex(payload)
return jsonify(status=status, message=result)
if __name__ == '__main__': if __name__ == '__main__':
print(" debut api") print(" debut api")

View File

@ -1,4 +1,5 @@
from pymongo import MongoClient
import pymongo
from difflib import SequenceMatcher from difflib import SequenceMatcher
import textdistance import textdistance
from datetime import datetime from datetime import datetime
@ -845,11 +846,96 @@ def FormationAlaUne():
return_document=ReturnDocument.AFTER return_document=ReturnDocument.AFTER
) )
return True, "ok" return True, "ok"
except Exception as e: except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info() exc_type, exc_obj, exc_tb = sys.exc_info()
print(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) print(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "KO" return False, "KO"
def Migration_internal_url():
try:
coll_name = MYSY_GV.dbname['myclass']
for retVal in coll_name.find({}):
user = retVal
result = coll_name.update_many(
{'_id':ObjectId(str(user['_id']))},
{
"$set": {"internal_url": str(user['_id'])}
})
'''print("raw:", result.raw_result)
print("acknowledged:", result.acknowledged)
print("matched_count:", result.matched_count)'''
return True, "ok"
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "KO"
'''
Cette fonction prends un mot de X digit
et retourne le top X des mot ayant le meme digit
/!\ cette fonction est glement utilisee dans le car des
des recherches vides
'''
def GetMotFromElaIndex(diction):
try :
'''
# Verification que les champs reçus dans l'API sont bien dans la liste des champs autorisés
# Cela evite le cas ou une entité tierce ajouter les valeurs inconnu dans l'API
# Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste
# field_list.
'''
field_list = ['mot', 'domaine']
incom_keys = diction.keys()
for val in incom_keys:
if val not in field_list:
mycommon.myprint(str(inspect.stack()[0][
3]) + " - Le champ '" + val + "' n'existe pas, Creation formation annulée")
return False, []
'''
Une fois qu'on a controlé que toutes les clés mise dans l'API sont correcte. etape precedente,
On controle que les champs obligatoires sont presents dans la liste
'''
field_list_obligatoire = ['mot']
for val in field_list_obligatoire:
if val not in diction:
mycommon.myprint(
str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ")
return False, []
coll_name = MYSY_GV.dbname['elaindex']
training_mots = []
mot = ""
if ("mot" in diction.keys()):
if diction['mot']:
mot = diction['mot']
if( len(mot) < 3 ):
return True, []
search = str(mot)
search_expr = re.compile(f".*{search}.*", re.I)
print(" #### mot recu "+mot+" search_expr = "+str(search_expr))
for x in coll_name.find({'mots': {'$regex': search_expr}}).sort([("occurence", pymongo.DESCENDING)]).limit(5):
if( x['mots'] not in training_mots ):
training_mots.append(x['mots'])
return True, training_mots
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, []