27/05/22 - 16h30

master
ChérifBALDE 2022-05-27 16:05:09 +02:00 committed by cherif
parent bfca587bb7
commit f982466549
8 changed files with 183 additions and 66 deletions

View File

@ -188,7 +188,7 @@ def Ela_Remove_Ponct_Special_Caractere(sentence):
text = re.sub(r"\-", " ", text)
list_noises = ['...', '.', ';', ',', ':', '!', '?', ')', '(', '[', ']', '\'', '"', '', '`','©', '',
'{', '}', '-', '=', '°', '#', '-', '/', '~', '&', '\\', '.', '^', '$', '*', '+','\\n',
'{', '}', '-', '=', '°', '#', '-', '/', '~', '&', '\\', '.', '^', '$', '*', '+','\\n','\n'
'?', '{', '}', '[', ']', '|', '(', ')', '-', '>', '<', '@','®', '', '«', '»']
sentence = text

View File

@ -97,4 +97,4 @@ QUERY_LIMIT_ROW = 50
'''
Cette varibal definit le nombre max
retour à faire sur page d'acceuil '''
MAINPAGE_QUERY_LIMIT_ROW = 100
MAINPAGE_QUERY_LIMIT_ROW = 50

View File

@ -5,9 +5,10 @@ Cette class gere les avis et articles sur une formation.
'''
from pymongo import MongoClient
import pymongo
import hashlib
import json
from bson import ObjectId
from unidecode import unidecode
import ela_index_bdd_classes as ela_index
import logging
@ -19,6 +20,7 @@ import inspect
import sys, os
import GlobalVariable as MYSY_GV
import user_message_mgt as umm
import Ela_Spacy as ES
@ -105,7 +107,7 @@ def get_articles_avis(diction):
try:
# Dictionnaire des champs utilisables
field_list = ['_id', 'token', 'user_ip', 'user_country_code', 'user_country_name',
field_list = ['internal_url', 'token', 'user_ip', 'user_country_code', 'user_country_name',
'user_city','user_postal', 'user_latitude', 'user_longitude', 'user_state']
incom_keys = diction.keys()
@ -125,7 +127,7 @@ def get_articles_avis(diction):
Une fois qu'on a controlé que toutes les clés mise dans l'API sont correcte. etape precedente,
On controle que les champs obligatoires sont presents dans la liste
'''
field_list_obligatoire = ['_id', 'token']
field_list_obligatoire = ['internal_url', 'token']
for val in field_list_obligatoire:
if val not in diction:
@ -175,17 +177,17 @@ def get_articles_avis(diction):
my_title_formation = diction['title_formation']
my_internal_id = ''
if ("_id" in diction.keys()):
if diction['_id']:
my_internal_id = diction['_id']
if ("internal_url" in diction.keys()):
if diction['internal_url']:
my_internal_id = diction['internal_url']
print(" My Data =" + str(mydata))
#print(" My Data =" + str(mydata))
coll_name = dbname['articles_avis']
insertObject = []
for x in coll_name.find({'valide': '1', 'locked': '0', '_id':ObjectId(my_internal_id)},
for x in coll_name.find({'valide': '1', 'locked': '0', 'internal_url':my_internal_id},
{ "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0,
"valide": 0, "locked": 0, }).sort([("title_formation",pymongo.ASCENDING),
("date_avis",pymongo.ASCENDING),
@ -290,6 +292,17 @@ def add_articles_avis(diction):
if ("title_formation" in diction.keys()):
if diction['title_formation']:
mydata['title_formation'] = diction['title_formation']
'''
Gestion de l'url interne. Pr un meilleur SEO, le champ 'internal_url' = titre avec des tiret
'''
my_internal_url = str(diction['title_formation']).replace("-","").replace(",", "").replace(" ", "").strip()
local_status, my_internal_url = ES.Ela_Remove_Ponct_Special_Caractere(my_internal_url)
my_internal_url = unidecode(my_internal_url.lower())
my_internal_url = my_internal_url.replace(" ","-")
suffix = hashlib.md5(my_internal_url.encode()).hexdigest()
mydata['internal_url'] = str(my_internal_url)+"-"+str(suffix[-3:])
if ("user_ip" in diction.keys()):
if diction['user_ip']:
@ -364,6 +377,7 @@ def add_articles_avis(diction):
mydata['indexed'] = '0'
mydata['indexed'] = '0'
mydata['date_update'] = str(datetime.now())
mydata['date_avis'] = str(datetime.now())
mydata['isalaune'] = "0"
print(mydata)
@ -711,7 +725,7 @@ def get_article_avis_alaune(diction):
for x in coll_name.find({'valide': '1', 'locked': '0', 'isalaune': '1'},
{"indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "isalaune": 0,
"valide": 0, "locked": 0, }).sort(
[ ("note", pymongo.DESCENDING), ("date_avis", pymongo.DESCENDING),]):
[ ("date_avis", pymongo.DESCENDING), ("note", pymongo.DESCENDING)]):
val = x['qualite']
if (len(x['qualite']) > MYSY_GV.MAX_CARACT_DEDUIT):
@ -730,7 +744,7 @@ def get_article_avis_alaune(diction):
user = x
insertObject.append(JSONEncoder().encode(user))
#print(" getObject = ", str(insertObject))
print(" getObject = ", str(insertObject))
return True, insertObject

View File

@ -784,7 +784,7 @@ def get_class(diction):
# Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste
# field_list.
'''
field_list = ['external_code', 'token', 'title', 'valide', 'locked', 'user_ip', 'user_country_code', 'user_country_name', 'user_city',
field_list = ['internal_url', 'token', 'title', 'valide', 'locked', 'user_ip', 'user_country_code', 'user_country_name', 'user_city',
'user_postal', 'user_latitude', 'user_longitude', 'user_state']
incom_keys = diction.keys()
@ -806,13 +806,13 @@ def get_class(diction):
# recuperation des paramettre
mydata = {}
my_external_code = ""
my_internal_url = ""
my_token = ""
if ("external_code" in diction.keys()):
if diction['external_code']:
my_external_code = diction['external_code']
if ("internal_url" in diction.keys()):
if diction['internal_url']:
my_internal_url = diction['internal_url']
if ("token" in diction.keys()):
if diction['token']:
@ -823,10 +823,10 @@ def get_class(diction):
Gestion des filters.
'''
external_code_crit = {}
if ("external_code" in diction.keys()):
if diction['external_code']:
external_code_crit['external_code'] = diction['external_code']
internal_url_crit = {}
if ("internal_url" in diction.keys()):
if diction['internal_url']:
internal_url_crit['internal_url'] = diction['internal_url']
title_crit = {}
if ("title" in diction.keys()):
@ -866,9 +866,10 @@ def get_class(diction):
RetObject = []
filt_external_code = {}
if ("external_code" in diction.keys()):
filt_external_code = {'external_code':str(diction['external_code'])}
external_code = str(diction['external_code'])
internal_url = ""
if ("internal_url" in diction.keys()):
filt_external_code = {'internal_url':str(diction['internal_url'])}
internal_url = str(diction['internal_url'])
#print(' ICICICIC '+str(filt_external_code))
@ -878,10 +879,10 @@ def get_class(diction):
print(" #### avant requete get partner_owner_recid ="+str(user_recid)+
" filt_external_code = "+str(filt_external_code)+
" internal_url = "+str(my_internal_url)+
" filt_title = "+str(filt_title))
for retVal in coll_name.find({'valide':'1','locked':'0','external_code':external_code},
for retVal in coll_name.find({'valide':'1','locked':'0','internal_url':internal_url},
{"_id": 0, "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0,
"valide": 0, "locked": 0, "partner_owner_recid": 0, }
):
@ -931,7 +932,7 @@ def get_class(diction):
retVal['pedagogie'] = tmp_str[:MYSY_GV.MAX_CARACT_DETAIL] + " ..."
mycommon.myprint(str(retVal))
#mycommon.myprint(str(retVal))
user = retVal
RetObject.append(JSONEncoder().encode(user))
@ -960,7 +961,7 @@ def get_partner_class(diction):
# Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste
# field_list.
'''
field_list = ['external_code', 'token', 'title', 'valide', 'locked']
field_list = ['internal_url', 'token', 'title', 'valide', 'locked']
incom_keys = diction.keys()
for val in incom_keys:
if val not in field_list:
@ -984,9 +985,9 @@ def get_partner_class(diction):
my_token = ""
if ("external_code" in diction.keys()):
if diction['external_code']:
my_external_code = diction['external_code']
if ("internal_url" in diction.keys()):
if diction['internal_url']:
my_internal_code = diction['internal_url']
if ("token" in diction.keys()):
if diction['token']:
@ -997,9 +998,9 @@ def get_partner_class(diction):
'''
external_code_crit = {}
if ("external_code" in diction.keys()):
if diction['external_code']:
external_code_crit['external_code'] = diction['external_code']
if ("internal_url" in diction.keys()):
if diction['internal_url']:
external_code_crit['internal_url'] = diction['internal_url']
title_crit = {}
if ("title" in diction.keys()):
@ -1043,7 +1044,7 @@ def get_partner_class(diction):
RetObject = []
filt_external_code = {}
if ("external_code" in diction.keys()):
filt_external_code = {'external_code':{'$regex':str(diction['external_code'])}}
filt_external_code = {'internal_url':{'$regex':str(diction['internal_url'])}}
filt_title = {}
if ("title" in diction.keys()):
@ -1056,8 +1057,10 @@ def get_partner_class(diction):
" filt_title = "+str(filt_title))
val_tmp = 1
for retVal in coll_name.find(({"$and": [ {'valide':'1'},{'locked':'0'},
{'partner_owner_recid':user_recid} ,filt_external_code, filt_title ]})):
for retVal in coll_name.find( {'valide':'1'},{'locked':'0'},
{'partner_owner_recid':user_recid} ,
filt_external_code,
filt_title):
mycommon.myprint(str(retVal))
user = retVal
user['id'] = str(val_tmp)

View File

@ -1,34 +1,34 @@
,index,mots,occurence,moyenne,id_formation,source_field
0,0,infrastructure,2,0.12,"
0,0,ressource,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default
1,1,concevoir,2,0.12,"
1,1,locau,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default
2,2,decouvrez,1,0.06,"
2,2,azure,3,0.18,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default
3,3,ressource,1,0.06,"
3,3,comment,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default
4,4,reseau,3,0.18,"
4,4,decouvrez,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default
5,5,connecter,1,0.06,"
5,5,securisee,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default
6,6,azure,3,0.18,"
6,6,connecter,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default
7,7,flexible,1,0.06,"
7,7,concevoir,2,0.12,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default
8,8,comment,1,0.06,"
8,8,flexible,1,0.06,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default
9,9,locau,1,0.06,"
9,9,reseau,3,0.18,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default
10,10,securisee,1,0.06,"
10,10,infrastructure,2,0.12,"
concevoir-une-infrastructure-de-reseau-dans-azure
-4",default

1 index mots occurence moyenne id_formation source_field
2 0 0 infrastructure ressource 2 1 0.12 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
3 1 1 concevoir locau 2 1 0.12 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
4 2 2 decouvrez azure 1 3 0.06 0.18 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
5 3 3 ressource comment 1 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
6 4 4 reseau decouvrez 3 1 0.18 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
7 5 5 connecter securisee 1 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
8 6 6 azure connecter 3 1 0.18 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
9 7 7 flexible concevoir 1 2 0.06 0.12 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
10 8 8 comment flexible 1 0.06 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
11 9 9 locau reseau 1 3 0.06 0.18 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
12 10 10 securisee infrastructure 1 2 0.06 0.12 concevoir-une-infrastructure-de-reseau-dans-azure -4 default
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

View File

@ -1,14 +1,14 @@
mots ... source_field
0 infrastructure ... default
1 concevoir ... default
2 decouvrez ... default
3 ressource ... default
4 reseau ... default
5 connecter ... default
6 azure ... default
7 flexible ... default
8 comment ... default
9 locau ... default
10 securisee ... default
0 ressource ... default
1 locau ... default
2 azure ... default
3 comment ... default
4 decouvrez ... default
5 securisee ... default
6 connecter ... default
7 concevoir ... default
8 flexible ... default
9 reseau ... default
10 infrastructure ... default
[11 rows x 5 columns]

20
main.py
View File

@ -1036,15 +1036,29 @@ def OrderClassRandomly():
return jsonify(status=status, message=result)
@app.route('/myclass/api/testch/', methods=['GET','POST'])
@app.route('/myclass/api/Migration_internal_url/', methods=['GET','POST'])
@crossdomain(origin='*')
def testch():
def Migration_internal_url():
# On recupere le corps (payload) de la requete
payload = request.form.to_dict()
print(" ### payload = ", payload)
status, result = mycommon.FormationAlaUne()
status, result = mycommon.Migration_internal_url()
return jsonify(status=status, message=result)
'''
Cette API va aller chercher dans la collection "elaindex"
les mots qui contiennent le mot "mot"'''
@app.route('/myclass/api/GetMotFromElaIndex/', methods=['GET','POST'])
@crossdomain(origin='*')
def GetMotFromElaIndex():
# On recupere le corps (payload) de la requete
payload = request.form.to_dict()
print(" ### payload = ", payload)
status, result = mycommon.GetMotFromElaIndex(payload)
return jsonify(status=status, message=result)
if __name__ == '__main__':
print(" debut api")

View File

@ -1,4 +1,5 @@
from pymongo import MongoClient
import pymongo
from difflib import SequenceMatcher
import textdistance
from datetime import datetime
@ -845,11 +846,96 @@ def FormationAlaUne():
return_document=ReturnDocument.AFTER
)
return True, "ok"
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "KO"
def Migration_internal_url():
try:
coll_name = MYSY_GV.dbname['myclass']
for retVal in coll_name.find({}):
user = retVal
result = coll_name.update_many(
{'_id':ObjectId(str(user['_id']))},
{
"$set": {"internal_url": str(user['_id'])}
})
'''print("raw:", result.raw_result)
print("acknowledged:", result.acknowledged)
print("matched_count:", result.matched_count)'''
return True, "ok"
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "KO"
'''
Cette fonction prends un mot de X digit
et retourne le top X des mot ayant le meme digit
/!\ cette fonction est glement utilisee dans le car des
des recherches vides
'''
def GetMotFromElaIndex(diction):
try :
'''
# Verification que les champs reçus dans l'API sont bien dans la liste des champs autorisés
# Cela evite le cas ou une entité tierce ajouter les valeurs inconnu dans l'API
# Ici on doit mettre tous les champs possible (obligatoire ou non) de la BDD dans la liste
# field_list.
'''
field_list = ['mot', 'domaine']
incom_keys = diction.keys()
for val in incom_keys:
if val not in field_list:
mycommon.myprint(str(inspect.stack()[0][
3]) + " - Le champ '" + val + "' n'existe pas, Creation formation annulée")
return False, []
'''
Une fois qu'on a controlé que toutes les clés mise dans l'API sont correcte. etape precedente,
On controle que les champs obligatoires sont presents dans la liste
'''
field_list_obligatoire = ['mot']
for val in field_list_obligatoire:
if val not in diction:
mycommon.myprint(
str(inspect.stack()[0][3]) + " - La valeur '" + val + "' n'est pas presente dans liste ")
return False, []
coll_name = MYSY_GV.dbname['elaindex']
training_mots = []
mot = ""
if ("mot" in diction.keys()):
if diction['mot']:
mot = diction['mot']
if( len(mot) < 3 ):
return True, []
search = str(mot)
search_expr = re.compile(f".*{search}.*", re.I)
print(" #### mot recu "+mot+" search_expr = "+str(search_expr))
for x in coll_name.find({'mots': {'$regex': search_expr}}).sort([("occurence", pymongo.DESCENDING)]).limit(5):
if( x['mots'] not in training_mots ):
training_mots.append(x['mots'])
return True, training_mots
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
print(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, []