28/04/22 - 13h30

master
ChérifBALDE 2022-04-28 14:16:03 +02:00 committed by cherif
parent 4d16f4039f
commit 934b21de5a
11 changed files with 566 additions and 214 deletions

View File

@ -23,6 +23,7 @@ import logging
from datetime import datetime from datetime import datetime
import prj_common as mycommon import prj_common as mycommon
import secrets import secrets
import GlobalVariable as MYSY_GV
class JSONEncoder(json.JSONEncoder): class JSONEncoder(json.JSONEncoder):
@ -31,7 +32,6 @@ class JSONEncoder(json.JSONEncoder):
return str(o) return str(o)
return json.JSONEncoder.default(self, o) return json.JSONEncoder.default(self, o)
CONNECTION_STRING = "mongodb://localhost/cherifdb"
''' '''
@ -40,7 +40,7 @@ Cette fonction supprimer tous les tokens d'un user
''' '''
def Dev_del_user_token(diction): def Dev_del_user_token(diction):
try: try:
client = MongoClient(CONNECTION_STRING) client = MongoClient(MYSY_GV.CONNECTION_STRING)
''' '''
# Verification que les champs reçus dans l'API sont bien dans la liste des champs autorisés # Verification que les champs reçus dans l'API sont bien dans la liste des champs autorisés

View File

@ -1,15 +1,10 @@
import spacy
from spacy import displacy
import nltk
from nltk.corpus import stopwords
from nltk.stem.snowball import SnowballStemmer
import datetime import datetime
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import pymongo import pymongo
from pymongo import MongoClient from pymongo import MongoClient
from collections import Counter from collections import Counter
import ela_spacy_common as lsc
import prj_common as mycommon import prj_common as mycommon
from unidecode import unidecode from unidecode import unidecode
import inspect import inspect
@ -17,32 +12,11 @@ import sys, os
from autocorrect import Speller from autocorrect import Speller
from datetime import datetime from datetime import datetime
import re import re
from spellchecker import SpellChecker import GlobalVariable as MYSY_GV
CONNECTION_STRING = "mongodb://localhost/cherifdb"
## Gle Variables
stemmer = SnowballStemmer(language='french')
nlp = spacy.load("fr_core_news_sm")
spell = SpellChecker(language='fr')
token_fr_pontuation = []
#assign the default stopwords list to a variable
STOP_WORDS = spacy.lang.fr.stop_words.STOP_WORDS
stopWords = set(stopwords.words('french'))
sentence = "Bouygues a eu une coupure de réseau à Marseille chez ses clients marseillais et son couteau"
lsc.update_stopWords(stopWords)
#print(type(stopWords))
lsc.update_token_fr_pontuation(token_fr_pontuation)
spell_fr = Speller(lang='fr')
CONNECTION_STRING = "mongodb://localhost/cherifdb"
client = MongoClient(CONNECTION_STRING)
dbname = client['cherifdb']
#print("token_fr_pontuation") #print("token_fr_pontuation")
#print(token_fr_pontuation) #print(token_fr_pontuation)
@ -51,21 +25,6 @@ dbname = client['cherifdb']
initialisation initialisation
''' '''
def init_ch():
stemmer = SnowballStemmer(language='french')
nlp = spacy.load("fr_core_news_sm")
token_fr_pontuation = []
# assign the default stopwords list to a variable
STOP_WORDS = spacy.lang.fr.stop_words.STOP_WORDS
stopWords = set(stopwords.words('french'))
sentence = "Bouygues a eu une coupure de réseau à Marseille chez ses clients marseillais et son couteau"
lsc.update_stopWords(stopWords)
# print(type(stopWords))
lsc.update_token_fr_pontuation(token_fr_pontuation)
''' '''
@ -94,19 +53,19 @@ def correct_fr_word(word):
try: try:
mydata = {} mydata = {}
print(" Fonction : correct_fr_word : '"+word+"' =======> "+spell_fr(word)) print(" Fonction : correct_fr_word : '"+word+"' =======> "+MYSY_GV.spell_fr(word))
mydata['mot'] = str(word) mydata['mot'] = str(word)
mydata['mot_corrected'] = str(spell_fr(word)) mydata['mot_corrected'] = str(MYSY_GV.spell_fr(word))
mydata['date_update'] = str(datetime.now()) mydata['date_update'] = str(datetime.now())
mydata['treated'] = 0 mydata['treated'] = 0
coll_name = dbname['correction_ortho'] coll_name = MYSY_GV.dbname['correction_ortho']
ret_val_tmp = coll_name.insert_one(mydata) ret_val_tmp = coll_name.insert_one(mydata)
if (ret_val_tmp is False): if (ret_val_tmp is False):
mycommon.myprint(str(inspect.stack()[0][3]) + " - Impossbile d'inserer le mot "+str(word)+" dans correction_ortho ") mycommon.myprint(str(inspect.stack()[0][3]) + " - Impossbile d'inserer le mot "+str(word)+" dans correction_ortho ")
return spell_fr(word) return MYSY_GV.spell_fr(word)
except Exception as e: except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info() exc_type, exc_obj, exc_tb = sys.exc_info()
@ -118,7 +77,7 @@ Suppression des ponctuations
''' '''
def Ela_remove_ponct(list): def Ela_remove_ponct(list):
try: try:
for tmp in token_fr_pontuation: for tmp in MYSY_GV.token_fr_pontuation:
while tmp in list: while tmp in list:
list.remove(tmp) list.remove(tmp)
@ -193,7 +152,7 @@ def Ela_Remove_Bad_Pattern(sentence):
# Gestion des chiffres, car non indexés # Gestion des chiffres, car non indexés
patter5 = re.compile(r"([0-9]+)+") patter5 = re.compile(r"([0-9]+)+")
doc = nlp(str(text).lower()) doc = MYSY_GV.nlp(str(text).lower())
final_text = "" final_text = ""
for val in doc : for val in doc :
#print(" str(val) = '"+str(val)+"' ") #print(" str(val) = '"+str(val)+"' ")
@ -237,7 +196,7 @@ def Ela_Remove_Ponct_Special_Caractere(sentence):
#print(" suppression de : '"+str(noise)+"' ") #print(" suppression de : '"+str(noise)+"' ")
sentence = sentence.replace(str(noise), " ") sentence = sentence.replace(str(noise), " ")
print(" AFTER REPLACE NOISES = "+str(sentence)) #print(" AFTER REPLACE NOISES = "+str(sentence))
return True, sentence return True, sentence
except Exception as e: except Exception as e:
@ -266,7 +225,7 @@ def Ela_Tokenize(sentence):
if (status is False): if (status is False):
return False return False
doc = nlp(str(sentence).lower()) doc = MYSY_GV.nlp(str(sentence).lower())
#print(" Tokenize = '" + str(doc)+"' ") #print(" Tokenize = '" + str(doc)+"' ")
@ -300,7 +259,7 @@ def Ela_remove_stop_words(tab_tokens):
clean_words = [] clean_words = []
for token in tab_tokens: for token in tab_tokens:
if token not in stopWords: if token not in MYSY_GV.stopWords:
clean_words.append(token) clean_words.append(token)
return True, clean_words return True, clean_words
@ -325,7 +284,7 @@ def Ela_remove_pronoun(tab_tokens):
try: try:
mywords = [] mywords = []
for token in tab_tokens: for token in tab_tokens:
mytok = nlp(str(token).lower()) mytok = MYSY_GV.nlp(str(token).lower())
for token2 in mytok: for token2 in mytok:
if token2.pos_ != 'DET' and token2.pos_ != 'CCONJ' and token2.pos_ != 'ADP': if token2.pos_ != 'DET' and token2.pos_ != 'CCONJ' and token2.pos_ != 'ADP':
mywords.append(str(mytok)) mywords.append(str(mytok))
@ -353,9 +312,9 @@ def Ela_stemmize(tab_tokens):
if( mycommon.check_word_in_fr_dict(str(mot)) ): if( mycommon.check_word_in_fr_dict(str(mot)) ):
if( type(mot) is str ): if( type(mot) is str ):
tab_ret_val.append(stemmer.stem(mot)) tab_ret_val.append(MYSY_GV.stemmer.stem(mot))
else: else:
tab_ret_val.append(stemmer.stem(mot.text)) tab_ret_val.append(MYSY_GV.stemmer.stem(mot.text))
else: else:
tab_ret_val.append(mot) tab_ret_val.append(mot)
@ -391,12 +350,12 @@ def Ela_stemmize_Class(tab_tokens):
tab_ret_val.append(unidecode(mot)) tab_ret_val.append(unidecode(mot))
else: else:
#print(" AVANT STEM MOT ="+str(mot)) #print(" AVANT STEM MOT ="+str(mot))
tab_ret_val.append( unidecode( str(stemmer.stem(mot)))) tab_ret_val.append( unidecode( str(MYSY_GV.stemmer.stem(mot))))
#print(" AVANT STEM MOT =" + unidecode( str(stemmer.stem(mot)))) #print(" AVANT STEM MOT =" + unidecode( str(stemmer.stem(mot))))
#print(" STMISATION TAB = "+str(tab_ret_val)) #print(" STMISATION TAB = "+str(tab_ret_val))
return tab_ret_val return True, tab_ret_val
except Exception as e: except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info() exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e)+" - Line : "+ str(exc_tb.tb_lineno) ) mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e)+" - Line : "+ str(exc_tb.tb_lineno) )
@ -424,9 +383,9 @@ def Ela_stemmize_search(tab_tokens):
''' '''
On fait la correction orthographe avant On fait la correction orthographe avant
''' '''
print("AVANT COORECTION ORH ="+str(mot)+" ==> APRES = "+unidecode(str(spell.correction(mot)))) print("AVANT COORECTION ORH ="+str(mot)+" ==> APRES = "+unidecode(str(MYSY_GV.spell.correction(mot))))
corrected_str = str(spell.correction(mot)) corrected_str = str(MYSY_GV.spell.correction(mot))
tab_ret_val.append( unidecode (str(stemmer.stem(corrected_str)))) tab_ret_val.append( unidecode (str(MYSY_GV.stemmer.stem(corrected_str))))
print(" STMISATION TAB = "+str(tab_ret_val)) print(" STMISATION TAB = "+str(tab_ret_val))
@ -457,9 +416,7 @@ def ela_index_record_field(lines, class_id, source_field = ""):
## Suppression de toutes les indexation qui existe de cette formation ## Suppression de toutes les indexation qui existe de cette formation
''' '''
client = MongoClient(CONNECTION_STRING) coll_name = MYSY_GV.dbname['elaindex']
dbname = client['cherifdb']
coll_name = dbname['elaindex']
# check default value of parameter : source_field # check default value of parameter : source_field
if len(str(source_field)) == 0 : if len(str(source_field)) == 0 :
@ -467,7 +424,7 @@ def ela_index_record_field(lines, class_id, source_field = ""):
myquery = {"id_formation": class_id, "source_field":source_field} myquery = {"id_formation": class_id, "source_field":source_field}
delete_row = coll_name.delete_many(myquery) delete_row = coll_name.delete_many(myquery)
mycommon.myprint(" elaindex - "+str(delete_row.deleted_count)+" documents deleted. Training ==> "+str(class_id)+" ") #mycommon.myprint(" elaindex - "+str(delete_row.deleted_count)+" documents deleted. Training ==> "+str(class_id)+" ")
''' '''
Ajout des indexe Ajout des indexe
@ -501,7 +458,7 @@ def ela_index_record_field(lines, class_id, source_field = ""):
#print("size_tab = " + str(size_tab)) #print("size_tab = " + str(size_tab))
occurrences = Counter(tab_tokens4) occurrences = Counter(tab_tokens4)
most_common = occurrences.most_common() most_common = occurrences.most_common()
print(most_common) #print(most_common)
return True return True
@ -511,6 +468,83 @@ def ela_index_record_field(lines, class_id, source_field = ""):
return False return False
'''
Version Youtubes
'''
def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
try:
'''
## Suppression de toutes les indexation qui existe de cette formation
'''
coll_name = MYSY_GV.YTUBES_dbname['mysyindex']
coll_name_StopWord = MYSY_GV.YTUBES_dbname['mysystopwords']
print("lolololo")
# check default value of parameter : source_field
if len(str(source_field)) == 0 :
source_field = 'default'
myquery = {"id_formation": class_id, "source_field":source_field}
delete_row = coll_name.delete_many(myquery)
#mycommon.myprint(" elaindex - "+str(delete_row.deleted_count)+" documents deleted. Training ==> "+str(class_id)+" ")
'''
Ajout des indexe
'''
status, tab_tokens = Ela_Tokenize(lines)
if( status is False):
return False
#print(" AFFICHAGE TAB TOKEN")
#print(tab_tokens)
#print(" FINNN TAB TOKEN")
status, tab_tokens2 = Ela_remove_stop_words(tab_tokens)
if (status is False):
return False
status, tab_tokens3 = Ela_remove_pronoun(tab_tokens2)
if (status is False):
return False
print(" AVANT STEM ")
# Mettre en unicode et virer d'autre stop word
tab_tokens3_unicode = []
for val in tab_tokens3:
val_unicode = unidecode(val)
val_tmp = coll_name_StopWord.count_documents({'stop_word': str(val_unicode)})
if( val_tmp <= 0 ):
tab_tokens3_unicode.append(val_unicode)
#print(str(tab_tokens3_unicode))
status, tab_tokens4 = Ela_stemmize_Class(tab_tokens3)
if (status is False):
return False
#tab_tokens4 = Ela_Remove_Noise_from_list(tab_tokens4)
#tab_tokens4 = Ela_remove_ponct(tab_tokens4)
# Enregistrement dans la base mongodb
YTUBES_Ela_list_to_mongo(tab_tokens3_unicode,class_id, source_field)
size_tab = len(tab_tokens4)
#print("size_tab = " + str(size_tab))
occurrences = Counter(tab_tokens4)
most_common = occurrences.most_common()
#print(most_common)
return True
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False
''' '''
def Ela_ntlk(mysentence, traning_id): def Ela_ntlk(mysentence, traning_id):
tab_tokens = Ela_Tokenize(mysentence) tab_tokens = Ela_Tokenize(mysentence)
@ -550,17 +584,72 @@ def Ela_list_to_mongo(tab_tokens, traning_id, source_field):
my_file.write(str(data)) my_file.write(str(data))
my_file.close() my_file.close()
# Making a Connection with MongoClient
client = MongoClient("mongodb://localhost:27017/") collection = MYSY_GV.dbname["elaindex"]
# database
db = client["cherifdb"]
# collection
collection = db["elaindex"]
data.reset_index(inplace=True) data.reset_index(inplace=True)
data_dict = data.to_dict("records") data_dict = data.to_dict("records")
print(data_dict) #print(data_dict)
# Insert collection
collection.insert_many(data_dict)
data.to_csv("data_indexees.csv")
return True
except Exception as e :
mycommon.myprint(e)
return False
'''
Version Youtubes
'''
def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field):
try:
# clear / truncate elaindex
#db.elaindex.remove({})
# Recherche des occurence d'une valeur
my_file = open("ela_output_test_file_pandas_2.txt", "w")
size_tab = len(tab_tokens)
occurrences = Counter(tab_tokens)
# insert the list to the set
list_set = set(tab_tokens)
# convert the set to the list
unique_list = (list(list_set))
final_lists = []
'''
Pour gerer la notion de limite d'occurrence >= 5, on definit un paramettre appelé : seuil
'''
seuil = round(int(str(MYSY_GV.INDEX_MIN_OCCURENCE)) / size_tab, 3)
print(" SEUIL = "+str(seuil))
for tmp in unique_list:
moyenne = round(int(occurrences[str(tmp)]) / size_tab, 3)
if( moyenne >= seuil ):
print(str(moyenne)+ " CMP "+str(seuil))
final_lists.append([str(tmp), int(str(occurrences[str(tmp)])), str(moyenne), str(traning_id), str(source_field)])
#print(final_lists)
data = pd.DataFrame(final_lists, columns=('mots', "occurence", 'moyenne', 'id_formation', 'source_field'))
my_file.write(str(data))
my_file.close()
print(" YTUBES INSERT MAY")
YTUBES_CONNECTION_STRING = "mongodb://localhost:27017/"
YTUBES_client = MongoClient(YTUBES_CONNECTION_STRING)
YTUBES_dbname = YTUBES_client['mysyvideodb']
collection = YTUBES_dbname["mysyindex"]
data.reset_index(inplace=True)
data_dict = data.to_dict("records")
#print(data_dict)
# Insert collection # Insert collection
collection.insert_many(data_dict) collection.insert_many(data_dict)
data.to_csv("data_indexees.csv") data.to_csv("data_indexees.csv")
@ -628,9 +717,9 @@ def ela_index_article_avis_record_field(lines, article_avis_id, source_field = "
''' '''
## Suppression de toutes les indexation qui existe de cette formation ## Suppression de toutes les indexation qui existe de cette formation
''' '''
client = MongoClient(CONNECTION_STRING) client = MongoClient(MYSY_GV.CONNECTION_STRING)
dbname = client['cherifdb'] MYSY_GV.dbname = client['cherifdb']
coll_name = dbname['elaindex_article_avis'] coll_name = MYSY_GV.dbname['elaindex_article_avis']
# check default value of parameter : source_field # check default value of parameter : source_field
if len(str(source_field)) == 0 : if len(str(source_field)) == 0 :

73
GlobalVariable.py Normal file
View File

@ -0,0 +1,73 @@
'''
Ce fichier contient les variables globales du systeme
'''
from pymongo import MongoClient
from spellchecker import SpellChecker
from nltk.stem.snowball import SnowballStemmer
import spacy
import logging
from autocorrect import Speller
from nltk.corpus import stopwords
import ela_spacy_common as lsc
CONNECTION_STRING = "mongodb://localhost:27017/cherifdb"
client = MongoClient(CONNECTION_STRING)
dbname = client['cherifdb']
YTUBES_CONNECTION_STRING = "mongodb://localhost:27017/"
YTUBES_client = MongoClient(YTUBES_CONNECTION_STRING)
YTUBES_dbname = YTUBES_client['mysyvideodb']
'''
Cette Variable defini le nombre minimum
de frequence d'un mot pour qu'il soit indexé.
en gros, dans une text, les mots apparaissent moins de 5 fois sont consideré
comme des mots à faible raisonnance'''
INDEX_MIN_OCCURENCE = 5
## Gle Variables
stemmer = SnowballStemmer(language='french')
nlp = spacy.load("fr_core_news_sm")
spell = SpellChecker(language='fr')
token_fr_pontuation = []
'''
La taille maximal d'un champs à retourner.
Au dela de 300 caractère, le système rame
'''
MAX_CARACT = 300
MAX_CARACT_DEDUIT = 50
TOKEN_SIZE = 25
DEBUG_LEVEL = logging.DEBUG
LOG_FILE_NAME = "./Log/log_file.log"
upload_folder = "./Data/"
ALLOWED_EXTENSIONS = {'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'}
logging.basicConfig( level=DEBUG_LEVEL, filename=LOG_FILE_NAME)
AUTORIZED_SOURCE_IPV4=["127.0.0.1", "localhost", "88.170.110.220", "192.168.1.21", "192.168.1.48"]
#assign the default stopwords list to a variable
STOP_WORDS = spacy.lang.fr.stop_words.STOP_WORDS
stopWords = set(stopwords.words('french'))
sentence = "Bouygues a eu une coupure de réseau à Marseille chez ses clients marseillais et son couteau"
lsc.update_stopWords(stopWords)
#print(type(stopWords))
lsc.update_token_fr_pontuation(token_fr_pontuation)
spell_fr = Speller(lang='fr')

View File

@ -22,6 +22,7 @@ import inspect
import sys, os import sys, os
import csv import csv
import pandas as pd import pandas as pd
import GlobalVariable as MYSY_GV
@ -37,13 +38,6 @@ CONNECTION_STRING = "mongodb://localhost/cherifdb"
client = MongoClient(CONNECTION_STRING) client = MongoClient(CONNECTION_STRING)
dbname = client['cherifdb'] dbname = client['cherifdb']
'''
La taille maximal d'un champs à retourner.
Au dela de 300 caractère, le système rame
'''
MAX_CARACT = 300
MAX_CARACT_DEDUIT = 50
''' '''
@ -79,18 +73,18 @@ def get_all_articles_avis(diction):
[("title_formation",pymongo.ASCENDING), ("date_avis",pymongo.ASCENDING)]): [("title_formation",pymongo.ASCENDING), ("date_avis",pymongo.ASCENDING)]):
# mycommon.myprint("AVANT ==> "+str(x['description'])) # mycommon.myprint("AVANT ==> "+str(x['description']))
val = x['qualite'] val = x['qualite']
if (len(x['qualite']) > MAX_CARACT): if (len(x['qualite']) > MYSY_GV.MAX_CARACT):
x['qualite'] = val[:MAX_CARACT] + " ..." x['qualite'] = val[:MYSY_GV.MAX_CARACT] + " ..."
else: else:
x['qualite'] = val[:MAX_CARACT] x['qualite'] = val[:MYSY_GV.MAX_CARACT]
val = x['avis'] val = x['avis']
if (len(x['avis']) > MAX_CARACT): if (len(x['avis']) > MYSY_GV.MAX_CARACT):
x['avis'] = val[:MAX_CARACT] + " ..." x['avis'] = val[:MYSY_GV.MAX_CARACT] + " ..."
else: else:
x['avis'] = val[:MAX_CARACT] x['avis'] = val[:MYSY_GV.MAX_CARACT]
# mycommon.myprint("APRES ==> " + str(x['description'])) # mycommon.myprint("APRES ==> " + str(x['description']))
@ -485,18 +479,18 @@ def recherche_articles_avis(diction):
val = x['qualite'] val = x['qualite']
if (len(x['qualite']) > MAX_CARACT): if (len(x['qualite']) > MYSY_GV.MAX_CARACT):
x['qualite'] = val[:MAX_CARACT] + " ..." x['qualite'] = val[:MYSY_GV.MAX_CARACT] + " ..."
else: else:
x['qualite'] = val[:MAX_CARACT] x['qualite'] = val[:MYSY_GV.MAX_CARACT]
val = x['avis'] val = x['avis']
if (len(x['avis']) > MAX_CARACT): if (len(x['avis']) > MYSY_GV.MAX_CARACT):
x['avis'] = val[:MAX_CARACT] + " ..." x['avis'] = val[:MYSY_GV.MAX_CARACT] + " ..."
else: else:
x['avis'] = val[:MAX_CARACT] x['avis'] = val[:MYSY_GV.MAX_CARACT]
insertObject.append(JSONEncoder().encode(user)) insertObject.append(JSONEncoder().encode(user))
@ -678,18 +672,18 @@ def get_article_avis_alaune(diction):
[("date_avis", pymongo.ASCENDING), ("title_formation", pymongo.ASCENDING)]): [("date_avis", pymongo.ASCENDING), ("title_formation", pymongo.ASCENDING)]):
val = x['qualite'] val = x['qualite']
if (len(x['qualite']) > MAX_CARACT_DEDUIT): if (len(x['qualite']) > MYSY_GV.MAX_CARACT_DEDUIT):
x['qualite'] = val[:MAX_CARACT_DEDUIT] + " ..." x['qualite'] = val[:MYSY_GV.MAX_CARACT_DEDUIT] + " ..."
else: else:
x['qualite'] = val[:MAX_CARACT_DEDUIT] x['qualite'] = val[:MYSY_GV.MAX_CARACT_DEDUIT]
val = x['avis'] val = x['avis']
if (len(x['avis']) > MAX_CARACT_DEDUIT): if (len(x['avis']) > MYSY_GV.MAX_CARACT_DEDUIT):
x['avis'] = val[:MAX_CARACT_DEDUIT] + " ..." x['avis'] = val[:MYSY_GV.MAX_CARACT_DEDUIT] + " ..."
else: else:
x['avis'] = val[:MAX_CARACT_DEDUIT] x['avis'] = val[:MYSY_GV.MAX_CARACT_DEDUIT]
user = x user = x
insertObject.append(JSONEncoder().encode(user)) insertObject.append(JSONEncoder().encode(user))

View File

@ -1,13 +1,21 @@
,index,mots,occurence,moyenne,id_formation,source_field ,index,mots,occurence,moyenne,id_formation,source_field
0,0,regl,1,0.08,8866,objectif 0,0,dynamiques,7,0.012,DmZxdIvG3YQ,video_text
1,1,const,1,0.08,8866,objectif 1,1,valeurs,5,0.009,DmZxdIvG3YQ,video_text
2,2,object,1,0.08,8866,objectif 2,2,emporter,5,0.009,DmZxdIvG3YQ,video_text
3,3,publiqu,1,0.08,8866,objectif 3,3,colonne,6,0.011,DmZxdIvG3YQ,video_text
4,4,format,1,0.08,8866,objectif 4,4,graphique,6,0.011,DmZxdIvG3YQ,video_text
5,5,comptabl,1,0.08,8866,objectif 5,5,nombre,5,0.009,DmZxdIvG3YQ,video_text
6,6,impos,1,0.08,8866,objectif 6,6,vendu,8,0.014,DmZxdIvG3YQ,video_text
7,7,evolu,1,0.08,8866,objectif 7,7,prix,10,0.018,DmZxdIvG3YQ,video_text
8,8,reglement,1,0.08,8866,objectif 8,8,croises,6,0.011,DmZxdIvG3YQ,video_text
9,9,appliqu,1,0.08,8866,objectif 9,9,somme,5,0.009,DmZxdIvG3YQ,video_text
10,10,maitris,1,0.08,8866,objectif 10,10,cellules,5,0.009,DmZxdIvG3YQ,video_text
11,11,princip,1,0.08,8866,objectif 11,11,fonction,5,0.009,DmZxdIvG3YQ,video_text
12,12,groupe,7,0.012,DmZxdIvG3YQ,video_text
13,13,tableaux,7,0.012,DmZxdIvG3YQ,video_text
14,14,modifier,6,0.011,DmZxdIvG3YQ,video_text
15,15,tableau,7,0.012,DmZxdIvG3YQ,video_text
16,16,donnees,8,0.014,DmZxdIvG3YQ,video_text
17,17,vis,6,0.011,DmZxdIvG3YQ,video_text
18,18,produits,12,0.021,DmZxdIvG3YQ,video_text
19,19,besoin,6,0.011,DmZxdIvG3YQ,video_text

1 index mots occurence moyenne id_formation source_field
2 0 0 regl dynamiques 1 7 0.08 0.012 8866 DmZxdIvG3YQ objectif video_text
3 1 1 const valeurs 1 5 0.08 0.009 8866 DmZxdIvG3YQ objectif video_text
4 2 2 object emporter 1 5 0.08 0.009 8866 DmZxdIvG3YQ objectif video_text
5 3 3 publiqu colonne 1 6 0.08 0.011 8866 DmZxdIvG3YQ objectif video_text
6 4 4 format graphique 1 6 0.08 0.011 8866 DmZxdIvG3YQ objectif video_text
7 5 5 comptabl nombre 1 5 0.08 0.009 8866 DmZxdIvG3YQ objectif video_text
8 6 6 impos vendu 1 8 0.08 0.014 8866 DmZxdIvG3YQ objectif video_text
9 7 7 evolu prix 1 10 0.08 0.018 8866 DmZxdIvG3YQ objectif video_text
10 8 8 reglement croises 1 6 0.08 0.011 8866 DmZxdIvG3YQ objectif video_text
11 9 9 appliqu somme 1 5 0.08 0.009 8866 DmZxdIvG3YQ objectif video_text
12 10 10 maitris cellules 1 5 0.08 0.009 8866 DmZxdIvG3YQ objectif video_text
13 11 11 princip fonction 1 5 0.08 0.009 8866 DmZxdIvG3YQ objectif video_text
14 12 12 groupe 7 0.012 DmZxdIvG3YQ video_text
15 13 13 tableaux 7 0.012 DmZxdIvG3YQ video_text
16 14 14 modifier 6 0.011 DmZxdIvG3YQ video_text
17 15 15 tableau 7 0.012 DmZxdIvG3YQ video_text
18 16 16 donnees 8 0.014 DmZxdIvG3YQ video_text
19 17 17 vis 6 0.011 DmZxdIvG3YQ video_text
20 18 18 produits 12 0.021 DmZxdIvG3YQ video_text
21 19 19 besoin 6 0.011 DmZxdIvG3YQ video_text

View File

@ -6,11 +6,7 @@ from pymongo import ReturnDocument
import inspect import inspect
import sys, os import sys, os
from operator import itemgetter, attrgetter from operator import itemgetter, attrgetter
import GlobalVariable as MYSY_GV
client = MongoClient("mongodb://localhost:27017/")
# database
db = client["cherifdb"]
# collection
''' '''
Cette fonction indexe une formation Cette fonction indexe une formation
@ -22,7 +18,7 @@ Par defaut sont pris en compte :
def ela_index_class(external_code="", source_field = ""): def ela_index_class(external_code="", source_field = ""):
try: try:
collection = db["myclass"] collection = MYSY_GV.dbname["myclass"]
if( len(str(source_field)) == 0): if( len(str(source_field)) == 0):
source_field = 'default' source_field = 'default'
@ -156,6 +152,30 @@ def ela_index_class(external_code="", source_field = ""):
else: else:
return False return False
# Indexation du champ : video_text , donc la transcription d'une video
elif (str(source_field) == "video_text"):
chaine = doc["video_text"]
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
retval = ls.ela_index_record_field(chaine, str(doc["external_code"]), source_field)
'''
# Apres indexation, on met a jour la formation pour
# mettre le champ "indexe" a 1.
'''
if (retval):
# mise à jour du champ "indexed"
myobjectif = {}
myobjectif['indexed_video_text'] = '1'
ret_val2 = collection.find_and_modify(query={"external_code": external_code},
update={"$set": myobjectif}
)
if (ret_val2 and ret_val2['_id']):
mycommon.myprint("La formation indexée (champs video_text) =" + str(ret_val2['_id']))
return True
else:
return False
return True return True
except Exception as e: except Exception as e:
@ -172,7 +192,7 @@ def ela_index_all_classes():
# Making a Connection with MongoClient # Making a Connection with MongoClient
try: try:
# collection # collection
collection = db["myclass"] collection = MYSY_GV.dbname["myclass"]
''' '''
ATTENTION ATTENTION ATTENTION ATTENTION
/!\ : avant l'indexation, on vide la table des indexes. /!\ : avant l'indexation, on vide la table des indexes.
@ -223,7 +243,7 @@ ex : mot1;mot2;mot3
''' '''
def ela_index_all_classes_keywords(): def ela_index_all_classes_keywords():
try: try:
collection = db["myclass"] collection = MYSY_GV.dbname["myclass"]
nb_class_indexed = 0 nb_class_indexed = 0
for doc in collection.find({'indexed_keyword': '0'}): for doc in collection.find({'indexed_keyword': '0'}):
if ("mots_cle" in doc.keys()): if ("mots_cle" in doc.keys()):
@ -257,7 +277,7 @@ def ela_index_all_classes_title():
# Making a Connection with MongoClient # Making a Connection with MongoClient
try: try:
collection = db["myclass"] collection = MYSY_GV.dbname["myclass"]
''' '''
ATTENTION ATTENTION ATTENTION ATTENTION
/!\ : avant l'indexation, on vide la table des indexes. /!\ : avant l'indexation, on vide la table des indexes.
@ -312,7 +332,7 @@ def ela_index_all_classes_desc():
# database # database
db = client["cherifdb"] db = client["cherifdb"]
# collection # collection
collection = db["myclass"] collection = MYSY_GV.dbname["myclass"]
''' '''
ATTENTION ATTENTION ATTENTION ATTENTION
/!\ : avant l'indexation, on vide la table des indexes. /!\ : avant l'indexation, on vide la table des indexes.
@ -364,7 +384,7 @@ def ela_index_all_classes_obj():
# Making a Connection with MongoClient # Making a Connection with MongoClient
try: try:
# collection # collection
collection = db["myclass"] collection = MYSY_GV.dbname["myclass"]
''' '''
ATTENTION ATTENTION ATTENTION ATTENTION
/!\ : avant l'indexation, on vide la table des indexes. /!\ : avant l'indexation, on vide la table des indexes.
@ -407,6 +427,61 @@ def ela_index_all_classes_obj():
return False, "Impossible d'indexer la BDD des formations ==> KO" return False, "Impossible d'indexer la BDD des formations ==> KO"
'''
Cette fonction lire la transcription d'une video et
l'indexer'''
def ela_index_all_classes_video_text():
# Making a Connection with MongoClient
try:
# collection
collection = MYSY_GV.dbname["myclass"]
'''
ATTENTION ATTENTION
/!\ : avant l'indexation, on vide la table des indexes.
cette logique est tres gourmande.
ce quil faudrait faire :
- si une formation est indexée, mettre un fag pour dire : indexé => OK
- si on met à jour une formation ==> Mettre le flag à KO
- si on ajoute une formation, mettre le flag à KO
ex : indexation du champ 'title' ou du champ 'objectif', je vais introduitre la notion de 'source_fied'
pour l'indexation title, 'source_fied' = 'title'
pour l'indexation objectif, 'source_fied' = 'objectif'
apres pour l'indexation, il ne faudra indexer que les formation dont le flag est KO.
aussi, avant d'indexer une formation, supprimer tous les index de la dite formation pour remplacer tous les anciens et eviter un doublon
'''
nb_class_indexed = 0
for doc in collection.find({'indexed_video_text': '0'}):
if ("video_text" in doc.keys()):
chaine = doc["video_text"]
#mycommon.myprint(" video_texte = " + str(doc["external_code"]) + " ==> " + str(chaine))
retval = ela_index_class(str(doc["external_code"]), "video_text")
#print(" apres : ela_index_all_classes_video_texte retval = " + str(retval))
if (retval is False):
mycommon.myprint("IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"]))
else:
nb_class_indexed = nb_class_indexed + 1
# retval = ls.ela_index_record_field(chaine, str(doc["external_code"]))
mycommon.myprint(str(nb_class_indexed) + " ont été indexes - champ video_text")
return True, str(nb_class_indexed) + " ont été indexes - champ video_text "
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, "Impossible d'indexer video_text la BDD des formations ==> KO"
''' '''
Cette fonction recherche une chaine de caractère Cette fonction recherche une chaine de caractère
dans la base de données. dans la base de données.
@ -455,7 +530,7 @@ def ela_recherche_tokens(sentence):
''' '''
# collection # collection
collection = db["elaindex"] collection = MYSY_GV.dbname["elaindex"]
print(" mot recherché tab_tokens4 = "+str(tab_tokens4)) print(" mot recherché tab_tokens4 = "+str(tab_tokens4))
@ -532,7 +607,7 @@ def ela_recherche_tokens_source_field(sentence, source_fied=""):
a quoi cela pourrait correspondre. a quoi cela pourrait correspondre.
''' '''
collection = db["elaindex"] collection = MYSY_GV.dbname["elaindex"]
for token in tab_tokens4: for token in tab_tokens4:
print(" #### Token rechercher dans l'index est : '"+str(token)+"' et le source_field = '"+str(source_fied)+"' ") print(" #### Token rechercher dans l'index est : '"+str(token)+"' et le source_field = '"+str(source_fied)+"' ")
@ -560,7 +635,7 @@ def ela_index_all_articles_avis():
# Making a Connection with MongoClient # Making a Connection with MongoClient
try: try:
# collection # collection
collection = db["articles_avis"] collection = MYSY_GV.dbname["articles_avis"]
nb_class_indexed = 0 nb_class_indexed = 0
for doc in collection.find({'indexed': '0'}): for doc in collection.find({'indexed': '0'}):
if ("title_formation" in doc.keys()): if ("title_formation" in doc.keys()):
@ -596,7 +671,7 @@ Par defaut sont pris en compte :
def ela_index_articles_avis(titre_formation="", source_field = ""): def ela_index_articles_avis(titre_formation="", source_field = ""):
try: try:
collection = db["articles_avis"] collection = MYSY_GV.dbname["articles_avis"]
if( len(str(source_field)) == 0): if( len(str(source_field)) == 0):
source_field = 'default' source_field = 'default'
@ -681,7 +756,7 @@ def ela_recherche_article_avis_tokens(sentence):
''' '''
# collection # collection
collection = db["elaindex_article_avis"] collection = MYSY_GV.dbname["elaindex_article_avis"]
print(" mot recherché tab_tokens4 = " + str(tab_tokens4)) print(" mot recherché tab_tokens4 = " + str(tab_tokens4))

View File

@ -1,13 +1,21 @@
mots occurence moyenne id_formation source_field mots occurence moyenne id_formation source_field
0 regl 1 0.08 8866 objectif 0 dynamiques 7 0.012 DmZxdIvG3YQ video_text
1 const 1 0.08 8866 objectif 1 valeurs 5 0.009 DmZxdIvG3YQ video_text
2 object 1 0.08 8866 objectif 2 emporter 5 0.009 DmZxdIvG3YQ video_text
3 publiqu 1 0.08 8866 objectif 3 colonne 6 0.011 DmZxdIvG3YQ video_text
4 format 1 0.08 8866 objectif 4 graphique 6 0.011 DmZxdIvG3YQ video_text
5 comptabl 1 0.08 8866 objectif 5 nombre 5 0.009 DmZxdIvG3YQ video_text
6 impos 1 0.08 8866 objectif 6 vendu 8 0.014 DmZxdIvG3YQ video_text
7 evolu 1 0.08 8866 objectif 7 prix 10 0.018 DmZxdIvG3YQ video_text
8 reglement 1 0.08 8866 objectif 8 croises 6 0.011 DmZxdIvG3YQ video_text
9 appliqu 1 0.08 8866 objectif 9 somme 5 0.009 DmZxdIvG3YQ video_text
10 maitris 1 0.08 8866 objectif 10 cellules 5 0.009 DmZxdIvG3YQ video_text
11 princip 1 0.08 8866 objectif 11 fonction 5 0.009 DmZxdIvG3YQ video_text
12 groupe 7 0.012 DmZxdIvG3YQ video_text
13 tableaux 7 0.012 DmZxdIvG3YQ video_text
14 modifier 6 0.011 DmZxdIvG3YQ video_text
15 tableau 7 0.012 DmZxdIvG3YQ video_text
16 donnees 8 0.014 DmZxdIvG3YQ video_text
17 vis 6 0.011 DmZxdIvG3YQ video_text
18 produits 12 0.021 DmZxdIvG3YQ video_text
19 besoin 6 0.011 DmZxdIvG3YQ video_text

52
main.py
View File

@ -9,7 +9,7 @@ from datetime import timedelta
import Ela_Spacy as ls import Ela_Spacy as ls
import user_message_mgt as um import user_message_mgt as um
import user_session as us import user_session as us
import logging
import partners as pa import partners as pa
import prj_common as mycommon import prj_common as mycommon
import ela_index_bdd_classes as ela_index_class import ela_index_bdd_classes as ela_index_class
@ -21,23 +21,15 @@ from OpenSSL import SSL
from flask import redirect from flask import redirect
from flask import Flask, render_template, request, redirect, url_for from flask import Flask, render_template, request, redirect, url_for
import articles_avis as aa import articles_avis as aa
import GlobalVariable as MYSY_GV
import youtubes_analyse as YTA
DEBUG_LEVEL = logging.DEBUG
LOG_FILE_NAME = "./Log/log_file.log"
upload_folder = "./Data/"
ALLOWED_EXTENSIONS = {'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'}
logging.basicConfig( level=DEBUG_LEVEL, filename=LOG_FILE_NAME)
app = Flask(__name__) app = Flask(__name__)
cors = CORS(app, resources={r"/foo": {"origins": "*"}}) cors = CORS(app, resources={r"/foo": {"origins": "*"}})
app.config['CORS_HEADERS'] = 'Content-Type' app.config['CORS_HEADERS'] = 'Content-Type'
app.config['UPLOAD_FOLDER'] = upload_folder app.config['UPLOAD_FOLDER'] = MYSY_GV.upload_folder
@app.before_request @app.before_request
def before_request(): def before_request():
@ -605,6 +597,39 @@ def ela_index_all_classes_title():
status, retval = ela_index_class.ela_index_all_classes_title() status, retval = ela_index_class.ela_index_all_classes_title()
return jsonify(status=status, message=retval) return jsonify(status=status, message=retval)
'''
Cette API index la base donnée en se basant sur les videos qui ont une transcription
le champs en question sera : "video_text".
La fonction "ela_index_all_classes" va aller chercher chaque cours dans la BDD
et créer les mots clées qui sont ensuite enregister dans la table "elaindex"
'''
@app.route('/myclass/api/ela_index_all_classes_video_text/', methods=['POST','GET'])
@crossdomain(origin='*')
def ela_index_all_classes_video_text():
# On recupere le corps (payload) de la requete
payload = request.form.to_dict()
print(" ### payload = ",payload)
status, retval = ela_index_class.ela_index_all_classes_video_text()
return jsonify(status=status, message=retval)
''' Analyse Youtubes
'''
@app.route('/myclass/api/YTUBES_RedText/', methods=['POST','GET'])
@crossdomain(origin='*')
def YTUBES_RedText():
# On recupere le corps (payload) de la requete
payload = request.form.to_dict()
print(" ### payload = ",payload)
status, retval = YTA.YTUBES_RedText()
return jsonify(status=status, message=retval)
''' '''
Cette API index la base donnée en se basant sur le description. Cette API index la base donnée en se basant sur le description.
La fonction "ela_index_all_classes" va aller chercher chaque cours dans la BDD La fonction "ela_index_all_classes" va aller chercher chaque cours dans la BDD
@ -801,7 +826,7 @@ par exemple avec un fichier CSV
def allowed_file(filename): def allowed_file(filename):
return '.' in filename and \ return '.' in filename and \
filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS filename.rsplit('.', 1)[1].lower() in MYSY_GV.ALLOWED_EXTENSIONS
@app.route('/myclass/api/add_class_mass/', methods=['POST','GET']) @app.route('/myclass/api/add_class_mass/', methods=['POST','GET'])
@ -975,6 +1000,7 @@ def send_mail_delete_user():
if __name__ == '__main__': if __name__ == '__main__':
print(" debut api") print(" debut api")
context = SSL.Context(SSL.SSLv23_METHOD) context = SSL.Context(SSL.SSLv23_METHOD)

View File

@ -15,15 +15,9 @@ import sys
from datetime import datetime from datetime import datetime
from pymongo import ReturnDocument from pymongo import ReturnDocument
from unidecode import unidecode from unidecode import unidecode
import GlobalVariable as MYSY_GV
TOKEN_SIZE = 25
CONNECTION_STRING = "mongodb://localhost/cherifdb"
client = MongoClient(CONNECTION_STRING)
dbname = client['cherifdb']
AUTORIZED_SOURCE_IPV4=["127.0.0.1", "localhost", "88.170.110.220", "192.168.1.21", "192.168.1.48"]
def myprint(message = ""): def myprint(message = ""):
logging.info(str(datetime.now()) + " : "+str(message) ) logging.info(str(datetime.now()) + " : "+str(message) )
@ -31,11 +25,11 @@ def myprint(message = ""):
def create_token_urlsafe(): def create_token_urlsafe():
return secrets.token_urlsafe(TOKEN_SIZE) return secrets.token_urlsafe(MYSY_GV.TOKEN_SIZE)
def create_user_recid(): def create_user_recid():
return secrets.token_hex(TOKEN_SIZE) return secrets.token_hex(MYSY_GV.TOKEN_SIZE)
''' '''
@ -88,7 +82,7 @@ puis verifie si la validité du trile (email, token, statut)
def check_token_validity(email="", token=""): def check_token_validity(email="", token=""):
try : try :
coll_token = dbname['user_token'] coll_token = MYSY_GV.dbname['user_token']
tmp_count = coll_token.count_documents({ 'token': str(token), 'valide': '1'}) tmp_count = coll_token.count_documents({ 'token': str(token), 'valide': '1'})
if (tmp_count <= 0): if (tmp_count <= 0):
@ -108,7 +102,7 @@ puis verifie si le compte utilisateur est actif
def check_user_validity(email="", token=""): def check_user_validity(email="", token=""):
try : try :
coll_token = dbname['user_account'] coll_token = MYSY_GV.dbname['user_account']
message = {} message = {}
ret = True ret = True
@ -142,7 +136,7 @@ donc la table partner_token
''' '''
def check_partner_token_validity(email="", token=""): def check_partner_token_validity(email="", token=""):
try : try :
coll_token = dbname['partner_token'] coll_token = MYSY_GV.dbname['partner_token']
tmp_count = coll_token.count_documents({ 'token': str(token), 'locked':'0', 'valide': '1'}) tmp_count = coll_token.count_documents({ 'token': str(token), 'locked':'0', 'valide': '1'})
#tmp_count = coll_token.find({ 'token': str(token), 'locked':'0', 'valide': '1'}).count() #tmp_count = coll_token.find({ 'token': str(token), 'locked':'0', 'valide': '1'}).count()
@ -166,7 +160,7 @@ def get_user_recid_from_token(token = ""):
myprint(" Le token est vide") myprint(" Le token est vide")
return False return False
coll_token = dbname['user_token'] coll_token = MYSY_GV.dbname['user_token']
tmp_val = coll_token.find({'token': str(token), 'valide': '1'}) tmp_val = coll_token.find({'token': str(token), 'valide': '1'})
user_recid = tmp_val[0]['recid'] user_recid = tmp_val[0]['recid']
return user_recid return user_recid
@ -180,7 +174,7 @@ def get_user_email_from_token(token = ""):
myprint(" Le token est vide") myprint(" Le token est vide")
return False return False
coll_token = dbname['user_token'] coll_token = MYSY_GV.dbname['user_token']
tmp_val = coll_token.find({'token': str(token), 'valide': '1'}) tmp_val = coll_token.find({'token': str(token), 'valide': '1'})
user_email = tmp_val[0]['email'] user_email = tmp_val[0]['email']
return user_email return user_email
@ -196,7 +190,7 @@ def get_user_email_from_recid(recid = ""):
myprint(" Le recid est vide") myprint(" Le recid est vide")
return False return False
coll_token = dbname['user_account'] coll_token = MYSY_GV.dbname['user_account']
tmp_val = coll_token.find({'recid': str(recid), 'active': '1'}) tmp_val = coll_token.find({'recid': str(recid), 'active': '1'})
if( tmp_val and tmp_val[0] and tmp_val[0]['email']): if( tmp_val and tmp_val[0] and tmp_val[0]['email']):
@ -219,7 +213,7 @@ def get_parnter_recid_from_token(token = ""):
myprint(" Le token partner est vide") myprint(" Le token partner est vide")
return False return False
coll_token = dbname['partner_token'] coll_token = MYSY_GV.dbname['partner_token']
tmp_val = coll_token.find({'token': str(token), 'valide': '1', 'locked':'0'}) tmp_val = coll_token.find({'token': str(token), 'valide': '1', 'locked':'0'})
user_recid = tmp_val[0]['recid'] user_recid = tmp_val[0]['recid']
return user_recid return user_recid
@ -233,7 +227,7 @@ def get_user_recid_from_email(email = ""):
return False return False
coll_user = dbname['user_account'] coll_user = MYSY_GV.dbname['user_account']
tmp_val = coll_user.find({'email': str(email), 'active': '1'}) tmp_val = coll_user.find({'email': str(email), 'active': '1'})
user_recid = tmp_val[0]['recid'] user_recid = tmp_val[0]['recid']
@ -337,7 +331,7 @@ stemisable à traver la tabla "word_not_stem
''' '''
def Word_Not_Stemmize(word = None): def Word_Not_Stemmize(word = None):
try: try:
coll_not_stem = dbname["word_not_stem"] coll_not_stem = MYSY_GV.dbname["word_not_stem"]
val_tmp = coll_not_stem.count_documents({'mot': str(word)}) val_tmp = coll_not_stem.count_documents({'mot': str(word)})
@ -360,8 +354,8 @@ Utilisé dans le cas de l'indexation d'une formation
def check_word_in_fr_dict(mot=None): def check_word_in_fr_dict(mot=None):
try: try:
print("#### analyse du mot "+str(mot)) print("#### analyse du mot "+str(mot))
col_name = dbname["list_mots_fr"] col_name = MYSY_GV.dbname["list_mots_fr"]
col_name_not_fr = dbname["list_mots_not_fr"] col_name_not_fr = MYSY_GV.dbname["list_mots_not_fr"]
mydata = {} mydata = {}
val_tmp = col_name.count_documents({'mot': str(mot)}) val_tmp = col_name.count_documents({'mot': str(mot)})
@ -397,7 +391,7 @@ recherche d'un utilisateur '''
def recherche_check_word_in_fr_dict(mot=None): def recherche_check_word_in_fr_dict(mot=None):
try: try:
print("#### analyse du mot "+str(mot)) print("#### analyse du mot "+str(mot))
col_name = dbname["list_mots_fr"] col_name = MYSY_GV.dbname["list_mots_fr"]
val_tmp = col_name.count_documents({'mot': str(mot)}) val_tmp = col_name.count_documents({'mot': str(mot)})
if (val_tmp <= 0): if (val_tmp <= 0):
@ -419,7 +413,7 @@ est autorisé ou pas.
''' '''
def check_source_ipv4(source_ip=None): def check_source_ipv4(source_ip=None):
try: try:
if source_ip in AUTORIZED_SOURCE_IPV4: if source_ip in MYSY_GV.AUTORIZED_SOURCE_IPV4:
myprint(" Security check : IP adresse '"+str(source_ip)+"' connected") myprint(" Security check : IP adresse '"+str(source_ip)+"' connected")
return True return True
else: else:

View File

@ -19,6 +19,7 @@ import sys, os
import csv import csv
import pandas as pd import pandas as pd
from unidecode import unidecode from unidecode import unidecode
import GlobalVariable as MYSY_GV
@ -30,15 +31,7 @@ class JSONEncoder(json.JSONEncoder):
print(o) print(o)
return json.JSONEncoder.default(self, o) return json.JSONEncoder.default(self, o)
CONNECTION_STRING = "mongodb://localhost/cherifdb"
client = MongoClient(CONNECTION_STRING)
dbname = client['cherifdb']
'''
La taille maximal d'un champs à retourner.
Au dela de 300 caractère, le système rame
'''
MAX_CARACT = 300
def get_recherche_gle_class(sentence): def get_recherche_gle_class(sentence):
try: try:
@ -53,7 +46,7 @@ def get_recherche_gle_class(sentence):
mycommon.myprint(tab_training) mycommon.myprint(tab_training)
coll_name = dbname['myclass'] coll_name = MYSY_GV.dbname['myclass']
insertObject = [] insertObject = []
@ -80,10 +73,8 @@ def get_class_by_list_attr(attribut, list_values):
ela_array = [] ela_array = []
ela_array = list_values.split(",") ela_array = list_values.split(",")
mycommon.myprint(" attribut"+ attribut+" ==> list_values = "+ela_array) mycommon.myprint(" attribut"+ attribut+" ==> list_values = "+ela_array)
client = MongoClient(CONNECTION_STRING)
dbname = client['cherifdb'] coll_name = MYSY_GV.dbname['myclass']
coll_name = dbname['myclass']
insertObject = [] insertObject = []
for x in coll_name.find({attribut:{ "$in":ela_array}},{"_id": 0, "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0, for x in coll_name.find({attribut:{ "$in":ela_array}},{"_id": 0, "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0,
@ -105,10 +96,8 @@ def get_class_by_list_attr(attribut, list_values):
def update_class_by_attribut(objId, attribut, value): def update_class_by_attribut(objId, attribut, value):
try: try:
mycommon.myprint("objId = "+objId+" attribut "+attribut+" value = "+value) mycommon.myprint("objId = "+objId+" attribut "+attribut+" value = "+value)
client = MongoClient(CONNECTION_STRING)
dbname = client['cherifdb'] coll_name = MYSY_GV.dbname['myclass']
coll_name = dbname['myclass']
insertObject = [] insertObject = []
for x in coll_name.find({attribut: value}, {"_id": 0, "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, for x in coll_name.find({attribut: value}, {"_id": 0, "indexed": 0, "indexed_desc": 0, "indexed_obj": 0,
@ -186,7 +175,7 @@ def get_all_class(diction):
print(" My Data ="+str(mydata)) print(" My Data ="+str(mydata))
coll_name = dbname['myclass'] coll_name = MYSY_GV.dbname['myclass']
val_tmp = 1 val_tmp = 1
insertObject = [] insertObject = []
@ -194,11 +183,11 @@ def get_all_class(diction):
"valide": 0, "locked": 0, "partner_owner_recid": 0, }): "valide": 0, "locked": 0, "partner_owner_recid": 0, }):
#mycommon.myprint("AVANT ==> "+str(x['description'])) #mycommon.myprint("AVANT ==> "+str(x['description']))
val = x['description'] val = x['description']
if( len(x['description']) > MAX_CARACT ): if( len(x['description']) > MYSY_GV.MAX_CARACT ):
x['description'] = val[:MAX_CARACT]+" ..." x['description'] = val[:MYSY_GV.MAX_CARACT]+" ..."
else: else:
x['description'] = val[:MAX_CARACT] x['description'] = val[:MYSY_GV.MAX_CARACT]
#mycommon.myprint("APRES ==> " + str(x['description'])) #mycommon.myprint("APRES ==> " + str(x['description']))
@ -350,7 +339,7 @@ def recherche_text_simple(diction):
''' '''
user_recid = "None" user_recid = "None"
coll_search_result = dbname['user_recherche_result'] coll_search_result = MYSY_GV.dbname['user_recherche_result']
# Verification de la validité du token/mail dans le cas des user en mode connecté # Verification de la validité du token/mail dans le cas des user en mode connecté
if ( len(str(token)) > 0 ): if ( len(str(token)) > 0 ):
@ -433,7 +422,7 @@ def recherche_text_simple(diction):
#print( " unique = "+str(t)) #print( " unique = "+str(t))
final_message2.append(t) final_message2.append(t)
coll_name = dbname['myclass'] coll_name = MYSY_GV.dbname['myclass']
print(" Dans la recherch TIPS , les critère = " + str(certif_crit) + " la langue = " + str( print(" Dans la recherch TIPS , les critère = " + str(certif_crit) + " la langue = " + str(
lang_crit) + " price = " + str(price_crit)+" support = "+str(support_crit), lang_crit) + " price = " + str(price_crit)+" support = "+str(support_crit),
@ -445,7 +434,7 @@ def recherche_text_simple(diction):
#store_recherche_Id #store_recherche_Id
find_result = {'find_result':str(final_message2)} find_result = {'find_result':str(final_message2)}
tab_user_recherche = dbname['user_recherche'] tab_user_recherche = MYSY_GV.dbname['user_recherche']
# seules les formation avec locked = 0 et valide=1 sont modifiables # seules les formation avec locked = 0 et valide=1 sont modifiables
#print(str(inspect.stack()[0][3]) + " ENREG DES RESULT :"+str(store_recherche_Id)+" --- "+str(final_message2)) #print(str(inspect.stack()[0][3]) + " ENREG DES RESULT :"+str(store_recherche_Id)+" --- "+str(final_message2))
@ -493,10 +482,10 @@ def recherche_text_simple(diction):
user = x user = x
val = x['description'] val = x['description']
if (len(x['description']) > MAX_CARACT): if (len(x['description']) > MYSY_GV.MAX_CARACT):
x['description'] = val[:MAX_CARACT] + " ..." x['description'] = val[:MYSY_GV.MAX_CARACT] + " ..."
else: else:
x['description'] = val[:MAX_CARACT] x['description'] = val[:MYSY_GV.MAX_CARACT]
insertObject.append(JSONEncoder().encode(user)) insertObject.append(JSONEncoder().encode(user))
@ -528,7 +517,7 @@ def recherche_text_simple(diction):
mycommon.myprint(" pour phrase : #" + search_text + "#, voici la liste des formations") mycommon.myprint(" pour phrase : #" + search_text + "#, voici la liste des formations")
mycommon.myprint(tab_training) mycommon.myprint(tab_training)
coll_name = dbname['myclass'] coll_name = MYSY_GV.dbname['myclass']
''' '''
@ -564,10 +553,10 @@ def recherche_text_simple(diction):
user = x user = x
val = x['description'] val = x['description']
if (len(x['description']) > MAX_CARACT): if (len(x['description']) > MYSY_GV.MAX_CARACT):
x['description'] = val[:MAX_CARACT] + " ..." x['description'] = val[:MYSY_GV.MAX_CARACT] + " ..."
else: else:
x['description'] = val[:MAX_CARACT] x['description'] = val[:MYSY_GV.MAX_CARACT]
insertObject.append(JSONEncoder().encode(user)) insertObject.append(JSONEncoder().encode(user))
#print(insertObject) #print(insertObject)
@ -732,7 +721,7 @@ def store_recherche(diction, user_recid=""):
mydata['valide'] = diction['valide'] mydata['valide'] = diction['valide']
coll_name = dbname['user_recherche'] coll_name = MYSY_GV.dbname['user_recherche']
# Si le champ "id" est renseigné, il s'agit d'une mis jour # Si le champ "id" est renseigné, il s'agit d'une mis jour
mycommon.myprint(str(inspect.stack()[0][3]) + " on va stocker "+str(mydata)+" id = "+str(mydata_id)) mycommon.myprint(str(inspect.stack()[0][3]) + " on va stocker "+str(mydata)+" id = "+str(mydata_id))
@ -892,7 +881,7 @@ def recherche_tips_ret_ref(diction):
#mycommon.myprint(" pour phrase : #" + str(chaine[1]).lower() + "# , Pour le tips #"+ str(my_tips)+"#, voici la liste des formations") #mycommon.myprint(" pour phrase : #" + str(chaine[1]).lower() + "# , Pour le tips #"+ str(my_tips)+"#, voici la liste des formations")
#mycommon.myprint(tab_training) #mycommon.myprint(tab_training)
coll_name = dbname['myclass'] coll_name = MYSY_GV.dbname['myclass']
return True, tab_training return True, tab_training
@ -1034,7 +1023,7 @@ def recherche_tips(diction):
mycommon.myprint(" pour phrase : #" + str(chaine[1]).lower() + "# , Pour le tips #"+ str(my_tips)+"#, voici la liste des formations") mycommon.myprint(" pour phrase : #" + str(chaine[1]).lower() + "# , Pour le tips #"+ str(my_tips)+"#, voici la liste des formations")
mycommon.myprint(tab_training) mycommon.myprint(tab_training)
coll_name = dbname['myclass'] coll_name = MYSY_GV.dbname['myclass']
insertObject = [] insertObject = []
for x in coll_name.find({"external_code": {"$in": tab_training}}, {"_id": 0, "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0, for x in coll_name.find({"external_code": {"$in": tab_training}}, {"_id": 0, "indexed": 0, "indexed_desc": 0, "indexed_obj": 0, "indexed_title": 0,
@ -1042,10 +1031,10 @@ def recherche_tips(diction):
#mycommon.myprint(x) #mycommon.myprint(x)
user = x user = x
val = x['description'] val = x['description']
if (len(x['description']) > MAX_CARACT): if (len(x['description']) > MYSY_GV.MAX_CARACT):
x['description'] = val[:MAX_CARACT] + " ..." x['description'] = val[:MYSY_GV.MAX_CARACT] + " ..."
else: else:
x['description'] = val[:MAX_CARACT] x['description'] = val[:MYSY_GV.MAX_CARACT]
insertObject.append(JSONEncoder().encode(user)) insertObject.append(JSONEncoder().encode(user))
# mycommon.myprint(" insertObject = ", insertObject) # mycommon.myprint(" insertObject = ", insertObject)
@ -1125,7 +1114,7 @@ def get_stored_recherche(diction):
return False, " Impossible de recuperer l'historique de recherche" return False, " Impossible de recuperer l'historique de recherche"
coll_name = dbname['user_recherche'] coll_name = MYSY_GV.dbname['user_recherche']
RetObject = [] RetObject = []
for retVal in coll_name.find({'user_recid': user_recid, 'valide':'1'}): for retVal in coll_name.find({'user_recid': user_recid, 'valide':'1'}):

96
youtubes_analyse.py Normal file
View File

@ -0,0 +1,96 @@
import GlobalVariable as MYSY_GV
from pymongo import MongoClient
from collections import Counter
import prj_common as mycommon
from unidecode import unidecode
import inspect
import sys, os
from autocorrect import Speller
from datetime import datetime
import re
import Ela_Spacy as ls
def YTUBES_ela_index_class(external_code="", source_field = ""):
try:
collection = MYSY_GV.YTUBES_dbname["mysyclass"]
print(" lalalalal1")
if( len(str(source_field)) == 0):
source_field = 'default'
print(" external_code = "+str(external_code)+ " source_field = "+str(source_field))
for doc in collection.find({"external_code":external_code}):
print(" 3333")
# Indexation du champ : video_text , donc la transcription d'une video
if (str(source_field) == "video_text"):
print(" lalalalal")
chaine = doc["video_text"]
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
retval = ls.YTUBES_ela_index_record_field(chaine, str(doc["external_code"]), source_field)
'''
# Apres indexation, on met a jour la formation pour
# mettre le champ "indexe" a 1.
'''
if (retval):
# mise à jour du champ "indexed"
myobjectif = {}
myobjectif['indexed_video_text'] = '0'
ret_val2 = collection.find_and_modify(query={"external_code": external_code},
update={"$set": myobjectif}
)
if (ret_val2 and ret_val2['_id']):
mycommon.myprint("La formation indexée YTUBES (champs video_text) =" + str(ret_val2['_id']))
return True
else:
return False
return True
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False
def YTUBES_RedText():
try:
print(" on est dans ytubes read")
Y_collection = MYSY_GV.YTUBES_dbname["mysyclass"]
nb_class_indexed = 0
for doc in Y_collection.find({'indexed_video_text': '0'}):
if ("video_text" in doc.keys()):
chaine = doc["video_text"]
#mycommon.myprint(" video_texte = " + str(doc["external_code"]) + " ==> " + str(chaine))
retval = YTUBES_ela_index_class(str(doc["external_code"]), "video_text")
# print(" apres : ela_index_all_classes_video_texte retval = " + str(retval))
if (retval is False):
mycommon.myprint("IMPOSSIBLE d'indexer la formation = " + str(doc["external_code"]))
else:
nb_class_indexed = nb_class_indexed + 1
# retval = ls.ela_index_record_field(chaine, str(doc["external_code"]))
mycommon.myprint(str(nb_class_indexed) + " ont été indexes YTUBES- champ video_text")
return True, str(nb_class_indexed) + " ont été indexes YTUBES - champ video_text "
return True
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False