29/04/22 - 17h30

master
ChérifBALDE 2022-04-29 17:29:31 +02:00 committed by cherif
parent 934b21de5a
commit ae76aa23a6
6 changed files with 123 additions and 73 deletions

View File

@ -478,10 +478,10 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
## Suppression de toutes les indexation qui existe de cette formation
'''
coll_name = MYSY_GV.YTUBES_dbname['mysyindex']
coll_name_StopWord = MYSY_GV.YTUBES_dbname['mysystopwords']
coll_name = MYSY_GV.dbname['elaindex']
coll_name_StopWord = MYSY_GV.dbname['mysystopwords']
print("lolololo")
# check default value of parameter : source_field
if len(str(source_field)) == 0 :
source_field = 'default'
@ -497,9 +497,6 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
if( status is False):
return False
#print(" AFFICHAGE TAB TOKEN")
#print(tab_tokens)
#print(" FINNN TAB TOKEN")
status, tab_tokens2 = Ela_remove_stop_words(tab_tokens)
if (status is False):
return False
@ -520,7 +517,7 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
#print(str(tab_tokens3_unicode))
status, tab_tokens4 = Ela_stemmize_Class(tab_tokens3)
status, tab_tokens4 = Ela_stemmize_Class(tab_tokens3_unicode)
if (status is False):
return False
@ -528,7 +525,7 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
#tab_tokens4 = Ela_remove_ponct(tab_tokens4)
# Enregistrement dans la base mongodb
YTUBES_Ela_list_to_mongo(tab_tokens3_unicode,class_id, source_field)
YTUBES_Ela_list_to_mongo(tab_tokens4,class_id, source_field)
size_tab = len(tab_tokens4)
#print("size_tab = " + str(size_tab))
@ -629,7 +626,7 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field):
for tmp in unique_list:
moyenne = round(int(occurrences[str(tmp)]) / size_tab, 3)
if( moyenne >= seuil ):
print(str(moyenne)+ " CMP "+str(seuil))
#print(str(moyenne)+ " CMP "+str(seuil))
final_lists.append([str(tmp), int(str(occurrences[str(tmp)])), str(moyenne), str(traning_id), str(source_field)])
#print(final_lists)
@ -639,12 +636,16 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field):
print(" YTUBES INSERT MAY")
'''
YTUBES_CONNECTION_STRING = "mongodb://localhost:27017/"
YTUBES_client = MongoClient(YTUBES_CONNECTION_STRING)
YTUBES_dbname = YTUBES_client['mysyvideodb']
collection = YTUBES_dbname["mysyindex"]
'''
collection = MYSY_GV.dbname["elaindex"]
data.reset_index(inplace=True)
data_dict = data.to_dict("records")
@ -652,7 +653,7 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field):
#print(data_dict)
# Insert collection
collection.insert_many(data_dict)
data.to_csv("data_indexees.csv")
#data.to_csv("data_indexees.csv")
return True
except Exception as e :

View File

@ -71,3 +71,8 @@ lsc.update_stopWords(stopWords)
lsc.update_token_fr_pontuation(token_fr_pontuation)
spell_fr = Speller(lang='fr')
'''
Repertoire de stockage de video YouTubes
'''
YTUBES_ENSCRIPT_LOCATION = ""

View File

@ -1,21 +1,29 @@
,index,mots,occurence,moyenne,id_formation,source_field
0,0,dynamiques,7,0.012,DmZxdIvG3YQ,video_text
1,1,valeurs,5,0.009,DmZxdIvG3YQ,video_text
2,2,emporter,5,0.009,DmZxdIvG3YQ,video_text
3,3,colonne,6,0.011,DmZxdIvG3YQ,video_text
4,4,graphique,6,0.011,DmZxdIvG3YQ,video_text
5,5,nombre,5,0.009,DmZxdIvG3YQ,video_text
6,6,vendu,8,0.014,DmZxdIvG3YQ,video_text
7,7,prix,10,0.018,DmZxdIvG3YQ,video_text
8,8,croises,6,0.011,DmZxdIvG3YQ,video_text
9,9,somme,5,0.009,DmZxdIvG3YQ,video_text
10,10,cellules,5,0.009,DmZxdIvG3YQ,video_text
11,11,fonction,5,0.009,DmZxdIvG3YQ,video_text
12,12,groupe,7,0.012,DmZxdIvG3YQ,video_text
13,13,tableaux,7,0.012,DmZxdIvG3YQ,video_text
14,14,modifier,6,0.011,DmZxdIvG3YQ,video_text
15,15,tableau,7,0.012,DmZxdIvG3YQ,video_text
16,16,donnees,8,0.014,DmZxdIvG3YQ,video_text
17,17,vis,6,0.011,DmZxdIvG3YQ,video_text
18,18,produits,12,0.021,DmZxdIvG3YQ,video_text
19,19,besoin,6,0.011,DmZxdIvG3YQ,video_text
0,0,driv,1,0.03,PUjv9Ua5db4,description
1,1,cump,1,0.03,PUjv9Ua5db4,description
2,2,resultat,1,0.03,PUjv9Ua5db4,description
3,3,cout,4,0.12,PUjv9Ua5db4,description
4,4,inventair,1,0.03,PUjv9Ua5db4,description
5,5,fil,1,0.03,PUjv9Ua5db4,description
6,6,repartit,1,0.03,PUjv9Ua5db4,description
7,7,revient,1,0.03,PUjv9Ua5db4,description
8,8,stock,1,0.03,PUjv9Ua5db4,description
9,9,distribu,1,0.03,PUjv9Ua5db4,description
10,10,googl,1,0.03,PUjv9Ua5db4,description
11,11,revis,1,0.03,PUjv9Ua5db4,description
12,12,http,1,0.03,PUjv9Ua5db4,description
13,13,approvision,1,0.03,PUjv9Ua5db4,description
14,14,comptabl,1,0.03,PUjv9Ua5db4,description
15,15,indirect,1,0.03,PUjv9Ua5db4,description
16,16,analyt,2,0.06,PUjv9Ua5db4,description
17,17,tableau,1,0.03,PUjv9Ua5db4,description
18,18,method,1,0.03,PUjv9Ua5db4,description
19,19,charg,1,0.03,PUjv9Ua5db4,description
20,20,product,1,0.03,PUjv9Ua5db4,description
21,21,usp,1,0.03,PUjv9Ua5db4,description
22,22,support,1,0.03,PUjv9Ua5db4,description
23,23,sujet,1,0.03,PUjv9Ua5db4,description
24,24,etap,1,0.03,PUjv9Ua5db4,description
25,25,view,1,0.03,PUjv9Ua5db4,description
26,26,sharing,1,0.03,PUjv9Ua5db4,description
27,27,com,1,0.03,PUjv9Ua5db4,description

1 index mots occurence moyenne id_formation source_field
2 0 0 dynamiques driv 7 1 0.012 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
3 1 1 valeurs cump 5 1 0.009 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
4 2 2 emporter resultat 5 1 0.009 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
5 3 3 colonne cout 6 4 0.011 0.12 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
6 4 4 graphique inventair 6 1 0.011 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
7 5 5 nombre fil 5 1 0.009 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
8 6 6 vendu repartit 8 1 0.014 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
9 7 7 prix revient 10 1 0.018 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
10 8 8 croises stock 6 1 0.011 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
11 9 9 somme distribu 5 1 0.009 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
12 10 10 cellules googl 5 1 0.009 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
13 11 11 fonction revis 5 1 0.009 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
14 12 12 groupe http 7 1 0.012 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
15 13 13 tableaux approvision 7 1 0.012 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
16 14 14 modifier comptabl 6 1 0.011 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
17 15 15 tableau indirect 7 1 0.012 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
18 16 16 donnees analyt 8 2 0.014 0.06 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
19 17 17 vis tableau 6 1 0.011 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
20 18 18 produits method 12 1 0.021 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
21 19 19 besoin charg 6 1 0.011 0.03 DmZxdIvG3YQ PUjv9Ua5db4 video_text description
22 20 20 product 1 0.03 PUjv9Ua5db4 description
23 21 21 usp 1 0.03 PUjv9Ua5db4 description
24 22 22 support 1 0.03 PUjv9Ua5db4 description
25 23 23 sujet 1 0.03 PUjv9Ua5db4 description
26 24 24 etap 1 0.03 PUjv9Ua5db4 description
27 25 25 view 1 0.03 PUjv9Ua5db4 description
28 26 26 sharing 1 0.03 PUjv9Ua5db4 description
29 27 27 com 1 0.03 PUjv9Ua5db4 description

View File

@ -1,21 +1,29 @@
mots occurence moyenne id_formation source_field
0 dynamiques 7 0.012 DmZxdIvG3YQ video_text
1 valeurs 5 0.009 DmZxdIvG3YQ video_text
2 emporter 5 0.009 DmZxdIvG3YQ video_text
3 colonne 6 0.011 DmZxdIvG3YQ video_text
4 graphique 6 0.011 DmZxdIvG3YQ video_text
5 nombre 5 0.009 DmZxdIvG3YQ video_text
6 vendu 8 0.014 DmZxdIvG3YQ video_text
7 prix 10 0.018 DmZxdIvG3YQ video_text
8 croises 6 0.011 DmZxdIvG3YQ video_text
9 somme 5 0.009 DmZxdIvG3YQ video_text
10 cellules 5 0.009 DmZxdIvG3YQ video_text
11 fonction 5 0.009 DmZxdIvG3YQ video_text
12 groupe 7 0.012 DmZxdIvG3YQ video_text
13 tableaux 7 0.012 DmZxdIvG3YQ video_text
14 modifier 6 0.011 DmZxdIvG3YQ video_text
15 tableau 7 0.012 DmZxdIvG3YQ video_text
16 donnees 8 0.014 DmZxdIvG3YQ video_text
17 vis 6 0.011 DmZxdIvG3YQ video_text
18 produits 12 0.021 DmZxdIvG3YQ video_text
19 besoin 6 0.011 DmZxdIvG3YQ video_text
mots occurence moyenne id_formation source_field
0 driv 1 0.03 PUjv9Ua5db4 description
1 cump 1 0.03 PUjv9Ua5db4 description
2 resultat 1 0.03 PUjv9Ua5db4 description
3 cout 4 0.12 PUjv9Ua5db4 description
4 inventair 1 0.03 PUjv9Ua5db4 description
5 fil 1 0.03 PUjv9Ua5db4 description
6 repartit 1 0.03 PUjv9Ua5db4 description
7 revient 1 0.03 PUjv9Ua5db4 description
8 stock 1 0.03 PUjv9Ua5db4 description
9 distribu 1 0.03 PUjv9Ua5db4 description
10 googl 1 0.03 PUjv9Ua5db4 description
11 revis 1 0.03 PUjv9Ua5db4 description
12 http 1 0.03 PUjv9Ua5db4 description
13 approvision 1 0.03 PUjv9Ua5db4 description
14 comptabl 1 0.03 PUjv9Ua5db4 description
15 indirect 1 0.03 PUjv9Ua5db4 description
16 analyt 2 0.06 PUjv9Ua5db4 description
17 tableau 1 0.03 PUjv9Ua5db4 description
18 method 1 0.03 PUjv9Ua5db4 description
19 charg 1 0.03 PUjv9Ua5db4 description
20 product 1 0.03 PUjv9Ua5db4 description
21 usp 1 0.03 PUjv9Ua5db4 description
22 support 1 0.03 PUjv9Ua5db4 description
23 sujet 1 0.03 PUjv9Ua5db4 description
24 etap 1 0.03 PUjv9Ua5db4 description
25 view 1 0.03 PUjv9Ua5db4 description
26 sharing 1 0.03 PUjv9Ua5db4 description
27 com 1 0.03 PUjv9Ua5db4 description

View File

@ -33,7 +33,6 @@ app.config['UPLOAD_FOLDER'] = MYSY_GV.upload_folder
@app.before_request
def before_request():
if mycommon.check_source_ipv4(str(request.remote_addr)) is False:
return "Security check : '" + request.remote_addr + "' is not autorized", 404
@ -619,13 +618,13 @@ def ela_index_all_classes_video_text():
''' Analyse Youtubes
'''
@app.route('/myclass/api/YTUBES_RedText/', methods=['POST','GET'])
@app.route('/myclass/api/ela_index_all_ytubes_video_transcription/', methods=['POST','GET'])
@crossdomain(origin='*')
def YTUBES_RedText():
def ela_index_all_ytubes_video_transcription():
# On recupere le corps (payload) de la requete
payload = request.form.to_dict()
print(" ### payload = ",payload)
status, retval = YTA.YTUBES_RedText()
status, retval = YTA.ela_index_all_ytubes_video_transcription()
return jsonify(status=status, message=retval)

View File

@ -14,24 +14,34 @@ import Ela_Spacy as ls
def YTUBES_ela_index_class(external_code="", source_field = ""):
def YTUBES_ela_index_class_transcription(external_code="", source_field = ""):
try:
collection = MYSY_GV.YTUBES_dbname["mysyclass"]
collection = MYSY_GV.dbname["myclass"]
print(" lalalalal1")
if( len(str(source_field)) == 0):
source_field = 'default'
print(" external_code = "+str(external_code)+ " source_field = "+str(source_field))
for doc in collection.find({"external_code":external_code}):
print(" 3333")
for doc in collection.find({"external_code":external_code, 'owner':'youtubes'}):
# Indexation du champ : video_text , donc la transcription d'une video
if (str(source_field) == "video_text"):
print(" lalalalal")
chaine = doc["video_text"]
# Recuperation du nom du fichier
if ("transciption_file_name" in doc.keys()):
file_name = doc["transciption_file_name"]
print(' file_name = ' + str(file_name))
status, Ytubes_text = YTUBES_Read_Transcritpion(file_name)
if (status is False):
return False
print(" text indexer "+ Ytubes_text[:MYSY_GV.MAX_CARACT] + " ...")
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
retval = ls.YTUBES_ela_index_record_field(chaine, str(doc["external_code"]), source_field)
retval = ls.YTUBES_ela_index_record_field(Ytubes_text, str(doc["external_code"]), source_field)
'''
# Apres indexation, on met a jour la formation pour
@ -40,7 +50,7 @@ def YTUBES_ela_index_class(external_code="", source_field = ""):
if (retval):
# mise à jour du champ "indexed"
myobjectif = {}
myobjectif['indexed_video_text'] = '0'
myobjectif['indexed_video_text'] = '1'
ret_val2 = collection.find_and_modify(query={"external_code": external_code},
update={"$set": myobjectif}
@ -59,22 +69,43 @@ def YTUBES_ela_index_class(external_code="", source_field = ""):
return False
def YTUBES_RedText():
'''
Cette fonction va lire la transcription youtubes qui se trouve
dans un fichier '''
def YTUBES_Read_Transcritpion(transcitp_file_name = None):
try:
text = ""
with open(transcitp_file_name, encoding="utf-8") as f:
text = text + f.read()
return True, text
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
return False, ""
'''
'''
def ela_index_all_ytubes_video_transcription():
try:
print(" on est dans ytubes read")
Y_collection = MYSY_GV.YTUBES_dbname["mysyclass"]
Y_collection = MYSY_GV.dbname["myclass"]
Ytubes_text = ""
nb_class_indexed = 0
for doc in Y_collection.find({'indexed_video_text': '0'}):
if ("video_text" in doc.keys()):
chaine = doc["video_text"]
#mycommon.myprint(" video_texte = " + str(doc["external_code"]) + " ==> " + str(chaine))
for doc in Y_collection.find({'indexed_video_text': '0', 'owner':'youtubes'}):
retval = YTUBES_ela_index_class(str(doc["external_code"]), "video_text")
if ("transciption_file_name" in doc.keys()):
retval = YTUBES_ela_index_class_transcription(str(doc["external_code"]), "video_text")
# print(" apres : ela_index_all_classes_video_texte retval = " + str(retval))
if (retval is False):
@ -84,12 +115,10 @@ def YTUBES_RedText():
# retval = ls.ela_index_record_field(chaine, str(doc["external_code"]))
mycommon.myprint(str(nb_class_indexed) + " ont été indexes YTUBES- champ video_text")
return True, str(nb_class_indexed) + " ont été indexes YTUBES - champ video_text "
return True
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))