29/04/22 - 17h30
parent
934b21de5a
commit
ae76aa23a6
21
Ela_Spacy.py
21
Ela_Spacy.py
|
@ -478,10 +478,10 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
|
||||||
## Suppression de toutes les indexation qui existe de cette formation
|
## Suppression de toutes les indexation qui existe de cette formation
|
||||||
'''
|
'''
|
||||||
|
|
||||||
coll_name = MYSY_GV.YTUBES_dbname['mysyindex']
|
coll_name = MYSY_GV.dbname['elaindex']
|
||||||
coll_name_StopWord = MYSY_GV.YTUBES_dbname['mysystopwords']
|
coll_name_StopWord = MYSY_GV.dbname['mysystopwords']
|
||||||
|
|
||||||
|
|
||||||
print("lolololo")
|
|
||||||
# check default value of parameter : source_field
|
# check default value of parameter : source_field
|
||||||
if len(str(source_field)) == 0 :
|
if len(str(source_field)) == 0 :
|
||||||
source_field = 'default'
|
source_field = 'default'
|
||||||
|
@ -497,9 +497,6 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
|
||||||
if( status is False):
|
if( status is False):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
#print(" AFFICHAGE TAB TOKEN")
|
|
||||||
#print(tab_tokens)
|
|
||||||
#print(" FINNN TAB TOKEN")
|
|
||||||
status, tab_tokens2 = Ela_remove_stop_words(tab_tokens)
|
status, tab_tokens2 = Ela_remove_stop_words(tab_tokens)
|
||||||
if (status is False):
|
if (status is False):
|
||||||
return False
|
return False
|
||||||
|
@ -520,7 +517,7 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
|
||||||
|
|
||||||
|
|
||||||
#print(str(tab_tokens3_unicode))
|
#print(str(tab_tokens3_unicode))
|
||||||
status, tab_tokens4 = Ela_stemmize_Class(tab_tokens3)
|
status, tab_tokens4 = Ela_stemmize_Class(tab_tokens3_unicode)
|
||||||
if (status is False):
|
if (status is False):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
@ -528,7 +525,7 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
|
||||||
#tab_tokens4 = Ela_remove_ponct(tab_tokens4)
|
#tab_tokens4 = Ela_remove_ponct(tab_tokens4)
|
||||||
|
|
||||||
# Enregistrement dans la base mongodb
|
# Enregistrement dans la base mongodb
|
||||||
YTUBES_Ela_list_to_mongo(tab_tokens3_unicode,class_id, source_field)
|
YTUBES_Ela_list_to_mongo(tab_tokens4,class_id, source_field)
|
||||||
|
|
||||||
size_tab = len(tab_tokens4)
|
size_tab = len(tab_tokens4)
|
||||||
#print("size_tab = " + str(size_tab))
|
#print("size_tab = " + str(size_tab))
|
||||||
|
@ -629,7 +626,7 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field):
|
||||||
for tmp in unique_list:
|
for tmp in unique_list:
|
||||||
moyenne = round(int(occurrences[str(tmp)]) / size_tab, 3)
|
moyenne = round(int(occurrences[str(tmp)]) / size_tab, 3)
|
||||||
if( moyenne >= seuil ):
|
if( moyenne >= seuil ):
|
||||||
print(str(moyenne)+ " CMP "+str(seuil))
|
#print(str(moyenne)+ " CMP "+str(seuil))
|
||||||
final_lists.append([str(tmp), int(str(occurrences[str(tmp)])), str(moyenne), str(traning_id), str(source_field)])
|
final_lists.append([str(tmp), int(str(occurrences[str(tmp)])), str(moyenne), str(traning_id), str(source_field)])
|
||||||
|
|
||||||
#print(final_lists)
|
#print(final_lists)
|
||||||
|
@ -639,12 +636,16 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field):
|
||||||
|
|
||||||
print(" YTUBES INSERT MAY")
|
print(" YTUBES INSERT MAY")
|
||||||
|
|
||||||
|
'''
|
||||||
YTUBES_CONNECTION_STRING = "mongodb://localhost:27017/"
|
YTUBES_CONNECTION_STRING = "mongodb://localhost:27017/"
|
||||||
YTUBES_client = MongoClient(YTUBES_CONNECTION_STRING)
|
YTUBES_client = MongoClient(YTUBES_CONNECTION_STRING)
|
||||||
YTUBES_dbname = YTUBES_client['mysyvideodb']
|
YTUBES_dbname = YTUBES_client['mysyvideodb']
|
||||||
|
|
||||||
|
|
||||||
collection = YTUBES_dbname["mysyindex"]
|
collection = YTUBES_dbname["mysyindex"]
|
||||||
|
'''
|
||||||
|
|
||||||
|
collection = MYSY_GV.dbname["elaindex"]
|
||||||
|
|
||||||
data.reset_index(inplace=True)
|
data.reset_index(inplace=True)
|
||||||
data_dict = data.to_dict("records")
|
data_dict = data.to_dict("records")
|
||||||
|
@ -652,7 +653,7 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field):
|
||||||
#print(data_dict)
|
#print(data_dict)
|
||||||
# Insert collection
|
# Insert collection
|
||||||
collection.insert_many(data_dict)
|
collection.insert_many(data_dict)
|
||||||
data.to_csv("data_indexees.csv")
|
#data.to_csv("data_indexees.csv")
|
||||||
|
|
||||||
return True
|
return True
|
||||||
except Exception as e :
|
except Exception as e :
|
||||||
|
|
|
@ -71,3 +71,8 @@ lsc.update_stopWords(stopWords)
|
||||||
lsc.update_token_fr_pontuation(token_fr_pontuation)
|
lsc.update_token_fr_pontuation(token_fr_pontuation)
|
||||||
spell_fr = Speller(lang='fr')
|
spell_fr = Speller(lang='fr')
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
Repertoire de stockage de video YouTubes
|
||||||
|
'''
|
||||||
|
YTUBES_ENSCRIPT_LOCATION = ""
|
||||||
|
|
|
@ -1,21 +1,29 @@
|
||||||
,index,mots,occurence,moyenne,id_formation,source_field
|
,index,mots,occurence,moyenne,id_formation,source_field
|
||||||
0,0,dynamiques,7,0.012,DmZxdIvG3YQ,video_text
|
0,0,driv,1,0.03,PUjv9Ua5db4,description
|
||||||
1,1,valeurs,5,0.009,DmZxdIvG3YQ,video_text
|
1,1,cump,1,0.03,PUjv9Ua5db4,description
|
||||||
2,2,emporter,5,0.009,DmZxdIvG3YQ,video_text
|
2,2,resultat,1,0.03,PUjv9Ua5db4,description
|
||||||
3,3,colonne,6,0.011,DmZxdIvG3YQ,video_text
|
3,3,cout,4,0.12,PUjv9Ua5db4,description
|
||||||
4,4,graphique,6,0.011,DmZxdIvG3YQ,video_text
|
4,4,inventair,1,0.03,PUjv9Ua5db4,description
|
||||||
5,5,nombre,5,0.009,DmZxdIvG3YQ,video_text
|
5,5,fil,1,0.03,PUjv9Ua5db4,description
|
||||||
6,6,vendu,8,0.014,DmZxdIvG3YQ,video_text
|
6,6,repartit,1,0.03,PUjv9Ua5db4,description
|
||||||
7,7,prix,10,0.018,DmZxdIvG3YQ,video_text
|
7,7,revient,1,0.03,PUjv9Ua5db4,description
|
||||||
8,8,croises,6,0.011,DmZxdIvG3YQ,video_text
|
8,8,stock,1,0.03,PUjv9Ua5db4,description
|
||||||
9,9,somme,5,0.009,DmZxdIvG3YQ,video_text
|
9,9,distribu,1,0.03,PUjv9Ua5db4,description
|
||||||
10,10,cellules,5,0.009,DmZxdIvG3YQ,video_text
|
10,10,googl,1,0.03,PUjv9Ua5db4,description
|
||||||
11,11,fonction,5,0.009,DmZxdIvG3YQ,video_text
|
11,11,revis,1,0.03,PUjv9Ua5db4,description
|
||||||
12,12,groupe,7,0.012,DmZxdIvG3YQ,video_text
|
12,12,http,1,0.03,PUjv9Ua5db4,description
|
||||||
13,13,tableaux,7,0.012,DmZxdIvG3YQ,video_text
|
13,13,approvision,1,0.03,PUjv9Ua5db4,description
|
||||||
14,14,modifier,6,0.011,DmZxdIvG3YQ,video_text
|
14,14,comptabl,1,0.03,PUjv9Ua5db4,description
|
||||||
15,15,tableau,7,0.012,DmZxdIvG3YQ,video_text
|
15,15,indirect,1,0.03,PUjv9Ua5db4,description
|
||||||
16,16,donnees,8,0.014,DmZxdIvG3YQ,video_text
|
16,16,analyt,2,0.06,PUjv9Ua5db4,description
|
||||||
17,17,vis,6,0.011,DmZxdIvG3YQ,video_text
|
17,17,tableau,1,0.03,PUjv9Ua5db4,description
|
||||||
18,18,produits,12,0.021,DmZxdIvG3YQ,video_text
|
18,18,method,1,0.03,PUjv9Ua5db4,description
|
||||||
19,19,besoin,6,0.011,DmZxdIvG3YQ,video_text
|
19,19,charg,1,0.03,PUjv9Ua5db4,description
|
||||||
|
20,20,product,1,0.03,PUjv9Ua5db4,description
|
||||||
|
21,21,usp,1,0.03,PUjv9Ua5db4,description
|
||||||
|
22,22,support,1,0.03,PUjv9Ua5db4,description
|
||||||
|
23,23,sujet,1,0.03,PUjv9Ua5db4,description
|
||||||
|
24,24,etap,1,0.03,PUjv9Ua5db4,description
|
||||||
|
25,25,view,1,0.03,PUjv9Ua5db4,description
|
||||||
|
26,26,sharing,1,0.03,PUjv9Ua5db4,description
|
||||||
|
27,27,com,1,0.03,PUjv9Ua5db4,description
|
||||||
|
|
|
|
@ -1,21 +1,29 @@
|
||||||
mots occurence moyenne id_formation source_field
|
mots occurence moyenne id_formation source_field
|
||||||
0 dynamiques 7 0.012 DmZxdIvG3YQ video_text
|
0 driv 1 0.03 PUjv9Ua5db4 description
|
||||||
1 valeurs 5 0.009 DmZxdIvG3YQ video_text
|
1 cump 1 0.03 PUjv9Ua5db4 description
|
||||||
2 emporter 5 0.009 DmZxdIvG3YQ video_text
|
2 resultat 1 0.03 PUjv9Ua5db4 description
|
||||||
3 colonne 6 0.011 DmZxdIvG3YQ video_text
|
3 cout 4 0.12 PUjv9Ua5db4 description
|
||||||
4 graphique 6 0.011 DmZxdIvG3YQ video_text
|
4 inventair 1 0.03 PUjv9Ua5db4 description
|
||||||
5 nombre 5 0.009 DmZxdIvG3YQ video_text
|
5 fil 1 0.03 PUjv9Ua5db4 description
|
||||||
6 vendu 8 0.014 DmZxdIvG3YQ video_text
|
6 repartit 1 0.03 PUjv9Ua5db4 description
|
||||||
7 prix 10 0.018 DmZxdIvG3YQ video_text
|
7 revient 1 0.03 PUjv9Ua5db4 description
|
||||||
8 croises 6 0.011 DmZxdIvG3YQ video_text
|
8 stock 1 0.03 PUjv9Ua5db4 description
|
||||||
9 somme 5 0.009 DmZxdIvG3YQ video_text
|
9 distribu 1 0.03 PUjv9Ua5db4 description
|
||||||
10 cellules 5 0.009 DmZxdIvG3YQ video_text
|
10 googl 1 0.03 PUjv9Ua5db4 description
|
||||||
11 fonction 5 0.009 DmZxdIvG3YQ video_text
|
11 revis 1 0.03 PUjv9Ua5db4 description
|
||||||
12 groupe 7 0.012 DmZxdIvG3YQ video_text
|
12 http 1 0.03 PUjv9Ua5db4 description
|
||||||
13 tableaux 7 0.012 DmZxdIvG3YQ video_text
|
13 approvision 1 0.03 PUjv9Ua5db4 description
|
||||||
14 modifier 6 0.011 DmZxdIvG3YQ video_text
|
14 comptabl 1 0.03 PUjv9Ua5db4 description
|
||||||
15 tableau 7 0.012 DmZxdIvG3YQ video_text
|
15 indirect 1 0.03 PUjv9Ua5db4 description
|
||||||
16 donnees 8 0.014 DmZxdIvG3YQ video_text
|
16 analyt 2 0.06 PUjv9Ua5db4 description
|
||||||
17 vis 6 0.011 DmZxdIvG3YQ video_text
|
17 tableau 1 0.03 PUjv9Ua5db4 description
|
||||||
18 produits 12 0.021 DmZxdIvG3YQ video_text
|
18 method 1 0.03 PUjv9Ua5db4 description
|
||||||
19 besoin 6 0.011 DmZxdIvG3YQ video_text
|
19 charg 1 0.03 PUjv9Ua5db4 description
|
||||||
|
20 product 1 0.03 PUjv9Ua5db4 description
|
||||||
|
21 usp 1 0.03 PUjv9Ua5db4 description
|
||||||
|
22 support 1 0.03 PUjv9Ua5db4 description
|
||||||
|
23 sujet 1 0.03 PUjv9Ua5db4 description
|
||||||
|
24 etap 1 0.03 PUjv9Ua5db4 description
|
||||||
|
25 view 1 0.03 PUjv9Ua5db4 description
|
||||||
|
26 sharing 1 0.03 PUjv9Ua5db4 description
|
||||||
|
27 com 1 0.03 PUjv9Ua5db4 description
|
7
main.py
7
main.py
|
@ -33,7 +33,6 @@ app.config['UPLOAD_FOLDER'] = MYSY_GV.upload_folder
|
||||||
|
|
||||||
@app.before_request
|
@app.before_request
|
||||||
def before_request():
|
def before_request():
|
||||||
|
|
||||||
if mycommon.check_source_ipv4(str(request.remote_addr)) is False:
|
if mycommon.check_source_ipv4(str(request.remote_addr)) is False:
|
||||||
return "Security check : '" + request.remote_addr + "' is not autorized", 404
|
return "Security check : '" + request.remote_addr + "' is not autorized", 404
|
||||||
|
|
||||||
|
@ -619,13 +618,13 @@ def ela_index_all_classes_video_text():
|
||||||
|
|
||||||
''' Analyse Youtubes
|
''' Analyse Youtubes
|
||||||
'''
|
'''
|
||||||
@app.route('/myclass/api/YTUBES_RedText/', methods=['POST','GET'])
|
@app.route('/myclass/api/ela_index_all_ytubes_video_transcription/', methods=['POST','GET'])
|
||||||
@crossdomain(origin='*')
|
@crossdomain(origin='*')
|
||||||
def YTUBES_RedText():
|
def ela_index_all_ytubes_video_transcription():
|
||||||
# On recupere le corps (payload) de la requete
|
# On recupere le corps (payload) de la requete
|
||||||
payload = request.form.to_dict()
|
payload = request.form.to_dict()
|
||||||
print(" ### payload = ",payload)
|
print(" ### payload = ",payload)
|
||||||
status, retval = YTA.YTUBES_RedText()
|
status, retval = YTA.ela_index_all_ytubes_video_transcription()
|
||||||
return jsonify(status=status, message=retval)
|
return jsonify(status=status, message=retval)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -14,24 +14,34 @@ import Ela_Spacy as ls
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def YTUBES_ela_index_class(external_code="", source_field = ""):
|
def YTUBES_ela_index_class_transcription(external_code="", source_field = ""):
|
||||||
try:
|
try:
|
||||||
|
|
||||||
collection = MYSY_GV.YTUBES_dbname["mysyclass"]
|
collection = MYSY_GV.YTUBES_dbname["mysyclass"]
|
||||||
|
collection = MYSY_GV.dbname["myclass"]
|
||||||
|
|
||||||
print(" lalalalal1")
|
|
||||||
if( len(str(source_field)) == 0):
|
if( len(str(source_field)) == 0):
|
||||||
source_field = 'default'
|
source_field = 'default'
|
||||||
|
|
||||||
print(" external_code = "+str(external_code)+ " source_field = "+str(source_field))
|
print(" external_code = "+str(external_code)+ " source_field = "+str(source_field))
|
||||||
for doc in collection.find({"external_code":external_code}):
|
for doc in collection.find({"external_code":external_code, 'owner':'youtubes'}):
|
||||||
print(" 3333")
|
|
||||||
# Indexation du champ : video_text , donc la transcription d'une video
|
# Indexation du champ : video_text , donc la transcription d'une video
|
||||||
if (str(source_field) == "video_text"):
|
if (str(source_field) == "video_text"):
|
||||||
print(" lalalalal")
|
print(" lalalalal")
|
||||||
chaine = doc["video_text"]
|
|
||||||
|
# Recuperation du nom du fichier
|
||||||
|
if ("transciption_file_name" in doc.keys()):
|
||||||
|
file_name = doc["transciption_file_name"]
|
||||||
|
print(' file_name = ' + str(file_name))
|
||||||
|
status, Ytubes_text = YTUBES_Read_Transcritpion(file_name)
|
||||||
|
if (status is False):
|
||||||
|
return False
|
||||||
|
|
||||||
|
print(" text indexer "+ Ytubes_text[:MYSY_GV.MAX_CARACT] + " ...")
|
||||||
|
|
||||||
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
|
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
|
||||||
retval = ls.YTUBES_ela_index_record_field(chaine, str(doc["external_code"]), source_field)
|
retval = ls.YTUBES_ela_index_record_field(Ytubes_text, str(doc["external_code"]), source_field)
|
||||||
|
|
||||||
'''
|
'''
|
||||||
# Apres indexation, on met a jour la formation pour
|
# Apres indexation, on met a jour la formation pour
|
||||||
|
@ -40,7 +50,7 @@ def YTUBES_ela_index_class(external_code="", source_field = ""):
|
||||||
if (retval):
|
if (retval):
|
||||||
# mise à jour du champ "indexed"
|
# mise à jour du champ "indexed"
|
||||||
myobjectif = {}
|
myobjectif = {}
|
||||||
myobjectif['indexed_video_text'] = '0'
|
myobjectif['indexed_video_text'] = '1'
|
||||||
|
|
||||||
ret_val2 = collection.find_and_modify(query={"external_code": external_code},
|
ret_val2 = collection.find_and_modify(query={"external_code": external_code},
|
||||||
update={"$set": myobjectif}
|
update={"$set": myobjectif}
|
||||||
|
@ -59,22 +69,43 @@ def YTUBES_ela_index_class(external_code="", source_field = ""):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def YTUBES_RedText():
|
|
||||||
|
'''
|
||||||
|
Cette fonction va lire la transcription youtubes qui se trouve
|
||||||
|
dans un fichier '''
|
||||||
|
|
||||||
|
def YTUBES_Read_Transcritpion(transcitp_file_name = None):
|
||||||
|
try:
|
||||||
|
text = ""
|
||||||
|
with open(transcitp_file_name, encoding="utf-8") as f:
|
||||||
|
text = text + f.read()
|
||||||
|
|
||||||
|
return True, text
|
||||||
|
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||||
|
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
|
||||||
|
return False, ""
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
'''
|
||||||
|
def ela_index_all_ytubes_video_transcription():
|
||||||
try:
|
try:
|
||||||
|
|
||||||
print(" on est dans ytubes read")
|
print(" on est dans ytubes read")
|
||||||
Y_collection = MYSY_GV.YTUBES_dbname["mysyclass"]
|
Y_collection = MYSY_GV.YTUBES_dbname["mysyclass"]
|
||||||
|
Y_collection = MYSY_GV.dbname["myclass"]
|
||||||
|
|
||||||
|
|
||||||
|
Ytubes_text = ""
|
||||||
nb_class_indexed = 0
|
nb_class_indexed = 0
|
||||||
for doc in Y_collection.find({'indexed_video_text': '0'}):
|
for doc in Y_collection.find({'indexed_video_text': '0', 'owner':'youtubes'}):
|
||||||
if ("video_text" in doc.keys()):
|
|
||||||
chaine = doc["video_text"]
|
|
||||||
#mycommon.myprint(" video_texte = " + str(doc["external_code"]) + " ==> " + str(chaine))
|
|
||||||
|
|
||||||
|
if ("transciption_file_name" in doc.keys()):
|
||||||
retval = YTUBES_ela_index_class(str(doc["external_code"]), "video_text")
|
retval = YTUBES_ela_index_class_transcription(str(doc["external_code"]), "video_text")
|
||||||
# print(" apres : ela_index_all_classes_video_texte retval = " + str(retval))
|
# print(" apres : ela_index_all_classes_video_texte retval = " + str(retval))
|
||||||
|
|
||||||
if (retval is False):
|
if (retval is False):
|
||||||
|
@ -84,12 +115,10 @@ def YTUBES_RedText():
|
||||||
|
|
||||||
# retval = ls.ela_index_record_field(chaine, str(doc["external_code"]))
|
# retval = ls.ela_index_record_field(chaine, str(doc["external_code"]))
|
||||||
|
|
||||||
|
|
||||||
mycommon.myprint(str(nb_class_indexed) + " ont été indexes YTUBES- champ video_text")
|
mycommon.myprint(str(nb_class_indexed) + " ont été indexes YTUBES- champ video_text")
|
||||||
return True, str(nb_class_indexed) + " ont été indexes YTUBES - champ video_text "
|
return True, str(nb_class_indexed) + " ont été indexes YTUBES - champ video_text "
|
||||||
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||||
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
|
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
|
||||||
|
|
Loading…
Reference in New Issue