29/04/22 - 17h30
parent
934b21de5a
commit
ae76aa23a6
21
Ela_Spacy.py
21
Ela_Spacy.py
|
@ -478,10 +478,10 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
|
|||
## Suppression de toutes les indexation qui existe de cette formation
|
||||
'''
|
||||
|
||||
coll_name = MYSY_GV.YTUBES_dbname['mysyindex']
|
||||
coll_name_StopWord = MYSY_GV.YTUBES_dbname['mysystopwords']
|
||||
coll_name = MYSY_GV.dbname['elaindex']
|
||||
coll_name_StopWord = MYSY_GV.dbname['mysystopwords']
|
||||
|
||||
|
||||
print("lolololo")
|
||||
# check default value of parameter : source_field
|
||||
if len(str(source_field)) == 0 :
|
||||
source_field = 'default'
|
||||
|
@ -497,9 +497,6 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
|
|||
if( status is False):
|
||||
return False
|
||||
|
||||
#print(" AFFICHAGE TAB TOKEN")
|
||||
#print(tab_tokens)
|
||||
#print(" FINNN TAB TOKEN")
|
||||
status, tab_tokens2 = Ela_remove_stop_words(tab_tokens)
|
||||
if (status is False):
|
||||
return False
|
||||
|
@ -520,7 +517,7 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
|
|||
|
||||
|
||||
#print(str(tab_tokens3_unicode))
|
||||
status, tab_tokens4 = Ela_stemmize_Class(tab_tokens3)
|
||||
status, tab_tokens4 = Ela_stemmize_Class(tab_tokens3_unicode)
|
||||
if (status is False):
|
||||
return False
|
||||
|
||||
|
@ -528,7 +525,7 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""):
|
|||
#tab_tokens4 = Ela_remove_ponct(tab_tokens4)
|
||||
|
||||
# Enregistrement dans la base mongodb
|
||||
YTUBES_Ela_list_to_mongo(tab_tokens3_unicode,class_id, source_field)
|
||||
YTUBES_Ela_list_to_mongo(tab_tokens4,class_id, source_field)
|
||||
|
||||
size_tab = len(tab_tokens4)
|
||||
#print("size_tab = " + str(size_tab))
|
||||
|
@ -629,7 +626,7 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field):
|
|||
for tmp in unique_list:
|
||||
moyenne = round(int(occurrences[str(tmp)]) / size_tab, 3)
|
||||
if( moyenne >= seuil ):
|
||||
print(str(moyenne)+ " CMP "+str(seuil))
|
||||
#print(str(moyenne)+ " CMP "+str(seuil))
|
||||
final_lists.append([str(tmp), int(str(occurrences[str(tmp)])), str(moyenne), str(traning_id), str(source_field)])
|
||||
|
||||
#print(final_lists)
|
||||
|
@ -639,12 +636,16 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field):
|
|||
|
||||
print(" YTUBES INSERT MAY")
|
||||
|
||||
'''
|
||||
YTUBES_CONNECTION_STRING = "mongodb://localhost:27017/"
|
||||
YTUBES_client = MongoClient(YTUBES_CONNECTION_STRING)
|
||||
YTUBES_dbname = YTUBES_client['mysyvideodb']
|
||||
|
||||
|
||||
collection = YTUBES_dbname["mysyindex"]
|
||||
'''
|
||||
|
||||
collection = MYSY_GV.dbname["elaindex"]
|
||||
|
||||
data.reset_index(inplace=True)
|
||||
data_dict = data.to_dict("records")
|
||||
|
@ -652,7 +653,7 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field):
|
|||
#print(data_dict)
|
||||
# Insert collection
|
||||
collection.insert_many(data_dict)
|
||||
data.to_csv("data_indexees.csv")
|
||||
#data.to_csv("data_indexees.csv")
|
||||
|
||||
return True
|
||||
except Exception as e :
|
||||
|
|
|
@ -71,3 +71,8 @@ lsc.update_stopWords(stopWords)
|
|||
lsc.update_token_fr_pontuation(token_fr_pontuation)
|
||||
spell_fr = Speller(lang='fr')
|
||||
|
||||
|
||||
'''
|
||||
Repertoire de stockage de video YouTubes
|
||||
'''
|
||||
YTUBES_ENSCRIPT_LOCATION = ""
|
||||
|
|
|
@ -1,21 +1,29 @@
|
|||
,index,mots,occurence,moyenne,id_formation,source_field
|
||||
0,0,dynamiques,7,0.012,DmZxdIvG3YQ,video_text
|
||||
1,1,valeurs,5,0.009,DmZxdIvG3YQ,video_text
|
||||
2,2,emporter,5,0.009,DmZxdIvG3YQ,video_text
|
||||
3,3,colonne,6,0.011,DmZxdIvG3YQ,video_text
|
||||
4,4,graphique,6,0.011,DmZxdIvG3YQ,video_text
|
||||
5,5,nombre,5,0.009,DmZxdIvG3YQ,video_text
|
||||
6,6,vendu,8,0.014,DmZxdIvG3YQ,video_text
|
||||
7,7,prix,10,0.018,DmZxdIvG3YQ,video_text
|
||||
8,8,croises,6,0.011,DmZxdIvG3YQ,video_text
|
||||
9,9,somme,5,0.009,DmZxdIvG3YQ,video_text
|
||||
10,10,cellules,5,0.009,DmZxdIvG3YQ,video_text
|
||||
11,11,fonction,5,0.009,DmZxdIvG3YQ,video_text
|
||||
12,12,groupe,7,0.012,DmZxdIvG3YQ,video_text
|
||||
13,13,tableaux,7,0.012,DmZxdIvG3YQ,video_text
|
||||
14,14,modifier,6,0.011,DmZxdIvG3YQ,video_text
|
||||
15,15,tableau,7,0.012,DmZxdIvG3YQ,video_text
|
||||
16,16,donnees,8,0.014,DmZxdIvG3YQ,video_text
|
||||
17,17,vis,6,0.011,DmZxdIvG3YQ,video_text
|
||||
18,18,produits,12,0.021,DmZxdIvG3YQ,video_text
|
||||
19,19,besoin,6,0.011,DmZxdIvG3YQ,video_text
|
||||
0,0,driv,1,0.03,PUjv9Ua5db4,description
|
||||
1,1,cump,1,0.03,PUjv9Ua5db4,description
|
||||
2,2,resultat,1,0.03,PUjv9Ua5db4,description
|
||||
3,3,cout,4,0.12,PUjv9Ua5db4,description
|
||||
4,4,inventair,1,0.03,PUjv9Ua5db4,description
|
||||
5,5,fil,1,0.03,PUjv9Ua5db4,description
|
||||
6,6,repartit,1,0.03,PUjv9Ua5db4,description
|
||||
7,7,revient,1,0.03,PUjv9Ua5db4,description
|
||||
8,8,stock,1,0.03,PUjv9Ua5db4,description
|
||||
9,9,distribu,1,0.03,PUjv9Ua5db4,description
|
||||
10,10,googl,1,0.03,PUjv9Ua5db4,description
|
||||
11,11,revis,1,0.03,PUjv9Ua5db4,description
|
||||
12,12,http,1,0.03,PUjv9Ua5db4,description
|
||||
13,13,approvision,1,0.03,PUjv9Ua5db4,description
|
||||
14,14,comptabl,1,0.03,PUjv9Ua5db4,description
|
||||
15,15,indirect,1,0.03,PUjv9Ua5db4,description
|
||||
16,16,analyt,2,0.06,PUjv9Ua5db4,description
|
||||
17,17,tableau,1,0.03,PUjv9Ua5db4,description
|
||||
18,18,method,1,0.03,PUjv9Ua5db4,description
|
||||
19,19,charg,1,0.03,PUjv9Ua5db4,description
|
||||
20,20,product,1,0.03,PUjv9Ua5db4,description
|
||||
21,21,usp,1,0.03,PUjv9Ua5db4,description
|
||||
22,22,support,1,0.03,PUjv9Ua5db4,description
|
||||
23,23,sujet,1,0.03,PUjv9Ua5db4,description
|
||||
24,24,etap,1,0.03,PUjv9Ua5db4,description
|
||||
25,25,view,1,0.03,PUjv9Ua5db4,description
|
||||
26,26,sharing,1,0.03,PUjv9Ua5db4,description
|
||||
27,27,com,1,0.03,PUjv9Ua5db4,description
|
||||
|
|
|
|
@ -1,21 +1,29 @@
|
|||
mots occurence moyenne id_formation source_field
|
||||
0 dynamiques 7 0.012 DmZxdIvG3YQ video_text
|
||||
1 valeurs 5 0.009 DmZxdIvG3YQ video_text
|
||||
2 emporter 5 0.009 DmZxdIvG3YQ video_text
|
||||
3 colonne 6 0.011 DmZxdIvG3YQ video_text
|
||||
4 graphique 6 0.011 DmZxdIvG3YQ video_text
|
||||
5 nombre 5 0.009 DmZxdIvG3YQ video_text
|
||||
6 vendu 8 0.014 DmZxdIvG3YQ video_text
|
||||
7 prix 10 0.018 DmZxdIvG3YQ video_text
|
||||
8 croises 6 0.011 DmZxdIvG3YQ video_text
|
||||
9 somme 5 0.009 DmZxdIvG3YQ video_text
|
||||
10 cellules 5 0.009 DmZxdIvG3YQ video_text
|
||||
11 fonction 5 0.009 DmZxdIvG3YQ video_text
|
||||
12 groupe 7 0.012 DmZxdIvG3YQ video_text
|
||||
13 tableaux 7 0.012 DmZxdIvG3YQ video_text
|
||||
14 modifier 6 0.011 DmZxdIvG3YQ video_text
|
||||
15 tableau 7 0.012 DmZxdIvG3YQ video_text
|
||||
16 donnees 8 0.014 DmZxdIvG3YQ video_text
|
||||
17 vis 6 0.011 DmZxdIvG3YQ video_text
|
||||
18 produits 12 0.021 DmZxdIvG3YQ video_text
|
||||
19 besoin 6 0.011 DmZxdIvG3YQ video_text
|
||||
mots occurence moyenne id_formation source_field
|
||||
0 driv 1 0.03 PUjv9Ua5db4 description
|
||||
1 cump 1 0.03 PUjv9Ua5db4 description
|
||||
2 resultat 1 0.03 PUjv9Ua5db4 description
|
||||
3 cout 4 0.12 PUjv9Ua5db4 description
|
||||
4 inventair 1 0.03 PUjv9Ua5db4 description
|
||||
5 fil 1 0.03 PUjv9Ua5db4 description
|
||||
6 repartit 1 0.03 PUjv9Ua5db4 description
|
||||
7 revient 1 0.03 PUjv9Ua5db4 description
|
||||
8 stock 1 0.03 PUjv9Ua5db4 description
|
||||
9 distribu 1 0.03 PUjv9Ua5db4 description
|
||||
10 googl 1 0.03 PUjv9Ua5db4 description
|
||||
11 revis 1 0.03 PUjv9Ua5db4 description
|
||||
12 http 1 0.03 PUjv9Ua5db4 description
|
||||
13 approvision 1 0.03 PUjv9Ua5db4 description
|
||||
14 comptabl 1 0.03 PUjv9Ua5db4 description
|
||||
15 indirect 1 0.03 PUjv9Ua5db4 description
|
||||
16 analyt 2 0.06 PUjv9Ua5db4 description
|
||||
17 tableau 1 0.03 PUjv9Ua5db4 description
|
||||
18 method 1 0.03 PUjv9Ua5db4 description
|
||||
19 charg 1 0.03 PUjv9Ua5db4 description
|
||||
20 product 1 0.03 PUjv9Ua5db4 description
|
||||
21 usp 1 0.03 PUjv9Ua5db4 description
|
||||
22 support 1 0.03 PUjv9Ua5db4 description
|
||||
23 sujet 1 0.03 PUjv9Ua5db4 description
|
||||
24 etap 1 0.03 PUjv9Ua5db4 description
|
||||
25 view 1 0.03 PUjv9Ua5db4 description
|
||||
26 sharing 1 0.03 PUjv9Ua5db4 description
|
||||
27 com 1 0.03 PUjv9Ua5db4 description
|
7
main.py
7
main.py
|
@ -33,7 +33,6 @@ app.config['UPLOAD_FOLDER'] = MYSY_GV.upload_folder
|
|||
|
||||
@app.before_request
|
||||
def before_request():
|
||||
|
||||
if mycommon.check_source_ipv4(str(request.remote_addr)) is False:
|
||||
return "Security check : '" + request.remote_addr + "' is not autorized", 404
|
||||
|
||||
|
@ -619,13 +618,13 @@ def ela_index_all_classes_video_text():
|
|||
|
||||
''' Analyse Youtubes
|
||||
'''
|
||||
@app.route('/myclass/api/YTUBES_RedText/', methods=['POST','GET'])
|
||||
@app.route('/myclass/api/ela_index_all_ytubes_video_transcription/', methods=['POST','GET'])
|
||||
@crossdomain(origin='*')
|
||||
def YTUBES_RedText():
|
||||
def ela_index_all_ytubes_video_transcription():
|
||||
# On recupere le corps (payload) de la requete
|
||||
payload = request.form.to_dict()
|
||||
print(" ### payload = ",payload)
|
||||
status, retval = YTA.YTUBES_RedText()
|
||||
status, retval = YTA.ela_index_all_ytubes_video_transcription()
|
||||
return jsonify(status=status, message=retval)
|
||||
|
||||
|
||||
|
|
|
@ -14,24 +14,34 @@ import Ela_Spacy as ls
|
|||
|
||||
|
||||
|
||||
def YTUBES_ela_index_class(external_code="", source_field = ""):
|
||||
def YTUBES_ela_index_class_transcription(external_code="", source_field = ""):
|
||||
try:
|
||||
|
||||
collection = MYSY_GV.YTUBES_dbname["mysyclass"]
|
||||
collection = MYSY_GV.dbname["myclass"]
|
||||
|
||||
print(" lalalalal1")
|
||||
if( len(str(source_field)) == 0):
|
||||
source_field = 'default'
|
||||
|
||||
print(" external_code = "+str(external_code)+ " source_field = "+str(source_field))
|
||||
for doc in collection.find({"external_code":external_code}):
|
||||
print(" 3333")
|
||||
for doc in collection.find({"external_code":external_code, 'owner':'youtubes'}):
|
||||
|
||||
# Indexation du champ : video_text , donc la transcription d'une video
|
||||
if (str(source_field) == "video_text"):
|
||||
print(" lalalalal")
|
||||
chaine = doc["video_text"]
|
||||
|
||||
# Recuperation du nom du fichier
|
||||
if ("transciption_file_name" in doc.keys()):
|
||||
file_name = doc["transciption_file_name"]
|
||||
print(' file_name = ' + str(file_name))
|
||||
status, Ytubes_text = YTUBES_Read_Transcritpion(file_name)
|
||||
if (status is False):
|
||||
return False
|
||||
|
||||
print(" text indexer "+ Ytubes_text[:MYSY_GV.MAX_CARACT] + " ...")
|
||||
|
||||
# mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine))
|
||||
retval = ls.YTUBES_ela_index_record_field(chaine, str(doc["external_code"]), source_field)
|
||||
retval = ls.YTUBES_ela_index_record_field(Ytubes_text, str(doc["external_code"]), source_field)
|
||||
|
||||
'''
|
||||
# Apres indexation, on met a jour la formation pour
|
||||
|
@ -40,7 +50,7 @@ def YTUBES_ela_index_class(external_code="", source_field = ""):
|
|||
if (retval):
|
||||
# mise à jour du champ "indexed"
|
||||
myobjectif = {}
|
||||
myobjectif['indexed_video_text'] = '0'
|
||||
myobjectif['indexed_video_text'] = '1'
|
||||
|
||||
ret_val2 = collection.find_and_modify(query={"external_code": external_code},
|
||||
update={"$set": myobjectif}
|
||||
|
@ -59,22 +69,43 @@ def YTUBES_ela_index_class(external_code="", source_field = ""):
|
|||
return False
|
||||
|
||||
|
||||
def YTUBES_RedText():
|
||||
|
||||
'''
|
||||
Cette fonction va lire la transcription youtubes qui se trouve
|
||||
dans un fichier '''
|
||||
|
||||
def YTUBES_Read_Transcritpion(transcitp_file_name = None):
|
||||
try:
|
||||
text = ""
|
||||
with open(transcitp_file_name, encoding="utf-8") as f:
|
||||
text = text + f.read()
|
||||
|
||||
return True, text
|
||||
|
||||
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
|
||||
return False, ""
|
||||
|
||||
|
||||
|
||||
'''
|
||||
'''
|
||||
def ela_index_all_ytubes_video_transcription():
|
||||
try:
|
||||
|
||||
print(" on est dans ytubes read")
|
||||
Y_collection = MYSY_GV.YTUBES_dbname["mysyclass"]
|
||||
Y_collection = MYSY_GV.dbname["myclass"]
|
||||
|
||||
|
||||
|
||||
Ytubes_text = ""
|
||||
nb_class_indexed = 0
|
||||
for doc in Y_collection.find({'indexed_video_text': '0'}):
|
||||
if ("video_text" in doc.keys()):
|
||||
chaine = doc["video_text"]
|
||||
#mycommon.myprint(" video_texte = " + str(doc["external_code"]) + " ==> " + str(chaine))
|
||||
for doc in Y_collection.find({'indexed_video_text': '0', 'owner':'youtubes'}):
|
||||
|
||||
|
||||
retval = YTUBES_ela_index_class(str(doc["external_code"]), "video_text")
|
||||
if ("transciption_file_name" in doc.keys()):
|
||||
retval = YTUBES_ela_index_class_transcription(str(doc["external_code"]), "video_text")
|
||||
# print(" apres : ela_index_all_classes_video_texte retval = " + str(retval))
|
||||
|
||||
if (retval is False):
|
||||
|
@ -84,12 +115,10 @@ def YTUBES_RedText():
|
|||
|
||||
# retval = ls.ela_index_record_field(chaine, str(doc["external_code"]))
|
||||
|
||||
|
||||
mycommon.myprint(str(nb_class_indexed) + " ont été indexes YTUBES- champ video_text")
|
||||
return True, str(nb_class_indexed) + " ont été indexes YTUBES - champ video_text "
|
||||
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))
|
||||
|
|
Loading…
Reference in New Issue