From ae76aa23a6a29db16bbe70095756186282b03885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ch=C3=A9rifBALDE?= Date: Fri, 29 Apr 2022 17:29:31 +0200 Subject: [PATCH] 29/04/22 - 17h30 --- Ela_Spacy.py | 21 +++++----- GlobalVariable.py | 5 +++ data_indexees.csv | 48 +++++++++++++---------- ela_output_test_file_pandas_2.txt | 50 ++++++++++++++---------- main.py | 7 ++-- youtubes_analyse.py | 65 ++++++++++++++++++++++--------- 6 files changed, 123 insertions(+), 73 deletions(-) diff --git a/Ela_Spacy.py b/Ela_Spacy.py index e02de48..375f779 100644 --- a/Ela_Spacy.py +++ b/Ela_Spacy.py @@ -478,10 +478,10 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""): ## Suppression de toutes les indexation qui existe de cette formation ''' - coll_name = MYSY_GV.YTUBES_dbname['mysyindex'] - coll_name_StopWord = MYSY_GV.YTUBES_dbname['mysystopwords'] + coll_name = MYSY_GV.dbname['elaindex'] + coll_name_StopWord = MYSY_GV.dbname['mysystopwords'] + - print("lolololo") # check default value of parameter : source_field if len(str(source_field)) == 0 : source_field = 'default' @@ -497,9 +497,6 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""): if( status is False): return False - #print(" AFFICHAGE TAB TOKEN") - #print(tab_tokens) - #print(" FINNN TAB TOKEN") status, tab_tokens2 = Ela_remove_stop_words(tab_tokens) if (status is False): return False @@ -520,7 +517,7 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""): #print(str(tab_tokens3_unicode)) - status, tab_tokens4 = Ela_stemmize_Class(tab_tokens3) + status, tab_tokens4 = Ela_stemmize_Class(tab_tokens3_unicode) if (status is False): return False @@ -528,7 +525,7 @@ def YTUBES_ela_index_record_field(lines, class_id, source_field = ""): #tab_tokens4 = Ela_remove_ponct(tab_tokens4) # Enregistrement dans la base mongodb - YTUBES_Ela_list_to_mongo(tab_tokens3_unicode,class_id, source_field) + YTUBES_Ela_list_to_mongo(tab_tokens4,class_id, source_field) size_tab = len(tab_tokens4) #print("size_tab = " + str(size_tab)) @@ -629,7 +626,7 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field): for tmp in unique_list: moyenne = round(int(occurrences[str(tmp)]) / size_tab, 3) if( moyenne >= seuil ): - print(str(moyenne)+ " CMP "+str(seuil)) + #print(str(moyenne)+ " CMP "+str(seuil)) final_lists.append([str(tmp), int(str(occurrences[str(tmp)])), str(moyenne), str(traning_id), str(source_field)]) #print(final_lists) @@ -639,12 +636,16 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field): print(" YTUBES INSERT MAY") + ''' YTUBES_CONNECTION_STRING = "mongodb://localhost:27017/" YTUBES_client = MongoClient(YTUBES_CONNECTION_STRING) YTUBES_dbname = YTUBES_client['mysyvideodb'] collection = YTUBES_dbname["mysyindex"] + ''' + + collection = MYSY_GV.dbname["elaindex"] data.reset_index(inplace=True) data_dict = data.to_dict("records") @@ -652,7 +653,7 @@ def YTUBES_Ela_list_to_mongo(tab_tokens, traning_id, source_field): #print(data_dict) # Insert collection collection.insert_many(data_dict) - data.to_csv("data_indexees.csv") + #data.to_csv("data_indexees.csv") return True except Exception as e : diff --git a/GlobalVariable.py b/GlobalVariable.py index 51d8b22..9e72e00 100644 --- a/GlobalVariable.py +++ b/GlobalVariable.py @@ -71,3 +71,8 @@ lsc.update_stopWords(stopWords) lsc.update_token_fr_pontuation(token_fr_pontuation) spell_fr = Speller(lang='fr') + +''' +Repertoire de stockage de video YouTubes +''' +YTUBES_ENSCRIPT_LOCATION = "" diff --git a/data_indexees.csv b/data_indexees.csv index 26110b9..924571b 100644 --- a/data_indexees.csv +++ b/data_indexees.csv @@ -1,21 +1,29 @@ ,index,mots,occurence,moyenne,id_formation,source_field -0,0,dynamiques,7,0.012,DmZxdIvG3YQ,video_text -1,1,valeurs,5,0.009,DmZxdIvG3YQ,video_text -2,2,emporter,5,0.009,DmZxdIvG3YQ,video_text -3,3,colonne,6,0.011,DmZxdIvG3YQ,video_text -4,4,graphique,6,0.011,DmZxdIvG3YQ,video_text -5,5,nombre,5,0.009,DmZxdIvG3YQ,video_text -6,6,vendu,8,0.014,DmZxdIvG3YQ,video_text -7,7,prix,10,0.018,DmZxdIvG3YQ,video_text -8,8,croises,6,0.011,DmZxdIvG3YQ,video_text -9,9,somme,5,0.009,DmZxdIvG3YQ,video_text -10,10,cellules,5,0.009,DmZxdIvG3YQ,video_text -11,11,fonction,5,0.009,DmZxdIvG3YQ,video_text -12,12,groupe,7,0.012,DmZxdIvG3YQ,video_text -13,13,tableaux,7,0.012,DmZxdIvG3YQ,video_text -14,14,modifier,6,0.011,DmZxdIvG3YQ,video_text -15,15,tableau,7,0.012,DmZxdIvG3YQ,video_text -16,16,donnees,8,0.014,DmZxdIvG3YQ,video_text -17,17,vis,6,0.011,DmZxdIvG3YQ,video_text -18,18,produits,12,0.021,DmZxdIvG3YQ,video_text -19,19,besoin,6,0.011,DmZxdIvG3YQ,video_text +0,0,driv,1,0.03,PUjv9Ua5db4,description +1,1,cump,1,0.03,PUjv9Ua5db4,description +2,2,resultat,1,0.03,PUjv9Ua5db4,description +3,3,cout,4,0.12,PUjv9Ua5db4,description +4,4,inventair,1,0.03,PUjv9Ua5db4,description +5,5,fil,1,0.03,PUjv9Ua5db4,description +6,6,repartit,1,0.03,PUjv9Ua5db4,description +7,7,revient,1,0.03,PUjv9Ua5db4,description +8,8,stock,1,0.03,PUjv9Ua5db4,description +9,9,distribu,1,0.03,PUjv9Ua5db4,description +10,10,googl,1,0.03,PUjv9Ua5db4,description +11,11,revis,1,0.03,PUjv9Ua5db4,description +12,12,http,1,0.03,PUjv9Ua5db4,description +13,13,approvision,1,0.03,PUjv9Ua5db4,description +14,14,comptabl,1,0.03,PUjv9Ua5db4,description +15,15,indirect,1,0.03,PUjv9Ua5db4,description +16,16,analyt,2,0.06,PUjv9Ua5db4,description +17,17,tableau,1,0.03,PUjv9Ua5db4,description +18,18,method,1,0.03,PUjv9Ua5db4,description +19,19,charg,1,0.03,PUjv9Ua5db4,description +20,20,product,1,0.03,PUjv9Ua5db4,description +21,21,usp,1,0.03,PUjv9Ua5db4,description +22,22,support,1,0.03,PUjv9Ua5db4,description +23,23,sujet,1,0.03,PUjv9Ua5db4,description +24,24,etap,1,0.03,PUjv9Ua5db4,description +25,25,view,1,0.03,PUjv9Ua5db4,description +26,26,sharing,1,0.03,PUjv9Ua5db4,description +27,27,com,1,0.03,PUjv9Ua5db4,description diff --git a/ela_output_test_file_pandas_2.txt b/ela_output_test_file_pandas_2.txt index cc2dfde..66de7c5 100644 --- a/ela_output_test_file_pandas_2.txt +++ b/ela_output_test_file_pandas_2.txt @@ -1,21 +1,29 @@ - mots occurence moyenne id_formation source_field -0 dynamiques 7 0.012 DmZxdIvG3YQ video_text -1 valeurs 5 0.009 DmZxdIvG3YQ video_text -2 emporter 5 0.009 DmZxdIvG3YQ video_text -3 colonne 6 0.011 DmZxdIvG3YQ video_text -4 graphique 6 0.011 DmZxdIvG3YQ video_text -5 nombre 5 0.009 DmZxdIvG3YQ video_text -6 vendu 8 0.014 DmZxdIvG3YQ video_text -7 prix 10 0.018 DmZxdIvG3YQ video_text -8 croises 6 0.011 DmZxdIvG3YQ video_text -9 somme 5 0.009 DmZxdIvG3YQ video_text -10 cellules 5 0.009 DmZxdIvG3YQ video_text -11 fonction 5 0.009 DmZxdIvG3YQ video_text -12 groupe 7 0.012 DmZxdIvG3YQ video_text -13 tableaux 7 0.012 DmZxdIvG3YQ video_text -14 modifier 6 0.011 DmZxdIvG3YQ video_text -15 tableau 7 0.012 DmZxdIvG3YQ video_text -16 donnees 8 0.014 DmZxdIvG3YQ video_text -17 vis 6 0.011 DmZxdIvG3YQ video_text -18 produits 12 0.021 DmZxdIvG3YQ video_text -19 besoin 6 0.011 DmZxdIvG3YQ video_text \ No newline at end of file + mots occurence moyenne id_formation source_field +0 driv 1 0.03 PUjv9Ua5db4 description +1 cump 1 0.03 PUjv9Ua5db4 description +2 resultat 1 0.03 PUjv9Ua5db4 description +3 cout 4 0.12 PUjv9Ua5db4 description +4 inventair 1 0.03 PUjv9Ua5db4 description +5 fil 1 0.03 PUjv9Ua5db4 description +6 repartit 1 0.03 PUjv9Ua5db4 description +7 revient 1 0.03 PUjv9Ua5db4 description +8 stock 1 0.03 PUjv9Ua5db4 description +9 distribu 1 0.03 PUjv9Ua5db4 description +10 googl 1 0.03 PUjv9Ua5db4 description +11 revis 1 0.03 PUjv9Ua5db4 description +12 http 1 0.03 PUjv9Ua5db4 description +13 approvision 1 0.03 PUjv9Ua5db4 description +14 comptabl 1 0.03 PUjv9Ua5db4 description +15 indirect 1 0.03 PUjv9Ua5db4 description +16 analyt 2 0.06 PUjv9Ua5db4 description +17 tableau 1 0.03 PUjv9Ua5db4 description +18 method 1 0.03 PUjv9Ua5db4 description +19 charg 1 0.03 PUjv9Ua5db4 description +20 product 1 0.03 PUjv9Ua5db4 description +21 usp 1 0.03 PUjv9Ua5db4 description +22 support 1 0.03 PUjv9Ua5db4 description +23 sujet 1 0.03 PUjv9Ua5db4 description +24 etap 1 0.03 PUjv9Ua5db4 description +25 view 1 0.03 PUjv9Ua5db4 description +26 sharing 1 0.03 PUjv9Ua5db4 description +27 com 1 0.03 PUjv9Ua5db4 description \ No newline at end of file diff --git a/main.py b/main.py index 240aebc..e6731fc 100644 --- a/main.py +++ b/main.py @@ -33,7 +33,6 @@ app.config['UPLOAD_FOLDER'] = MYSY_GV.upload_folder @app.before_request def before_request(): - if mycommon.check_source_ipv4(str(request.remote_addr)) is False: return "Security check : '" + request.remote_addr + "' is not autorized", 404 @@ -619,13 +618,13 @@ def ela_index_all_classes_video_text(): ''' Analyse Youtubes ''' -@app.route('/myclass/api/YTUBES_RedText/', methods=['POST','GET']) +@app.route('/myclass/api/ela_index_all_ytubes_video_transcription/', methods=['POST','GET']) @crossdomain(origin='*') -def YTUBES_RedText(): +def ela_index_all_ytubes_video_transcription(): # On recupere le corps (payload) de la requete payload = request.form.to_dict() print(" ### payload = ",payload) - status, retval = YTA.YTUBES_RedText() + status, retval = YTA.ela_index_all_ytubes_video_transcription() return jsonify(status=status, message=retval) diff --git a/youtubes_analyse.py b/youtubes_analyse.py index bbc9a50..50b3086 100644 --- a/youtubes_analyse.py +++ b/youtubes_analyse.py @@ -14,24 +14,34 @@ import Ela_Spacy as ls -def YTUBES_ela_index_class(external_code="", source_field = ""): +def YTUBES_ela_index_class_transcription(external_code="", source_field = ""): try: collection = MYSY_GV.YTUBES_dbname["mysyclass"] + collection = MYSY_GV.dbname["myclass"] - print(" lalalalal1") if( len(str(source_field)) == 0): source_field = 'default' print(" external_code = "+str(external_code)+ " source_field = "+str(source_field)) - for doc in collection.find({"external_code":external_code}): - print(" 3333") + for doc in collection.find({"external_code":external_code, 'owner':'youtubes'}): + # Indexation du champ : video_text , donc la transcription d'une video if (str(source_field) == "video_text"): print(" lalalalal") - chaine = doc["video_text"] + + # Recuperation du nom du fichier + if ("transciption_file_name" in doc.keys()): + file_name = doc["transciption_file_name"] + print(' file_name = ' + str(file_name)) + status, Ytubes_text = YTUBES_Read_Transcritpion(file_name) + if (status is False): + return False + + print(" text indexer "+ Ytubes_text[:MYSY_GV.MAX_CARACT] + " ...") + # mycommon.myprint(" external_code = "+str(doc["external_code"])+" ==> "+str(chaine)) - retval = ls.YTUBES_ela_index_record_field(chaine, str(doc["external_code"]), source_field) + retval = ls.YTUBES_ela_index_record_field(Ytubes_text, str(doc["external_code"]), source_field) ''' # Apres indexation, on met a jour la formation pour @@ -40,7 +50,7 @@ def YTUBES_ela_index_class(external_code="", source_field = ""): if (retval): # mise à jour du champ "indexed" myobjectif = {} - myobjectif['indexed_video_text'] = '0' + myobjectif['indexed_video_text'] = '1' ret_val2 = collection.find_and_modify(query={"external_code": external_code}, update={"$set": myobjectif} @@ -59,22 +69,43 @@ def YTUBES_ela_index_class(external_code="", source_field = ""): return False -def YTUBES_RedText(): + +''' +Cette fonction va lire la transcription youtubes qui se trouve +dans un fichier ''' + +def YTUBES_Read_Transcritpion(transcitp_file_name = None): + try: + text = "" + with open(transcitp_file_name, encoding="utf-8") as f: + text = text + f.read() + + return True, text + + + except Exception as e: + exc_type, exc_obj, exc_tb = sys.exc_info() + mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno)) + return False, "" + + + +''' +''' +def ela_index_all_ytubes_video_transcription(): try: print(" on est dans ytubes read") Y_collection = MYSY_GV.YTUBES_dbname["mysyclass"] + Y_collection = MYSY_GV.dbname["myclass"] - + Ytubes_text = "" nb_class_indexed = 0 - for doc in Y_collection.find({'indexed_video_text': '0'}): - if ("video_text" in doc.keys()): - chaine = doc["video_text"] - #mycommon.myprint(" video_texte = " + str(doc["external_code"]) + " ==> " + str(chaine)) + for doc in Y_collection.find({'indexed_video_text': '0', 'owner':'youtubes'}): - - retval = YTUBES_ela_index_class(str(doc["external_code"]), "video_text") + if ("transciption_file_name" in doc.keys()): + retval = YTUBES_ela_index_class_transcription(str(doc["external_code"]), "video_text") # print(" apres : ela_index_all_classes_video_texte retval = " + str(retval)) if (retval is False): @@ -84,12 +115,10 @@ def YTUBES_RedText(): # retval = ls.ela_index_record_field(chaine, str(doc["external_code"])) + mycommon.myprint(str(nb_class_indexed) + " ont été indexes YTUBES- champ video_text") return True, str(nb_class_indexed) + " ont été indexes YTUBES - champ video_text " - - return True - except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() mycommon.myprint(str(inspect.stack()[0][3]) + " -" + str(e) + " - Line : " + str(exc_tb.tb_lineno))