diff --git a/data_indexees.csv b/data_indexees.csv index b4d31af..730c296 100644 --- a/data_indexees.csv +++ b/data_indexees.csv @@ -1,238 +1,4 @@ ,index,mots,occurence,moyenne,id_formation,source_field -0,0,importe,1,0.01," -les-fondamentaux-de-la-negociation --7",default -1,1,respectueuse,1,0.01," -les-fondamentaux-de-la-negociation --7",default -2,2,productif,1,0.01," -les-fondamentaux-de-la-negociation --7",default -3,3,industrie,1,0.01," -les-fondamentaux-de-la-negociation --7",default -4,4,personne,1,0.01," -les-fondamentaux-de-la-negociation --7",default -5,5,decider,1,0.01," -les-fondamentaux-de-la-negociation --7",default -6,6,effet,1,0.01," -les-fondamentaux-de-la-negociation --7",default -7,7,probleme,1,0.01," -les-fondamentaux-de-la-negociation --7",default -8,8,prenante,1,0.01," -les-fondamentaux-de-la-negociation --7",default -9,9,colson,1,0.01," -les-fondamentaux-de-la-negociation --7",default -10,10,negociation,7,0.07," -les-fondamentaux-de-la-negociation --7",default -11,11,partenariat,2,0.02," -les-fondamentaux-de-la-negociation --7",default -12,12,equitable,1,0.01," -les-fondamentaux-de-la-negociation --7",default -13,13,aidera,1,0.01," -les-fondamentaux-de-la-negociation --7",default -14,14,createur,1,0.01," -les-fondamentaux-de-la-negociation --7",default -15,15,comment,1,0.01," -les-fondamentaux-de-la-negociation --7",default -16,16,traver,1,0.01," -les-fondamentaux-de-la-negociation --7",default -17,17,fondamentau,1,0.01," -les-fondamentaux-de-la-negociation --7",default -18,18,pay,1,0.01," -les-fondamentaux-de-la-negociation --7",default -19,19,publique,1,0.01," -les-fondamentaux-de-la-negociation --7",default -20,20,gagnant,2,0.02," -les-fondamentaux-de-la-negociation --7",default -21,21,valeur,1,0.01," -les-fondamentaux-de-la-negociation --7",default -22,22,secteur,1,0.01," -les-fondamentaux-de-la-negociation --7",default -23,23,donne,1,0.01," -les-fondamentaux-de-la-negociation --7",default -24,24,pri,1,0.01," -les-fondamentaux-de-la-negociation --7",default -25,25,preparer,1,0.01," -les-fondamentaux-de-la-negociation --7",default -26,26,acce,1,0.01," -les-fondamentaux-de-la-negociation --7",default -27,27,cour,4,0.04," -les-fondamentaux-de-la-negociation --7",default -28,28,trouver,1,0.01," -les-fondamentaux-de-la-negociation --7",default -29,29,vente,1,0.01," -les-fondamentaux-de-la-negociation --7",default -30,30,organisation,1,0.01," -les-fondamentaux-de-la-negociation --7",default -31,31,long,1,0.01," -les-fondamentaux-de-la-negociation --7",default -32,32,eprouve,1,0.01," -les-fondamentaux-de-la-negociation --7",default -33,33,terme,1,0.01," -les-fondamentaux-de-la-negociation --7",default -34,34,menera,1,0.01," -les-fondamentaux-de-la-negociation --7",default -35,35,blocage,1,0.01," -les-fondamentaux-de-la-negociation --7",default -36,36,durable,1,0.01," -les-fondamentaux-de-la-negociation --7",default -37,37,maniere,2,0.02," -les-fondamentaux-de-la-negociation --7",default -38,38,equipe,1,0.01," -les-fondamentaux-de-la-negociation --7",default -39,39,creer,1,0.01," -les-fondamentaux-de-la-negociation --7",default -40,40,strategique,1,0.01," -les-fondamentaux-de-la-negociation --7",default -41,41,haut,1,0.01," -les-fondamentaux-de-la-negociation --7",default -42,42,partie,1,0.01," -les-fondamentaux-de-la-negociation --7",default -43,43,structurer,1,0.01," -les-fondamentaux-de-la-negociation --7",default -44,44,professeur,1,0.01," -les-fondamentaux-de-la-negociation --7",default -45,45,impact,1,0.01," -les-fondamentaux-de-la-negociation --7",default -46,46,approche,1,0.01," -les-fondamentaux-de-la-negociation --7",default -47,47,agit,1,0.01," -les-fondamentaux-de-la-negociation --7",default -48,48,conclure,2,0.02," -les-fondamentaux-de-la-negociation --7",default -49,49,innovante,1,0.01," -les-fondamentaux-de-la-negociation --7",default -50,50,guide,1,0.01," -les-fondamentaux-de-la-negociation --7",default -51,51,obtient,1,0.01," -les-fondamentaux-de-la-negociation --7",default -52,52,negocier,1,0.01," -les-fondamentaux-de-la-negociation --7",default -53,53,surmonter,1,0.01," -les-fondamentaux-de-la-negociation --7",default -54,54,eviter,1,0.01," -les-fondamentaux-de-la-negociation --7",default -55,55,service,1,0.01," -les-fondamentaux-de-la-negociation --7",default -56,56,outil,1,0.01," -les-fondamentaux-de-la-negociation --7",default -57,57,haute,1,0.01," -les-fondamentaux-de-la-negociation --7",default -58,58,aurelien,1,0.01," -les-fondamentaux-de-la-negociation --7",default -59,59,niveau,2,0.02," -les-fondamentaux-de-la-negociation --7",default -60,60,savoir,1,0.01," -les-fondamentaux-de-la-negociation --7",default -61,61,impliquant,1,0.01," -les-fondamentaux-de-la-negociation --7",default -62,62,solution,1,0.01," -les-fondamentaux-de-la-negociation --7",default -63,63,fort,1,0.01," -les-fondamentaux-de-la-negociation --7",default -64,64,technologie,1,0.01," -les-fondamentaux-de-la-negociation --7",default -65,65,rassemble,1,0.01," -les-fondamentaux-de-la-negociation --7",default -66,66,sequence,1,0.01," -les-fondamentaux-de-la-negociation --7",default -67,67,efficace,2,0.02," -les-fondamentaux-de-la-negociation --7",default -68,68,accord,1,0.01," -les-fondamentaux-de-la-negociation --7",default -69,69,piege,1,0.01," -les-fondamentaux-de-la-negociation --7",default -70,70,consiste,1,0.01," -les-fondamentaux-de-la-negociation --7",default -71,71,pratique,2,0.02," -les-fondamentaux-de-la-negociation --7",default -72,72,bonne,1,0.01," -les-fondamentaux-de-la-negociation --7",default -73,73,susciter,1,0.01," -les-fondamentaux-de-la-negociation --7",default -74,74,interactive,1,0.01," -les-fondamentaux-de-la-negociation --7",default -75,75,eventail,1,0.01," -les-fondamentaux-de-la-negociation --7",default -76,76,agisse,1,0.01," -les-fondamentaux-de-la-negociation --7",default -77,77,large,1,0.01," -les-fondamentaux-de-la-negociation --7",default -78,78,gestion,1,0.01," -les-fondamentaux-de-la-negociation --7",default +0,0,responsive,1,0.33,bFvfqUMjvsA,keyword +1,1,javascript,1,0.33,bFvfqUMjvsA,keyword +2,2,sidebar,1,0.33,bFvfqUMjvsA,keyword diff --git a/ela_index_bdd_classes.py b/ela_index_bdd_classes.py index ae5880a..2f56b49 100644 --- a/ela_index_bdd_classes.py +++ b/ela_index_bdd_classes.py @@ -551,6 +551,8 @@ def ela_recherche_tokens(sentence): {'$sort': {'count_class': -1, 'sum_occurence': -1}}, {'$limit': MYSY_GV.QUERY_LIMIT_ROW} ] + + print(" #### CRITERE DE CHERCHE pipe2 = "+str(pipe2)) # ici limite à 4 for result in collection.aggregate(pipe2): tab_training_id.append(str(result["_id"])) @@ -558,8 +560,6 @@ def ela_recherche_tokens(sentence): #print("resultat tab_training_id = "+str(tab_training_id)) - - return tab_training_id except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() diff --git a/ela_output_test_file_pandas_2.txt b/ela_output_test_file_pandas_2.txt index 8d9413d..666e082 100644 --- a/ela_output_test_file_pandas_2.txt +++ b/ela_output_test_file_pandas_2.txt @@ -1,14 +1,4 @@ - mots ... source_field -0 importe ... default -1 respectueuse ... default -2 productif ... default -3 industrie ... default -4 personne ... default -.. ... ... ... -74 interactive ... default -75 eventail ... default -76 agisse ... default -77 large ... default -78 gestion ... default - -[79 rows x 5 columns] \ No newline at end of file + mots occurence moyenne id_formation source_field +0 responsive 1 0.33 bFvfqUMjvsA keyword +1 javascript 1 0.33 bFvfqUMjvsA keyword +2 sidebar 1 0.33 bFvfqUMjvsA keyword \ No newline at end of file diff --git a/prj_common.py b/prj_common.py index dbe7509..c3c85d6 100644 --- a/prj_common.py +++ b/prj_common.py @@ -1120,3 +1120,27 @@ def clean_internal_url(): exc_type, exc_obj, exc_tb = sys.exc_info() print(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno)) return False, "clean internal url KO" + +''' +Cette fonction supprime les emoji des text +''' +def clean_emoji(sentence=None): + try: + emoji_pattern = re.compile("[" + u"\U0001F600-\U0001F64F" # emoticons + u"\U0001F300-\U0001F5FF" # symbols & pictographs + u"\U0001F680-\U0001F6FF" # transport & map symbols + u"\U0001F1E0-\U0001F1FF" # flags (iOS) + "]+", flags=re.UNICODE) + + + new_sentence = emoji_pattern.sub(r'', sentence) + + return new_sentence + + + except Exception as e: + exc_type, exc_obj, exc_tb = sys.exc_info() + print(str(inspect.stack()[0][3]) + " -" + str(e) + " - ERRORRRR AT Line : " + str(exc_tb.tb_lineno)) + return "" + diff --git a/wrapper.py b/wrapper.py index 58aea2a..d0e3d4a 100644 --- a/wrapper.py +++ b/wrapper.py @@ -498,13 +498,13 @@ def recherche_text_simple(diction): return False, "La recheche est impossible " user = x - val = x['description'] + + val = mycommon.clean_emoji(str(x['description'])) + if (len(x['description']) > MYSY_GV.MAX_CARACT): x['description'] = val[:MYSY_GV.MAX_CARACT] + " ..." else: x['description'] = val[:MYSY_GV.MAX_CARACT] - - insertObject.append(JSONEncoder().encode(user)) '''