2022-02-14 16:02:47
def process_sql_using_pandas():
    # create engine for credentials
    engine = create_engine('mysql://{0}:{1}@{2}:{3}/{4}?charset=utf8'.format(USER_NAME, PASSWORD, HOST, PORT, DB_NAME))
 
 
 
    #create connection
    conn = engine.connect()
    #load synonym data
    synonym_data=[]
    synonym_data=get_synonym(conn)
 
    print("creating conn again")
    conn = engine.connect().execution_options(stream_results=True)
    print("conn created")
 
    # query to get data
    select_query="select * from places_for_search  pfs left join places_search_bangla psb on pfs.uCode=psb.uCode ;"
 
    # index data to es
 
    # chunk - how many data want to pull and index at a time
    chunk=10000
    isNewIndex=False
    for chunk_dataframe in pd.read_sql(select_query, conn, chunksize=chunk):
        #print(f"Got dataframe w/{len(chunk_dataframe)} rows")
        # ... do something with dataframe ...
        #print(f"column types:\n{chunk_dataframe.dtypes}")
        chunk_dataframe=chunk_dataframe.T.groupby(level=0).first().T
        #print(chunk_dataframe.columns)
        #data=chunk_dataframe.T.to_dict().values()
 
        #index_data.index_data(chunk_dataframe)
        #data=chunk_dataframe.to_dict(orient='records')
        #print(data)
        data=[]
        for index,doc in chunk_dataframe.iterrows():
Invalid Email or Password