def process_sql_using_pandas(): # create engine for credentials engine = create_engine('mysql://{0}:{1}@{2}:{3}/{4}?charset=utf8'.format(USER_NAME, PASSWORD, HOST, PORT, DB_NAME)) #create connection conn = engine.connect() #load synonym data synonym_data=[] synonym_data=get_synonym(conn) print("creating conn again") conn = engine.connect().execution_options(stream_results=True) print("conn created") # query to get data select_query="select * from places_for_search pfs left join places_search_bangla psb on pfs.uCode=psb.uCode ;" # index data to es # chunk - how many data want to pull and index at a time chunk=10000 isNewIndex=False for chunk_dataframe in pd.read_sql(select_query, conn, chunksize=chunk): #print(f"Got dataframe w/{len(chunk_dataframe)} rows") # ... do something with dataframe ... #print(f"column types:\n{chunk_dataframe.dtypes}") chunk_dataframe=chunk_dataframe.T.groupby(level=0).first().T #print(chunk_dataframe.columns) #data=chunk_dataframe.T.to_dict().values() #index_data.index_data(chunk_dataframe) #data=chunk_dataframe.to_dict(orient='records') #print(data) data=[] for index,doc in chunk_dataframe.iterrows():