import pandas as pd import argparse from sqlalchemy import create_engine, text # Function to read a parquet file and return a pandas dataframe def read_parquet(file_path): try: # Reading the parquet file dataframe = pd.read_parquet(file_path, engine='pyarrow') return dataframe except Exception as e: print(f"Error reading parquet file: {e}") return None # Function to convert dataframe to SQL def dataframe_to_sql(df, table_name): # Using the Pandas and SQLAlchemy libraries to create a SQL string engine = create_engine('sqlite://', echo=False) df.to_sql(table_name, con=engine, if_exists='replace', index=False) # Extracting the SQL commands from the engine with engine.begin() as conn: create_table_statement = conn.execute(text(f"SELECT sql FROM sqlite_master WHERE name = '{table_name}'")).fetchone()[0] print(create_table_statement) rows = conn.execute(text(f"SELECT * FROM {table_name}")).fetchall() insert_statement = [] for row in rows: placeholders = ', '.join(['?'] * len(row)) insert_statement.append(f"({placeholders})".replace('?', '{}').format(*[repr(val) for val in row])) print(f"INSERT INTO {table_name} VALUES", (', ').join(insert_statement), ';') # Main execution if __name__ == "__main__": # Initialize argument parser parser = argparse.ArgumentParser(description="Read Apache Parquet file and convert its contents to SQL.") # Add argument for file path parser.add_argument('file_path', type=str, help='Path to the parquet file') parser.add_argument('table_name', type=str, help='Name of the SQL table') # Parse the arguments args = parser.parse_args() # Read the parquet file df = read_parquet(args.file_path) # Check if dataframe is not empty if df is not None: # Convert dataframe to SQL and print dataframe_to_sql(df, args.table_name) else: print("No data to process.")