Useful Code Snippets in Python and Pyspark

dbname.table_name want to save dbname and table_name in seperate variable and then to pass them as parameters in pyspark/python script

# String containing dbname and table_name
full_table_name = "my_database.my_table"

# Split into dbname and table_name
dbname, table_name = full_table_name.split('.')

# Print the variables
print(f"Database Name: {dbname}")
print(f"Table Name: {table_name}")

# Use these variables in a PySpark job
query = f"SELECT * FROM {dbname}.{table_name} WHERE some_column = 'some_value'"

# Example usage in PySpark
df = spark.sql(query)
df.show()

import argparse
from pyspark.sql import SparkSession

# Initialize Spark
spark = SparkSession.builder.appName("Database Table Processing").getOrCreate()

# Argument parser
parser = argparse.ArgumentParser(description="Process a database table")
parser.add_argument("--dbname", required=True, help="Database name")
parser.add_argument("--table_name", required=True, help="Table name")

# Parse the arguments
args = parser.parse_args()
dbname = args.dbname
table_name = args.table_name

# Use dbname and table_name in your query
query = f"SELECT * FROM {dbname}.{table_name} WHERE some_column = 'some_value'"

# Execute the query in PySpark
df = spark.sql(query)
df.show()

import argparse
from pyspark.sql import SparkSession

# Initialize Spark
spark = SparkSession.builder.appName("Database Table Processing").getOrCreate()

# Argument parser
parser = argparse.ArgumentParser(description="Process a database table")
parser.add_argument("--dbname", required=True, help="Database name")
parser.add_argument("--table_name", required=True, help="Table name")

# Parse the arguments
args = parser.parse_args()
dbname = args.dbname
table_name = args.table_name

# Use dbname and table_name in your query
query = "SELECT * FROM {}.{} WHERE some_column = 'some_value'".format(dbname, table_name)

# Execute the query in PySpark
df = spark.sql(query)
df.show()


spark-submit myscript.py --dbname my_database --table_name my_table

To create a list of columns from a Pandas DataFrame or PySpark DataFrame, formatted with different delimiters or enclosed in quotes.

Pandas DataFrame

Example DataFrame

import pandas as pd

df = pd.DataFrame({
    "col1": [1, 2],
    "col2": [3, 4],
    "col3": [5, 6]
})

Creating a List of Columns

# Get column names
columns = df.columns.tolist()

# Separate by comma
comma_separated = ", ".join(columns)
print("Comma-separated:", comma_separated)

# Separate by space
space_separated = " ".join(columns)
print("Space-separated:", space_separated)

# Enclose in quotes and separate by comma
quoted_comma_separated = ", ".join(f"'{col}'" for col in columns)
print("Quoted, comma-separated:", quoted_comma_separated)

For PySpark DataFrame

Example DataFrame

from pyspark.sql import SparkSession

spark = SparkSession.builder.appName("ColumnListExample").getOrCreate()

data = [(1, 3, 5), (2, 4, 6)]
columns = ["col1", "col2", "col3"]
df = spark.createDataFrame(data, columns)

Creating a List of Columns

# Get column names
columns = df.columns

# Separate by comma
comma_separated = ", ".join(columns)
print("Comma-separated:", comma_separated)

# Separate by space
space_separated = " ".join(columns)
print("Space-separated:", space_separated)

# Enclose in quotes and separate by comma
quoted_comma_separated = ", ".join(f"'{col}'" for col in columns)
print("Quoted, comma-separated:", quoted_comma_separated)

Outputs

For the DataFrame columns ["col1", "col2", "col3"], you would get:

1. Comma-separated: col1, col2, col3
2. Space-separated: col1 col2 col3
3. Quoted, comma-separated: 'col1', 'col2', 'col3'