Hands-on Examples on Advanced Beginner

Environment Setup

d8lzz1gpw@mozmail.com (kimbenji) — Wed, 07 Jan 2026 00:00:00 +0000

Environment Setup#

Configure the environment for using Spark in Java/Spring Boot projects.

Pure Java Project#

The simplest configuration.

build.gradle#

plugins {
 id 'java'
 id 'application'
}

group = 'com.example'
version = '1.0.0'

java {
 sourceCompatibility = '17'
}

repositories {
 mavenCentral()
}

dependencies {
 // Spark Core
 implementation 'org.apache.spark:spark-core_2.13:3.5.1'

 // Spark SQL (DataFrame API)
 implementation 'org.apache.spark:spark-sql_2.13:3.5.1'

 // Kafka integration (optional)
 implementation 'org.apache.spark:spark-sql-kafka-0-10_2.13:3.5.1'

 // MLlib (optional)
 implementation 'org.apache.spark:spark-mllib_2.13:3.5.1'

 // Logging (conflict prevention)
 implementation 'org.slf4j:slf4j-simple:2.0.9'
}

// Logging conflict prevention
configurations.all {
 exclude group: 'org.slf4j', module: 'slf4j-log4j12'
 exclude group: 'log4j', module: 'log4j'
}

application {
 mainClass = 'com.example.SparkApp'
}

// Memory settings for execution
tasks.named('run') {
 jvmArgs = ['-Xmx2g']
}

Basic Structure#

src/
├── main/
│ ├── java/
│ │ └── com/example/
│ │ └── SparkApp.java
│ └── resources/
│ └── data/
│ └── sample.csv

SparkApp.java#

package com.example;

import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;

public class SparkApp {
 public static void main(String[] args) {
 SparkSession spark = SparkSession.builder()
 .appName("My Spark App")
 .master("local[*]")
 .config("spark.driver.memory", "2g")
 .getOrCreate();

 spark.sparkContext().setLogLevel("WARN");

 // Data processing
 Dataset<Row> df = spark.read()
 .option("header", "true")
 .option("inferSchema", "true")
 .csv("src/main/resources/data/sample.csv");

 df.show();

 spark.stop();
 }
}

Spring Boot Integration#

Configuration for using Spark with Spring Boot.

Basic Examples

d8lzz1gpw@mozmail.com (kimbenji) — Wed, 07 Jan 2026 00:00:00 +0000

Basic Examples#

Example code utilizing Spark’s core features.

Data Loading#

Reading CSV Files#

import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;

public class CsvExample {
 public static void main(String[] args) {
 SparkSession spark = SparkSession.builder()
 .appName("CSV Example")
 .master("local[*]")
 .getOrCreate();

 // Basic read
 Dataset<Row> df = spark.read()
 .option("header", "true") // First row as header
 .option("inferSchema", "true") // Auto type inference
 .csv("data/employees.csv");

 // Detailed options
 Dataset<Row> dfDetailed = spark.read()
 .option("header", "true")
 .option("inferSchema", "true")
 .option("sep", ",") // Delimiter
 .option("quote", "\"") // Quote character
 .option("escape", "\\") // Escape character
 .option("nullValue", "NA") // Null representation
 .option("dateFormat", "yyyy-MM-dd")
 .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
 .csv("data/employees.csv");

 df.show();
 df.printSchema();

 spark.stop();
 }
}

Reading JSON Files#

// Single file
Dataset<Row> df = spark.read().json("data/users.json");

// Multiline JSON
Dataset<Row> dfMulti = spark.read()
 .option("multiLine", "true")
 .json("data/users_multiline.json");

// JSON Lines (one JSON object per line)
Dataset<Row> dfLines = spark.read().json("data/users.jsonl");

df.show();

Reading Parquet Files#

// Parquet (recommended format)
Dataset<Row> df = spark.read().parquet("data/users.parquet");

// Read only specific columns (column pruning)
Dataset<Row> selected = spark.read()
 .parquet("data/users.parquet")
 .select("id", "name");

Reading from Database Using JDBC#

Dataset<Row> df = spark.read()
 .format("jdbc")
 .option("url", "jdbc:mysql://localhost:3306/mydb")
 .option("dbtable", "employees")
 .option("user", "user")
 .option("password", "password")
 .option("driver", "com.mysql.cj.jdbc.Driver")
 .load();

// Using query
Dataset<Row> dfQuery = spark.read()
 .format("jdbc")
 .option("url", "jdbc:mysql://localhost:3306/mydb")
 .option("query", "SELECT * FROM employees WHERE age > 30")
 .option("user", "user")
 .option("password", "password")
 .load();

Data Transformation#

Column Operations#

import static org.apache.spark.sql.functions.*;

Dataset<Row> employees = spark.read()
 .option("header", "true")
 .option("inferSchema", "true")
 .csv("data/employees.csv");

// Add new column
Dataset<Row> withBonus = employees.withColumn(
 "bonus",
 col("salary").multiply(0.1)
);

// Add multiple columns
Dataset<Row> enhanced = employees
 .withColumn("bonus", col("salary").multiply(0.1))
 .withColumn("total_compensation", col("salary").plus(col("bonus")))
 .withColumn("hire_year", year(col("hire_date")))
 .withColumn("name_upper", upper(col("name")));

// Rename column
Dataset<Row> renamed = employees.withColumnRenamed("name", "employee_name");

// Drop columns
Dataset<Row> dropped = employees.drop("middle_name", "suffix");

// Cast column type
Dataset<Row> casted = employees.withColumn(
 "salary",
 col("salary").cast("double")
);

enhanced.show();

Filtering#

// Single condition
Dataset<Row> highEarners = employees.filter(col("salary").gt(60000));

// Multiple conditions
Dataset<Row> filtered = employees.filter(
 col("age").geq(30)
 .and(col("department").equalTo("Engineering"))
 .and(col("salary").between(50000, 80000))
);

// String conditions
Dataset<Row> smithFamily = employees.filter(col("name").startsWith("Smith"));
Dataset<Row> hasEmail = employees.filter(col("email").contains("@company.com"));
Dataset<Row> pattern = employees.filter(col("name").rlike("^[A-Z][a-z]+$"));

// NULL handling
Dataset<Row> withManager = employees.filter(col("manager_id").isNotNull());
Dataset<Row> noManager = employees.filter(col("manager_id").isNull());

// IN condition
Dataset<Row> depts = employees.filter(
 col("department").isin("Engineering", "Marketing", "Sales")
);

filtered.show();

Sorting#

// Single column
Dataset<Row> sorted = employees.orderBy("salary");
Dataset<Row> sortedDesc = employees.orderBy(col("salary").desc());

// Multiple columns
Dataset<Row> multiSort = employees.orderBy(
 col("department").asc(),
 col("salary").desc()
);

// NULL handling
Dataset<Row> nullsFirst = employees.orderBy(col("manager_id").asc_nulls_first());
Dataset<Row> nullsLast = employees.orderBy(col("salary").desc_nulls_last());

multiSort.show();

Aggregation#

Basic Aggregation#

// Overall aggregation
Dataset<Row> stats = employees.agg(
 count("*").alias("total_count"),
 countDistinct("department").alias("dept_count"),
 sum("salary").alias("total_salary"),
 avg("salary").alias("avg_salary"),
 max("salary").alias("max_salary"),
 min("salary").alias("min_salary"),
 stddev("salary").alias("stddev_salary")
);

stats.show();

Group Aggregation#

// Single group
Dataset<Row> byDept = employees
 .groupBy("department")
 .agg(
 count("*").alias("employee_count"),
 avg("salary").alias("avg_salary"),
 sum("salary").alias("total_salary")
 )
 .orderBy(col("total_salary").desc());

// Multiple groups
Dataset<Row> byDeptLevel = employees
 .groupBy("department", "level")
 .agg(
 count("*").alias("count"),
 avg("salary").alias("avg_salary")
 );

byDept.show();

Pivot#

// Pivot table
Dataset<Row> pivoted = employees
 .groupBy("department")
 .pivot("level", Arrays.asList("Junior", "Senior", "Lead"))
 .agg(avg("salary"));

pivoted.show();
// +----------+-------+-------+------+
// |department| Junior| Senior| Lead|
// +----------+-------+-------+------+
// | Sales| 45000 | 65000 | 85000|
// | Eng | 55000 | 75000 | 95000|
// +----------+-------+-------+------+

Joins#

Basic Join#

Dataset<Row> employees = spark.read().parquet("employees.parquet");
Dataset<Row> departments = spark.read().parquet("departments.parquet");

// Inner Join
Dataset<Row> joined = employees.join(
 departments,
 employees.col("department_id").equalTo(departments.col("id"))
);

// When column names are the same
Dataset<Row> simpleJoin = employees.join(departments, "department_id");

// Left Join
Dataset<Row> leftJoined = employees.join(
 departments,
 employees.col("department_id").equalTo(departments.col("id")),
 "left"
);

// Join types: inner, left, right, full, left_semi, left_anti, cross

Multi-Condition Join#

Dataset<Row> multiJoin = orders.join(
 products,
 orders.col("product_id").equalTo(products.col("id"))
 .and(orders.col("region").equalTo(products.col("region")))
);

Broadcast Join#

import static org.apache.spark.sql.functions.broadcast;

// Broadcast small table
Dataset<Row> optimized = employees.join(
 broadcast(departments),
 "department_id"
);

Using SQL#

// Register temporary views
employees.createOrReplaceTempView("employees");
departments.createOrReplaceTempView("departments");

// Execute SQL query
Dataset<Row> result = spark.sql("""
 SELECT
 e.name,
 e.salary,
 d.department_name,
 AVG(e.salary) OVER (PARTITION BY e.department_id) as dept_avg_salary
 FROM employees e
 JOIN departments d ON e.department_id = d.id
 WHERE e.age > 25
 ORDER BY e.salary DESC
 LIMIT 100
 """);

result.show();

// Using CTE
Dataset<Row> cteResult = spark.sql("""
 WITH dept_stats AS (
 SELECT
 department_id,
 AVG(salary) as avg_salary,
 COUNT(*) as emp_count
 FROM employees
 GROUP BY department_id
 )
 SELECT
 d.department_name,
 ds.avg_salary,
 ds.emp_count
 FROM dept_stats ds
 JOIN departments d ON ds.department_id = d.id
 ORDER BY ds.avg_salary DESC
 """);

cteResult.show();

Saving Data#

Saving to Files#

// Parquet (recommended)
result.write()
 .mode("overwrite")
 .parquet("output/result.parquet");

// Partitioning
result.write()
 .mode("overwrite")
 .partitionBy("year", "month")
 .parquet("output/partitioned");

// CSV
result.write()
 .mode("overwrite")
 .option("header", "true")
 .csv("output/result.csv");

// Save as single file
result.coalesce(1)
 .write()
 .mode("overwrite")
 .option("header", "true")
 .csv("output/single_file");

Save Modes#

Mode	Description
`overwrite`	Overwrite existing data
`append`	Append to existing data
`ignore`	Ignore if already exists
`error` (default)	Error if already exists

Complete Example: Sales Analysis#

public class SalesAnalysisExample {
 public static void main(String[] args) {
 SparkSession spark = SparkSession.builder()
 .appName("Sales Analysis")
 .master("local[*]")
 .getOrCreate();

 spark.sparkContext().setLogLevel("WARN");

 // Load data
 Dataset<Row> orders = spark.read()
 .option("header", "true")
 .option("inferSchema", "true")
 .csv("data/orders.csv");

 Dataset<Row> products = spark.read()
 .option("header", "true")
 .option("inferSchema", "true")
 .csv("data/products.csv");

 Dataset<Row> customers = spark.read()
 .option("header", "true")
 .option("inferSchema", "true")
 .csv("data/customers.csv");

 // 1. Join orders with products
 Dataset<Row> ordersWithProducts = orders
 .join(broadcast(products), "product_id")
 .withColumn("revenue", col("quantity").multiply(col("price")));

 // 2. Monthly revenue aggregation
 Dataset<Row> monthlyRevenue = ordersWithProducts
 .withColumn("month", date_format(col("order_date"), "yyyy-MM"))
 .groupBy("month")
 .agg(
 sum("revenue").alias("total_revenue"),
 count("*").alias("order_count"),
 countDistinct("customer_id").alias("unique_customers")
 )
 .orderBy("month");

 System.out.println("=== Monthly Revenue ===");
 monthlyRevenue.show();

 // 3. Top 5 products by category
 WindowSpec windowSpec = Window
 .partitionBy("category")
 .orderBy(col("total_revenue").desc());

 Dataset<Row> topProducts = ordersWithProducts
 .groupBy("category", "product_name")
 .agg(sum("revenue").alias("total_revenue"))
 .withColumn("rank", rank().over(windowSpec))
 .filter(col("rank").leq(5));

 System.out.println("=== Top 5 Products by Category ===");
 topProducts.show(20);

 // 4. Customer segment analysis
 Dataset<Row> customerStats = ordersWithProducts
 .join(customers, "customer_id")
 .groupBy("customer_id", "customer_name", "segment")
 .agg(
 sum("revenue").alias("total_spent"),
 count("*").alias("order_count"),
 avg("revenue").alias("avg_order_value")
 );

 Dataset<Row> segmentAnalysis = customerStats
 .groupBy("segment")
 .agg(
 count("*").alias("customer_count"),
 avg("total_spent").alias("avg_customer_value"),
 sum("total_spent").alias("segment_revenue")
 )
 .orderBy(col("segment_revenue").desc());

 System.out.println("=== Customer Segment Analysis ===");
 segmentAnalysis.show();

 // 5. Complex analysis with SQL
 ordersWithProducts.createOrReplaceTempView("orders_enriched");

 Dataset<Row> sqlAnalysis = spark.sql("""
 WITH monthly_category AS (
 SELECT
 DATE_FORMAT(order_date, 'yyyy-MM') as month,
 category,
 SUM(revenue) as revenue
 FROM orders_enriched
 GROUP BY DATE_FORMAT(order_date, 'yyyy-MM'), category
 ),
 category_growth AS (
 SELECT
 month,
 category,
 revenue,
 LAG(revenue) OVER (PARTITION BY category ORDER BY month) as prev_revenue
 FROM monthly_category
 )
 SELECT
 month,
 category,
 revenue,
 prev_revenue,
 ROUND((revenue - prev_revenue) / prev_revenue * 100, 2) as growth_rate
 FROM category_growth
 WHERE prev_revenue IS NOT NULL
 ORDER BY month, category
 """);

 System.out.println("=== Monthly Growth Rate by Category ===");
 sqlAnalysis.show(20);

 // 6. Save results
 monthlyRevenue.write()
 .mode("overwrite")
 .parquet("output/monthly_revenue");

 segmentAnalysis.write()
 .mode("overwrite")
 .parquet("output/segment_analysis");

 spark.stop();
 }
}

Real Public Dataset Examples#

Examples using public datasets commonly utilized in practice.

Monitoring Setup

d8lzz1gpw@mozmail.com (kimbenji) — Wed, 07 Jan 2026 00:00:00 +0000

Spark Monitoring Setup Guide#

Monitoring configuration guide for stable operation of Spark applications in production environments.

Monitoring Architecture#

flowchart TB
 subgraph Spark["Spark Cluster"]
 Driver[Driver]
 E1[Executor 1]
 E2[Executor 2]
 E3[Executor N]
 end

 subgraph Metrics["Metrics Collection"]
 SparkUI[Spark UI :4040]
 History[History Server :18080]
 Prometheus[Prometheus]
 Graphite[Graphite]
 end

 subgraph Viz["Visualization/Alerting"]
 Grafana[Grafana Dashboard]
 AlertManager[Alert Manager]
 end

 Driver --> SparkUI
 Driver --> History
 E1 --> Prometheus
 E2 --> Prometheus
 E3 --> Prometheus
 Prometheus --> Grafana
 Grafana --> AlertManager

Spark UI Configuration#

Enable Basic Spark UI#

SparkSession spark = SparkSession.builder()
 .appName("Monitored Application")
 // Spark UI settings
 .config("spark.ui.enabled", "true")
 .config("spark.ui.port", "4040")
 .config("spark.ui.retainedJobs", "1000")
 .config("spark.ui.retainedStages", "1000")
 .config("spark.ui.retainedTasks", "10000")
 // Shuffle/storage metrics retention
 .config("spark.sql.ui.retainedExecutions", "200")
 .getOrCreate();

History Server Setup#

View logs even after application termination.

Spring Boot Integration

d8lzz1gpw@mozmail.com (kimbenji) — Wed, 07 Jan 2026 00:00:00 +0000

Spring Boot and Spark Integration Guide#

Spark and Spring Boot integration patterns for Java/Spring developers.

Architecture Pattern#

flowchart TB
 subgraph SpringBoot["Spring Boot Application"]
 Controller[REST Controller]
 Service[Business Service]
 SparkService[Spark Service]
 Config[Spark Configuration]
 end

 subgraph Spark["Spark Context"]
 SparkSession[SparkSession Bean]
 Jobs[Batch Jobs]
 end

 subgraph Storage["Data Storage"]
 DB[(Database)]
 HDFS[(HDFS/S3)]
 Kafka[Kafka]
 end

 Controller --> Service
 Service --> SparkService
 SparkService --> SparkSession
 SparkSession --> Jobs
 Jobs --> HDFS
 Jobs --> DB
 Jobs --> Kafka
 Config --> SparkSession

Gradle Configuration#

// build.gradle.kts
plugins {
 java
 id("org.springframework.boot") version "3.2.5"
 id("io.spring.dependency-management") version "1.1.4"
}

java {
 toolchain {
 languageVersion.set(JavaLanguageVersion.of(17))
 }
}

repositories {
 mavenCentral()
}

val sparkVersion = "3.5.1"
val scalaVersion = "2.13"

dependencies {
 // Spring Boot
 implementation("org.springframework.boot:spring-boot-starter-web")
 implementation("org.springframework.boot:spring-boot-starter-actuator")

 // Spark (Log4j exclusion required)
 implementation("org.apache.spark:spark-core_$scalaVersion:$sparkVersion") {
 exclude(group = "org.slf4j", module = "slf4j-log4j12")
 exclude(group = "log4j", module = "log4j")
 exclude(group = "org.apache.logging.log4j")
 }
 implementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") {
 exclude(group = "org.slf4j", module = "slf4j-log4j12")
 }

 // Spark JDBC (optional)
 implementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion")

 // Jackson for JSON (version unification)
 implementation("com.fasterxml.jackson.module:jackson-module-scala_$scalaVersion:2.15.3")

 // Logging (Spring Boot standard)
 implementation("org.springframework.boot:spring-boot-starter-logging")

 // Test
 testImplementation("org.springframework.boot:spring-boot-starter-test")
}

// Dependency conflict resolution
configurations.all {
 exclude(group = "org.slf4j", module = "slf4j-log4j12")
 exclude(group = "log4j", module = "log4j")
 resolutionStrategy {
 force("com.fasterxml.jackson.core:jackson-databind:2.15.3")
 force("com.fasterxml.jackson.core:jackson-core:2.15.3")
 }
}

SparkSession Bean Configuration#

Basic Configuration#

package com.example.spark.config;

import org.apache.spark.SparkConf;
import org.apache.spark.sql.SparkSession;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import jakarta.annotation.PreDestroy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Configuration
public class SparkConfig {
 private static final Logger logger = LoggerFactory.getLogger(SparkConfig.class);

 @Value("${spark.app.name:spring-spark-app}")
 private String appName;

 @Value("${spark.master:local[*]}")
 private String masterUrl;

 @Value("${spark.driver.memory:2g}")
 private String driverMemory;

 @Value("${spark.executor.memory:2g}")
 private String executorMemory;

 @Value("${spark.sql.shuffle.partitions:200}")
 private int shufflePartitions;

 private SparkSession sparkSession;

 @Bean
 @Profile("!test") // Use separate configuration for tests
 public SparkSession sparkSession() {
 logger.info("Initializing SparkSession: app={}, master={}", appName, masterUrl);

 SparkConf conf = new SparkConf()
 .setAppName(appName)
 .setMaster(masterUrl)
 .set("spark.driver.memory", driverMemory)
 .set("spark.executor.memory", executorMemory)
 .set("spark.sql.shuffle.partitions", String.valueOf(shufflePartitions))
 // Adaptive Query Execution
 .set("spark.sql.adaptive.enabled", "true")
 .set("spark.sql.adaptive.coalescePartitions.enabled", "true")
 // Serialization optimization
 .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
 .set("spark.kryoserializer.buffer.max", "1024m")
 // UI settings
 .set("spark.ui.enabled", "true")
 .set("spark.ui.port", "4040");

 this.sparkSession = SparkSession.builder()
 .config(conf)
 .getOrCreate();

 // Adjust Spark log level
 sparkSession.sparkContext().setLogLevel("WARN");

 logger.info("SparkSession initialized successfully");
 return sparkSession;
 }

 @Bean
 @Profile("test")
 public SparkSession testSparkSession() {
 logger.info("Initializing Test SparkSession");

 this.sparkSession = SparkSession.builder()
 .appName("test-spark")
 .master("local[2]")
 .config("spark.sql.shuffle.partitions", "2")
 .config("spark.ui.enabled", "false")
 .config("spark.driver.bindAddress", "127.0.0.1")
 .getOrCreate();

 return sparkSession;
 }

 @PreDestroy
 public void cleanup() {
 if (sparkSession != null && !sparkSession.sparkContext().isStopped()) {
 logger.info("Stopping SparkSession");
 sparkSession.stop();
 }
 }
}

application.yml Configuration#

# application.yml
spring:
 application:
 name: spark-spring-app

# Spark settings
spark:
 app:
 name: ${spring.application.name}
 master: local[*]
 driver:
 memory: 2g
 executor:
 memory: 2g
 sql:
 shuffle:
 partitions: 100

# Profile-specific settings
---
spring:
 config:
 activate:
 on-profile: production

spark:
 master: spark://spark-master:7077
 driver:
 memory: 4g
 executor:
 memory: 8g
 sql:
 shuffle:
 partitions: 200

---
spring:
 config:
 activate:
 on-profile: test

spark:
 master: local[2]
 driver:
 memory: 512m
 sql:
 shuffle:
 partitions: 2

Service Layer Patterns#

Basic Spark Service#

package com.example.spark.service;

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.springframework.stereotype.Service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.spark.sql.functions.*;

@Service
public class DataAnalysisService {
 private static final Logger logger = LoggerFactory.getLogger(DataAnalysisService.class);

 private final SparkSession spark;

 public DataAnalysisService(SparkSession spark) {
 this.spark = spark;
 }

 /**
 * Generate sales summary report
 */
 public SalesSummary generateSalesSummary(String dataPath, String startDate, String endDate) {
 logger.info("Generating sales summary: {} ~ {}", startDate, endDate);

 Dataset<Row> sales = spark.read()
 .option("header", "true")
 .option("inferSchema", "true")
 .parquet(dataPath);

 Dataset<Row> filtered = sales
 .filter(col("sale_date").between(startDate, endDate));

 Row summary = filtered.agg(
 count("*").alias("total_orders"),
 sum("amount").alias("total_revenue"),
 avg("amount").alias("avg_order_value"),
 countDistinct("customer_id").alias("unique_customers")
 ).first();

 return new SalesSummary(
 summary.getLong(0),
 summary.getDouble(1),
 summary.getDouble(2),
 summary.getLong(3)
 );
 }

 /**
 * Revenue aggregation by category
 */
 public List<CategoryRevenue> getCategoryRevenue(String dataPath) {
 Dataset<Row> result = spark.read()
 .parquet(dataPath)
 .groupBy("category")
 .agg(
 sum("amount").alias("revenue"),
 count("*").alias("order_count")
 )
 .orderBy(col("revenue").desc());

 return result.collectAsList().stream()
 .map(row -> new CategoryRevenue(
 row.getString(0),
 row.getDouble(1),
 row.getLong(2)
 ))
 .toList();
 }

 // DTO classes
 public record SalesSummary(
 long totalOrders,
 double totalRevenue,
 double avgOrderValue,
 long uniqueCustomers
 ) {}

 public record CategoryRevenue(
 String category,
 double revenue,
 long orderCount
 ) {}
}

Asynchronous Batch Processing#

package com.example.spark.service;

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.CompletableFuture;

import static org.apache.spark.sql.functions.*;

@Service
public class BatchProcessingService {
 private static final Logger logger = LoggerFactory.getLogger(BatchProcessingService.class);

 private final SparkSession spark;

 public BatchProcessingService(SparkSession spark) {
 this.spark = spark;
 }

 /**
 * Daily ETL job (scheduled)
 */
 @Scheduled(cron = "0 0 2 * * *") // Daily at 2 AM
 public void dailyETL() {
 String yesterday = LocalDate.now().minusDays(1)
 .format(DateTimeFormatter.ISO_DATE);

 logger.info("Starting daily ETL: {}", yesterday);

 try {
 // 1. Read raw data
 Dataset<Row> rawData = spark.read()
 .parquet("s3://data-lake/raw/events/date=" + yesterday);

 // 2. Data cleaning
 Dataset<Row> cleaned = rawData
 .filter(col("event_type").isNotNull())
 .dropDuplicates("event_id")
 .withColumn("processed_at", current_timestamp());

 // 3. Create aggregation table
 Dataset<Row> aggregated = cleaned
 .groupBy("user_id", "event_type")
 .agg(
 count("*").alias("event_count"),
 sum("value").alias("total_value")
 );

 // 4. Save
 aggregated.write()
 .mode(SaveMode.Overwrite)
 .partitionBy("event_type")
 .parquet("s3://data-lake/processed/daily/" + yesterday);

 logger.info("Daily ETL completed: {} records processed", cleaned.count());

 } catch (Exception e) {
 logger.error("Daily ETL failed: {}", e.getMessage(), e);
 throw new RuntimeException("ETL failed", e);
 }
 }

 /**
 * Asynchronous large dataset processing
 */
 @Async
 public CompletableFuture<ProcessingResult> processLargeDataset(String inputPath, String outputPath) {
 logger.info("Starting async processing: {}", inputPath);
 long startTime = System.currentTimeMillis();

 try {
 Dataset<Row> data = spark.read().parquet(inputPath);

 Dataset<Row> processed = data
 .repartition(200) // Parallel processing optimization
 .transform(this::applyBusinessLogic);

 processed.write()
 .mode(SaveMode.Overwrite)
 .parquet(outputPath);

 long duration = System.currentTimeMillis() - startTime;
 long recordCount = processed.count();

 logger.info("Async processing completed: {} records, {}ms", recordCount, duration);

 return CompletableFuture.completedFuture(
 new ProcessingResult(true, recordCount, duration, null)
 );

 } catch (Exception e) {
 logger.error("Async processing failed: {}", e.getMessage(), e);
 return CompletableFuture.completedFuture(
 new ProcessingResult(false, 0, 0, e.getMessage())
 );
 }
 }

 private Dataset<Row> applyBusinessLogic(Dataset<Row> df) {
 return df
 .filter(col("status").equalTo("ACTIVE"))
 .withColumn("score", col("value").multiply(0.8).plus(col("bonus")));
 }

 public record ProcessingResult(
 boolean success,
 long recordCount,
 long durationMs,
 String errorMessage
 ) {}
}

REST API Integration#

Analytics API Controller#

package com.example.spark.controller;

import com.example.spark.service.DataAnalysisService;
import com.example.spark.service.DataAnalysisService.SalesSummary;
import com.example.spark.service.DataAnalysisService.CategoryRevenue;
import com.example.spark.service.BatchProcessingService;
import com.example.spark.service.BatchProcessingService.ProcessingResult;

import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;

import java.util.List;
import java.util.concurrent.CompletableFuture;

@RestController
@RequestMapping("/api/v1/analytics")
public class AnalyticsController {

 private final DataAnalysisService analysisService;
 private final BatchProcessingService batchService;

 public AnalyticsController(
 DataAnalysisService analysisService,
 BatchProcessingService batchService) {
 this.analysisService = analysisService;
 this.batchService = batchService;
 }

 /**
 * Get sales summary
 * GET /api/v1/analytics/sales/summary?start=2024-01-01&end=2024-01-31
 */
 @GetMapping("/sales/summary")
 public ResponseEntity<SalesSummary> getSalesSummary(
 @RequestParam String start,
 @RequestParam String end) {

 SalesSummary summary = analysisService.generateSalesSummary(
 "data/sales.parquet", start, end);

 return ResponseEntity.ok(summary);
 }

 /**
 * Get revenue by category
 * GET /api/v1/analytics/category/revenue
 */
 @GetMapping("/category/revenue")
 public ResponseEntity<List<CategoryRevenue>> getCategoryRevenue() {
 List<CategoryRevenue> revenues = analysisService.getCategoryRevenue(
 "data/sales.parquet");

 return ResponseEntity.ok(revenues);
 }

 /**
 * Start async batch processing
 * POST /api/v1/analytics/batch/process
 */
 @PostMapping("/batch/process")
 public ResponseEntity<String> startBatchProcess(
 @RequestBody BatchRequest request) {

 CompletableFuture<ProcessingResult> future = batchService
 .processLargeDataset(request.inputPath(), request.outputPath());

 // Return job ID (integrate with job tracking system in real implementation)
 String jobId = "job-" + System.currentTimeMillis();

 return ResponseEntity.accepted()
 .body("{\"jobId\": \"" + jobId + "\", \"status\": \"PROCESSING\"}");
 }

 public record BatchRequest(String inputPath, String outputPath) {}
}

Writing Tests#

Spark Integration Tests#

package com.example.spark.service;

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.junit.jupiter.api.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;

import java.util.Arrays;
import java.util.List;

import static org.assertj.core.api.Assertions.assertThat;

@SpringBootTest
@ActiveProfiles("test")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class DataAnalysisServiceTest {

 @Autowired
 private SparkSession spark;

 @Autowired
 private DataAnalysisService analysisService;

 private String testDataPath;

 @BeforeAll
 void setupTestData() {
 // Test data schema
 StructType schema = new StructType(new StructField[]{
 DataTypes.createStructField("order_id", DataTypes.StringType, false),
 DataTypes.createStructField("category", DataTypes.StringType, false),
 DataTypes.createStructField("amount", DataTypes.DoubleType, false),
 DataTypes.createStructField("customer_id", DataTypes.StringType, false),
 DataTypes.createStructField("sale_date", DataTypes.StringType, false)
 });

 // Generate test data
 List<Row> testData = Arrays.asList(
 RowFactory.create("O001", "Electronics", 150.0, "C001", "2024-01-15"),
 RowFactory.create("O002", "Electronics", 200.0, "C002", "2024-01-15"),
 RowFactory.create("O003", "Clothing", 50.0, "C001", "2024-01-16"),
 RowFactory.create("O004", "Books", 25.0, "C003", "2024-01-17"),
 RowFactory.create("O005", "Electronics", 300.0, "C002", "2024-01-18")
 );

 Dataset<Row> testDf = spark.createDataFrame(testData, schema);

 testDataPath = "target/test-data/sales.parquet";
 testDf.write()
 .mode("overwrite")
 .parquet(testDataPath);
 }

 @Test
 @DisplayName("Sales summary should be calculated correctly")
 void shouldCalculateSalesSummary() {
 // when
 var summary = analysisService.generateSalesSummary(
 testDataPath, "2024-01-01", "2024-01-31");

 // then
 assertThat(summary.totalOrders()).isEqualTo(5);
 assertThat(summary.totalRevenue()).isEqualTo(725.0);
 assertThat(summary.avgOrderValue()).isEqualTo(145.0);
 assertThat(summary.uniqueCustomers()).isEqualTo(3);
 }

 @Test
 @DisplayName("Category revenues should be sorted descending")
 void shouldReturnCategoryRevenuesSorted() {
 // when
 var revenues = analysisService.getCategoryRevenue(testDataPath);

 // then
 assertThat(revenues).hasSize(3);
 assertThat(revenues.get(0).category()).isEqualTo("Electronics");
 assertThat(revenues.get(0).revenue()).isEqualTo(650.0);
 }

 @AfterAll
 void cleanup() {
 // Clean up test data
 new java.io.File(testDataPath).delete();
 }
}

Java vs Scala Comparison#

Aspect	Java	Scala
Type declaration	`Dataset<Row>`	`DataFrame` (type alias)
Lambda expressions	`df.filter(row -> row.getInt(0) > 10)`	`df.filter(row => row.getInt(0) > 10)`
Column reference	`col("name")`	`$"name"`
String interpolation	`"value: " + value`	`s"value: $value"`
Case Class	`record` (Java 17+)	`case class`
Pattern matching	switch expression (Java 21+)	`match` expression
Spring integration	Native support	Annotation compatible
IDE support	IntelliJ, VS Code	IntelliJ + Scala plugin

Same Logic Comparison#

Java:

Delta Lake Integration

d8lzz1gpw@mozmail.com (kimbenji) — Thu, 08 Jan 2026 00:00:00 +0000

Add ACID transactions, schema management, and time travel capabilities to your data lake using Delta Lake.

What is Delta Lake?#

Problems with Traditional Data Lakes#

Problem	Description
No ACID Support	Data corruption possible during concurrent writes
Schema Inconsistency	Different schemas across files
Small Files	Performance degradation
No Rollback	Difficult to recover from bad data

Delta Lake Solution#

flowchart LR
 subgraph Before["Parquet (Traditional)"]
 P1[part-001.parquet]
 P2[part-002.parquet]
 P3[part-003.parquet]
 end

 subgraph After["Delta Lake"]
 D1[part-001.parquet]
 D2[part-002.parquet]
 D3[_delta_log/]
 D3 --> D4[00000.json<br>Transaction Log]
 end

Feature	Description
ACID Transactions	Atomic writes, concurrency control
Schema Evolution	Safe column addition/modification
Time Travel	Query/restore past versions
Compaction	Auto merge small files
Z-Order	Query optimization clustering

Environment Setup#

build.sbt#

ThisBuild / scalaVersion := "2.13.12"

lazy val root = (project in file("."))
 .settings(
 name := "spark-delta-example",
 libraryDependencies ++= Seq(
 "org.apache.spark" %% "spark-core" % "3.5.1",
 "org.apache.spark" %% "spark-sql" % "3.5.1",
 "io.delta" %% "delta-spark" % "3.1.0"
 )
 )

SparkSession Configuration#

import org.apache.spark.sql.SparkSession
import io.delta.sql.DeltaSparkSessionExtension

val spark = SparkSession.builder()
 .appName("Delta Lake Example")
 .master("local[*]")
 .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
 .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
 .getOrCreate()

Basic CRUD Operations#

Create: Create Delta Table#

import io.delta.tables._
import org.apache.spark.sql.functions._

case class Order(
 orderId: String,
 customerId: String,
 product: String,
 quantity: Int,
 price: Double,
 orderDate: String
)

val orders = Seq(
 Order("O001", "C1", "Laptop", 1, 1200.0, "2024-01-15"),
 Order("O002", "C2", "Phone", 2, 800.0, "2024-01-15"),
 Order("O003", "C1", "Tablet", 1, 500.0, "2024-01-16")
).toDF()

// Save as Delta table
orders.write
 .format("delta")
 .mode("overwrite")
 .save("/data/orders")

Read: Query Data#

// DataFrame API
val df = spark.read.format("delta").load("/data/orders")
df.show()

// SQL
spark.sql("SELECT * FROM delta.`/data/orders`").show()

Update: Modify Data#

import io.delta.tables.DeltaTable

val deltaTable = DeltaTable.forPath(spark, "/data/orders")

// Conditional update
deltaTable.update(
 condition = expr("orderId = 'O002'"),
 set = Map(
 "quantity" -> lit(3),
 "price" -> lit(750.0)
 )
)

// SQL update
spark.sql("""
 UPDATE orders
 SET quantity = 3, price = 750.0
 WHERE orderId = 'O002'
""")

Delete: Remove Data#

// Conditional delete
deltaTable.delete(expr("customerId = 'C2'"))

// SQL delete
spark.sql("DELETE FROM orders WHERE customerId = 'C2'")

Merge (Upsert)#

val newOrders = Seq(
 Order("O002", "C2", "Phone", 5, 700.0, "2024-01-17"), // Update existing
 Order("O004", "C3", "Monitor", 2, 300.0, "2024-01-17") // Insert new
).toDF()

deltaTable.as("target")
 .merge(
 newOrders.as("source"),
 "target.orderId = source.orderId"
 )
 .whenMatched
 .updateAll()
 .whenNotMatched
 .insertAll()
 .execute()

Time Travel#

Query by Version#

// Query specific version
val version0 = spark.read
 .format("delta")
 .option("versionAsOf", 0)
 .load("/data/orders")

// Query as of timestamp
val asOf = spark.read
 .format("delta")
 .option("timestampAsOf", "2024-01-16 10:00:00")
 .load("/data/orders")

// SQL query
spark.sql("""
 SELECT * FROM orders VERSION AS OF 0
""").show()

spark.sql("""
 SELECT * FROM orders TIMESTAMP AS OF '2024-01-16 10:00:00'
""").show()

View Version History#

val history = deltaTable.history()
history.select("version", "timestamp", "operation", "operationParameters").show(false)

// Result:
// +-------+--------------------+---------+---------------------------+
// |version|timestamp |operation|operationParameters |
// +-------+--------------------+---------+---------------------------+
// |3 |2024-01-17 15:30:00|MERGE |{predicate -> ...} |
// |2 |2024-01-17 14:00:00|UPDATE |{predicate -> orderId = ...}|
// |1 |2024-01-16 10:00:00|WRITE |{mode -> Append} |
// |0 |2024-01-15 09:00:00|WRITE |{mode -> Overwrite} |
// +-------+--------------------+---------+---------------------------+

Restore to Previous Version#

// Restore to version
deltaTable.restoreToVersion(1)

// Restore to timestamp
deltaTable.restoreToTimestamp("2024-01-16 10:00:00")

// SQL restore
spark.sql("RESTORE orders TO VERSION AS OF 1")

Schema Evolution#

Add Columns#

// Data with new column
val ordersWithStatus = Seq(
 ("O005", "C4", "Keyboard", 1, 100.0, "2024-01-18", "CONFIRMED")
).toDF("orderId", "customerId", "product", "quantity", "price", "orderDate", "status")

// Auto merge schema
ordersWithStatus.write
 .format("delta")
 .mode("append")
 .option("mergeSchema", "true")
 .save("/data/orders")

Optimization#

Compaction (File Merging)#

// Merge small files
deltaTable.optimize().executeCompaction()

// Optimize specific partition
deltaTable.optimize()
 .where("orderDate = '2024-01-15'")
 .executeCompaction()

// SQL
spark.sql("OPTIMIZE orders")

Z-Order (Data Clustering)#

// Cluster by frequently filtered columns
deltaTable.optimize()
 .executeZOrderBy("customerId", "product")

// SQL
spark.sql("OPTIMIZE orders ZORDER BY (customerId, product)")

Vacuum (Clean Old Files)#

// Delete versions older than 7 days
deltaTable.vacuum(168) // 168 hours = 7 days

// SQL
spark.sql("VACUUM orders RETAIN 168 HOURS")

// Warning: Versions before retention period cannot time travel

Change Data Feed (CDC)#

Enable CDC#

// Enable when creating table
spark.sql("""
 CREATE TABLE orders_cdc (
 orderId STRING,
 customerId STRING,
 product STRING,
 quantity INT,
 price DOUBLE
 )
 USING DELTA
 TBLPROPERTIES (delta.enableChangeDataFeed = true)
""")

// Enable on existing table
spark.sql("""
 ALTER TABLE orders
 SET TBLPROPERTIES (delta.enableChangeDataFeed = true)
""")

Query Changes#

// Query changes by version range
val changes = spark.read
 .format("delta")
 .option("readChangeFeed", "true")
 .option("startingVersion", 2)
 .option("endingVersion", 5)
 .table("orders")

changes.show()
// +-------+----------+-------+--------+-----+------------------+---------------+
// |orderId|customerId|product|quantity|price|_change_type |_commit_version|
// +-------+----------+-------+--------+-----+------------------+---------------+
// |O002 |C2 |Phone |5 |700.0|update_postimage |3 |
// |O002 |C2 |Phone |3 |750.0|update_preimage |3 |
// |O004 |C3 |Monitor|2 |300.0|insert |3 |
// +-------+----------+-------+--------+-----+------------------+---------------+

Practical Example: ETL Pipeline#

Bronze → Silver → Gold Architecture#

object DeltaLakePipeline extends App {
 val spark = SparkSession.builder()
 .appName("Delta Lake Pipeline")
 .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
 .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
 .getOrCreate()

 import spark.implicits._

 // Bronze: Store raw data as-is
 def ingestToBronze(): Unit = {
 val rawData = spark.read
 .option("header", "true")
 .csv("/data/raw/orders/*.csv")

 rawData.write
 .format("delta")
 .mode("append")
 .option("mergeSchema", "true")
 .save("/data/bronze/orders")

 println("Bronze layer ingestion complete")
 }

 // Silver: Clean and validate
 def transformToSilver(): Unit = {
 val bronze = spark.read.format("delta").load("/data/bronze/orders")

 val silver = bronze
 // Data cleaning
 .filter($"orderId".isNotNull && $"price" > 0)
 // Type conversion
 .withColumn("price", $"price".cast("double"))
 .withColumn("quantity", $"quantity".cast("int"))
 .withColumn("orderDate", to_date($"orderDate"))
 // Deduplicate
 .dropDuplicates("orderId")
 // Derived columns
 .withColumn("totalAmount", $"price" * $"quantity")
 .withColumn("processedAt", current_timestamp())

 // Incremental processing with Merge
 val silverTable = DeltaTable.forPath(spark, "/data/silver/orders")

 silverTable.as("target")
 .merge(silver.as("source"), "target.orderId = source.orderId")
 .whenMatched.updateAll()
 .whenNotMatched.insertAll()
 .execute()

 println("Silver layer transformation complete")
 }

 // Gold: Business aggregations
 def aggregateToGold(): Unit = {
 val silver = spark.read.format("delta").load("/data/silver/orders")

 // Daily sales aggregation
 val dailySales = silver
 .groupBy($"orderDate")
 .agg(
 count("*").as("orderCount"),
 sum("totalAmount").as("totalRevenue"),
 avg("totalAmount").as("avgOrderValue"),
 countDistinct("customerId").as("uniqueCustomers")
 )
 .withColumn("updatedAt", current_timestamp())

 dailySales.write
 .format("delta")
 .mode("overwrite")
 .save("/data/gold/daily_sales")

 // Customer metrics
 val customerMetrics = silver
 .groupBy($"customerId")
 .agg(
 count("*").as("totalOrders"),
 sum("totalAmount").as("lifetimeValue"),
 min("orderDate").as("firstOrderDate"),
 max("orderDate").as("lastOrderDate")
 )
 .withColumn("updatedAt", current_timestamp())

 customerMetrics.write
 .format("delta")
 .mode("overwrite")
 .save("/data/gold/customer_metrics")

 println("Gold layer aggregation complete")
 }

 // Run pipeline
 def runPipeline(): Unit = {
 ingestToBronze()
 transformToSilver()
 aggregateToGold()

 // Optimize
 DeltaTable.forPath(spark, "/data/silver/orders").optimize().executeCompaction()
 DeltaTable.forPath(spark, "/data/silver/orders").vacuum(168)

 println("Pipeline complete")
 }

 runPipeline()
 spark.stop()
}

Considerations#

Item	Note
Vacuum	Versions before retention period cannot time travel
Concurrent Writes	Conflicts possible when writing to same partition
Schema Changes	Type changes require overwriteSchema
File Size	Recommend target file size setting (128MB~1GB)

Next Steps#

Structured Streaming - Real-time processing
Performance Tuning - Spark optimization
Kafka Integration - Stream source

ETL Pipeline

d8lzz1gpw@mozmail.com (kimbenji) — Wed, 07 Jan 2026 00:00:00 +0000

Production ETL Pipeline Examples#

Complete ETL (Extract-Transform-Load) pipeline examples ready for production environments.

Pipeline Architecture#

flowchart LR
 subgraph Extract["Extract"]
 S3[(S3/HDFS)]
 DB[(Database)]
 API[REST API]
 end

 subgraph Transform["Transform"]
 Clean[Data Cleaning]
 Enrich[Data Enrichment]
 Agg[Aggregation]
 end

 subgraph Load["Load"]
 DW[(Data Warehouse)]
 Lake[(Data Lake)]
 Cache[(Redis Cache)]
 end

 S3 --> Clean
 DB --> Clean
 API --> Clean
 Clean --> Enrich
 Enrich --> Agg
 Agg --> DW
 Agg --> Lake
 Agg --> Cache

Project Structure#

etl-pipeline/
├── src/main/java/com/example/etl/
│ ├── EtlApplication.java
│ ├── config/
│ │ └── SparkConfig.java
│ ├── job/
│ │ ├── AbstractEtlJob.java
│ │ ├── SalesEtlJob.java
│ │ └── CustomerEtlJob.java
│ ├── transformer/
│ │ ├── DataCleaner.java
│ │ └── DataEnricher.java
│ └── util/
│ ├── SchemaUtils.java
│ └── DateUtils.java
├── src/main/resources/
│ ├── application.yml
│ └── schemas/
│ └── sales-schema.json
└── build.gradle.kts

Basic ETL Structure#

Abstract ETL Job Class#

package com.example.etl.job;

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.LocalDate;
import java.time.format.DateTimeFormatter;

/**
 * Abstract class defining the basic structure of ETL jobs
 */
public abstract class AbstractEtlJob {
 protected final Logger logger = LoggerFactory.getLogger(getClass());
 protected final SparkSession spark;
 protected final LocalDate processDate;

 protected AbstractEtlJob(SparkSession spark, LocalDate processDate) {
 this.spark = spark;
 this.processDate = processDate;
 }

 /**
 * Execute ETL job (Template Method pattern)
 */
 public final EtlResult execute() {
 String jobName = getJobName();
 long startTime = System.currentTimeMillis();
 logger.info("ETL job started: {} (processing date: {})", jobName, processDate);

 try {
 // 1. Extract
 logger.info("[Extract] Starting data extraction");
 Dataset<Row> rawData = extract();
 long extractCount = rawData.count();
 logger.info("[Extract] Complete: {} records", extractCount);

 // 2. Transform
 logger.info("[Transform] Starting data transformation");
 Dataset<Row> transformedData = transform(rawData);
 long transformCount = transformedData.count();
 logger.info("[Transform] Complete: {} records", transformCount);

 // 3. Validate
 logger.info("[Validate] Starting data validation");
 ValidationResult validation = validate(transformedData);
 if (!validation.isValid()) {
 throw new EtlValidationException(validation.getErrors());
 }
 logger.info("[Validate] Complete: validation passed");

 // 4. Load
 logger.info("[Load] Starting data loading");
 load(transformedData);
 logger.info("[Load] Complete");

 long duration = System.currentTimeMillis() - startTime;
 logger.info("ETL job completed: {} (duration: {}ms)", jobName, duration);

 return EtlResult.success(extractCount, transformCount, duration);

 } catch (Exception e) {
 long duration = System.currentTimeMillis() - startTime;
 logger.error("ETL job failed: {} - {}", jobName, e.getMessage(), e);
 return EtlResult.failure(e.getMessage(), duration);
 }
 }

 // Implemented by subclasses
 protected abstract String getJobName();
 protected abstract Dataset<Row> extract();
 protected abstract Dataset<Row> transform(Dataset<Row> data);
 protected abstract ValidationResult validate(Dataset<Row> data);
 protected abstract void load(Dataset<Row> data);

 // Common utility methods
 protected String getPartitionPath() {
 return processDate.format(DateTimeFormatter.ofPattern("year=yyyy/month=MM/day=dd"));
 }
}

ETL Result Class#

package com.example.etl.job;

public record EtlResult(
 boolean success,
 long inputRecords,
 long outputRecords,
 long durationMs,
 String errorMessage
) {
 public static EtlResult success(long input, long output, long duration) {
 return new EtlResult(true, input, output, duration, null);
 }

 public static EtlResult failure(String error, long duration) {
 return new EtlResult(false, 0, 0, duration, error);
 }
}

Sales Data ETL Example#

Full Implementation#

package com.example.etl.job;

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.*;

import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;

import static org.apache.spark.sql.functions.*;

/**
 * Sales Data ETL Pipeline
 * - Source: S3 JSON files
 * - Transform: Cleaning, derived columns, aggregation
 * - Load: Parquet partitioned table
 */
public class SalesEtlJob extends AbstractEtlJob {

 private final String inputPath;
 private final String outputPath;
 private final StructType inputSchema;

 public SalesEtlJob(SparkSession spark, LocalDate processDate,
 String inputPath, String outputPath) {
 super(spark, processDate);
 this.inputPath = inputPath;
 this.outputPath = outputPath;
 this.inputSchema = createInputSchema();
 }

 @Override
 protected String getJobName() {
 return "SalesETL";
 }

 private StructType createInputSchema() {
 return new StructType()
 .add("order_id", DataTypes.StringType, false)
 .add("customer_id", DataTypes.StringType, false)
 .add("product_id", DataTypes.StringType, false)
 .add("quantity", DataTypes.IntegerType, false)
 .add("unit_price", DataTypes.DoubleType, false)
 .add("discount", DataTypes.DoubleType, true)
 .add("order_timestamp", DataTypes.TimestampType, false)
 .add("shipping_address", DataTypes.StringType, true)
 .add("payment_method", DataTypes.StringType, true);
 }

 @Override
 protected Dataset<Row> extract() {
 String datePath = processDate.format(
 java.time.format.DateTimeFormatter.ofPattern("yyyy/MM/dd")
 );

 return spark.read()
 .schema(inputSchema)
 .option("mode", "PERMISSIVE") // Allow corrupt records
 .option("columnNameOfCorruptRecord", "_corrupt_record")
 .json(inputPath + "/" + datePath + "/*.json");
 }

 @Override
 protected Dataset<Row> transform(Dataset<Row> data) {
 return data
 // 1. Remove corrupt records
 .filter(col("_corrupt_record").isNull())
 .drop("_corrupt_record")

 // 2. Handle NULL values
 .withColumn("discount",
 when(col("discount").isNull(), lit(0.0))
 .otherwise(col("discount")))
 .withColumn("shipping_address",
 when(col("shipping_address").isNull(), lit("Unknown"))
 .otherwise(col("shipping_address")))

 // 3. Create derived columns
 .withColumn("gross_amount",
 col("quantity").multiply(col("unit_price")))
 .withColumn("discount_amount",
 col("gross_amount").multiply(col("discount")))
 .withColumn("net_amount",
 col("gross_amount").minus(col("discount_amount")))

 // 4. Time derived columns
 .withColumn("order_date",
 to_date(col("order_timestamp")))
 .withColumn("order_hour",
 hour(col("order_timestamp")))
 .withColumn("order_dayofweek",
 dayofweek(col("order_timestamp")))

 // 5. Extract region (from address)
 .withColumn("region",
 when(col("shipping_address").contains("New York"), "Northeast")
 .when(col("shipping_address").contains("Los Angeles"), "West")
 .when(col("shipping_address").contains("Chicago"), "Midwest")
 .when(col("shipping_address").contains("Houston"), "South")
 .otherwise("Other"))

 // 6. Add processing metadata
 .withColumn("etl_timestamp", current_timestamp())
 .withColumn("etl_batch_id",
 lit(processDate.format(
 java.time.format.DateTimeFormatter.BASIC_ISO_DATE
 )));
 }

 @Override
 protected ValidationResult validate(Dataset<Row> data) {
 List<String> errors = new ArrayList<>();

 // 1. Check required columns exist
 String[] requiredColumns = {"order_id", "customer_id", "net_amount"};
 for (String col : requiredColumns) {
 if (!java.util.Arrays.asList(data.columns()).contains(col)) {
 errors.add("Missing required column: " + col);
 }
 }

 // 2. Data quality checks
 long totalCount = data.count();
 if (totalCount == 0) {
 errors.add("No data to process");
 }

 // Check for negative amounts
 long negativeAmounts = data.filter(col("net_amount").lt(0)).count();
 if (negativeAmounts > 0) {
 double ratio = (double) negativeAmounts / totalCount * 100;
 if (ratio > 1.0) { // Error if exceeds 1%
 errors.add(String.format("Negative amount ratio exceeded: %.2f%%", ratio));
 } else {
 logger.warn("Negative amount records found: {} ({}%)", negativeAmounts, ratio);
 }
 }

 // Check for duplicate orders
 long uniqueOrders = data.select("order_id").distinct().count();
 if (uniqueOrders < totalCount) {
 long duplicates = totalCount - uniqueOrders;
 errors.add("Duplicate orders found: " + duplicates);
 }

 // 3. Schema validation
 StructType actualSchema = data.schema();
 if (!actualSchema.fieldNames().length >= 10) {
 errors.add("Schema mismatch from expected");
 }

 return new ValidationResult(errors.isEmpty(), errors);
 }

 @Override
 protected void load(Dataset<Row> data) {
 // Save partitioned by date
 data.write()
 .mode(SaveMode.Overwrite)
 .partitionBy("order_date")
 .option("compression", "snappy")
 .parquet(outputPath + "/sales_fact");

 // Also create daily summary table
 Dataset<Row> dailySummary = data
 .groupBy("order_date", "region")
 .agg(
 count("*").alias("order_count"),
 sum("net_amount").alias("total_revenue"),
 avg("net_amount").alias("avg_order_value"),
 countDistinct("customer_id").alias("unique_customers")
 );

 dailySummary.write()
 .mode(SaveMode.Overwrite)
 .parquet(outputPath + "/daily_summary/" + getPartitionPath());
 }
}

Validation Result Class#

package com.example.etl.job;

import java.util.List;

public record ValidationResult(
 boolean valid,
 List<String> errors
) {
 public boolean isValid() {
 return valid;
 }

 public List<String> getErrors() {
 return errors;
 }
}

Data Cleaning Utilities#

Reusable Data Cleaner#

package com.example.etl.transformer;

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.types.DataTypes;

import java.util.Arrays;
import java.util.List;

import static org.apache.spark.sql.functions.*;

/**
 * Reusable data cleaning transformers
 */
public class DataCleaner {

 /**
 * Remove duplicates (based on specified keys)
 */
 public Dataset<Row> removeDuplicates(Dataset<Row> df, String... keyColumns) {
 return df.dropDuplicates(keyColumns);
 }

 /**
 * Handle NULL values (replace with defaults)
 */
 public Dataset<Row> fillNulls(Dataset<Row> df,
 java.util.Map<String, Object> defaults) {
 Dataset<Row> result = df;
 for (var entry : defaults.entrySet()) {
 String colName = entry.getKey();
 Object defaultValue = entry.getValue();

 if (Arrays.asList(df.columns()).contains(colName)) {
 result = result.withColumn(colName,
 when(col(colName).isNull(), lit(defaultValue))
 .otherwise(col(colName)));
 }
 }
 return result;
 }

 /**
 * Normalize strings (trim whitespace, lowercase)
 */
 public Dataset<Row> normalizeStrings(Dataset<Row> df, String... columns) {
 Dataset<Row> result = df;
 for (String colName : columns) {
 result = result.withColumn(colName,
 lower(trim(col(colName))));
 }
 return result;
 }

 /**
 * Filter outliers (IQR method)
 */
 public Dataset<Row> removeOutliers(Dataset<Row> df, String column,
 double lowerPercentile,
 double upperPercentile) {
 // Calculate percentiles
 double[] percentiles = df.stat().approxQuantile(
 column,
 new double[]{lowerPercentile, upperPercentile},
 0.01
 );

 double lower = percentiles[0];
 double upper = percentiles[1];

 return df.filter(
 col(column).geq(lower).and(col(column).leq(upper))
 );
 }

 /**
 * Standardize date format
 */
 public Dataset<Row> standardizeDates(Dataset<Row> df,
 String column,
 String inputFormat) {
 return df.withColumn(column,
 to_date(col(column), inputFormat));
 }

 /**
 * Validate email format
 */
 public Dataset<Row> validateEmails(Dataset<Row> df, String column) {
 String emailPattern = "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$";
 return df.withColumn(column + "_valid",
 col(column).rlike(emailPattern));
 }

 /**
 * Normalize phone numbers (US format)
 */
 public Dataset<Row> normalizePhoneNumbers(Dataset<Row> df, String column) {
 return df.withColumn(column,
 regexp_replace(col(column), "[^0-9]", ""))
 .withColumn(column,
 when(col(column).startsWith("1"),
 substring(col(column), 2, 100))
 .otherwise(col(column)));
 }
}

Incremental ETL#

CDC-Based Incremental Processing#

package com.example.etl.job;

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;

import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;

import static org.apache.spark.sql.functions.*;

/**
 * Incremental ETL Job (Delta approach)
 */
public class IncrementalEtlJob {
 private final SparkSession spark;
 private final String watermarkPath;

 public IncrementalEtlJob(SparkSession spark, String watermarkPath) {
 this.spark = spark;
 this.watermarkPath = watermarkPath;
 }

 /**
 * Get last processing timestamp
 */
 public LocalDateTime getLastWatermark() {
 try {
 Dataset<Row> watermark = spark.read().json(watermarkPath);
 String lastTs = watermark.first().getString(0);
 return LocalDateTime.parse(lastTs, DateTimeFormatter.ISO_DATE_TIME);
 } catch (Exception e) {
 // Return default if watermark file doesn't exist
 return LocalDateTime.of(1970, 1, 1, 0, 0);
 }
 }

 /**
 * Update watermark
 */
 public void updateWatermark(LocalDateTime newWatermark) {
 spark.createDataFrame(
 java.util.List.of(new Watermark(
 newWatermark.format(DateTimeFormatter.ISO_DATE_TIME)
 )),
 Watermark.class
 ).write().mode(SaveMode.Overwrite).json(watermarkPath);
 }

 /**
 * Extract incremental data
 */
 public Dataset<Row> extractIncremental(String sourcePath,
 LocalDateTime fromTs,
 LocalDateTime toTs) {
 return spark.read()
 .parquet(sourcePath)
 .filter(col("updated_at").gt(lit(fromTs.toString()))
 .and(col("updated_at").leq(lit(toTs.toString()))));
 }

 /**
 * Upsert (Insert or Update)
 */
 public void upsertToTarget(Dataset<Row> updates,
 String targetPath,
 String... keyColumns) {
 // Read existing data
 Dataset<Row> existing;
 try {
 existing = spark.read().parquet(targetPath);
 } catch (Exception e) {
 // Create new if target table doesn't exist
 updates.write()
 .mode(SaveMode.Overwrite)
 .parquet(targetPath);
 return;
 }

 // Remove duplicate keys (new data takes priority)
 Dataset<Row> merged = existing
 .join(updates.select(keyColumns), scala.collection.JavaConverters
 .asScalaBuffer(java.util.Arrays.asList(keyColumns)).toSeq(), "left_anti")
 .union(updates);

 // Save result
 merged.write()
 .mode(SaveMode.Overwrite)
 .parquet(targetPath);
 }

 // DTO
 public static class Watermark implements java.io.Serializable {
 private String timestamp;

 public Watermark() {}

 public Watermark(String timestamp) {
 this.timestamp = timestamp;
 }

 public String getTimestamp() { return timestamp; }
 public void setTimestamp(String timestamp) { this.timestamp = timestamp; }
 }
}

Error Handling and Retry#

Robust ETL Runner#

package com.example.etl;

import com.example.etl.job.AbstractEtlJob;
import com.example.etl.job.EtlResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.LocalDate;
import java.util.concurrent.TimeUnit;

/**
 * ETL runner with retry logic
 */
public class EtlRunner {
 private static final Logger logger = LoggerFactory.getLogger(EtlRunner.class);

 private final int maxRetries;
 private final long retryDelayMs;

 public EtlRunner(int maxRetries, long retryDelayMs) {
 this.maxRetries = maxRetries;
 this.retryDelayMs = retryDelayMs;
 }

 public EtlResult runWithRetry(AbstractEtlJob job) {
 int attempt = 0;
 EtlResult lastResult = null;

 while (attempt < maxRetries) {
 attempt++;
 logger.info("ETL execution attempt {}/{}", attempt, maxRetries);

 lastResult = job.execute();

 if (lastResult.success()) {
 logger.info("ETL success: input={}, output={}, time={}ms",
 lastResult.inputRecords(),
 lastResult.outputRecords(),
 lastResult.durationMs());
 return lastResult;
 }

 if (attempt < maxRetries) {
 logger.warn("ETL failed, retrying in {}ms: {}",
 retryDelayMs, lastResult.errorMessage());
 try {
 TimeUnit.MILLISECONDS.sleep(retryDelayMs);
 } catch (InterruptedException e) {
 Thread.currentThread().interrupt();
 break;
 }
 }
 }

 logger.error("ETL final failure ({} attempts): {}",
 attempt, lastResult != null ? lastResult.errorMessage() : "Unknown");
 return lastResult;
 }

 /**
 * Run ETL for date range
 */
 public void runForDateRange(java.util.function.Function<LocalDate, AbstractEtlJob> jobFactory,
 LocalDate startDate,
 LocalDate endDate) {
 LocalDate current = startDate;

 while (!current.isAfter(endDate)) {
 logger.info("=== Processing date: {} ===", current);

 AbstractEtlJob job = jobFactory.apply(current);
 EtlResult result = runWithRetry(job);

 if (!result.success()) {
 logger.error("Failed processing date {}, aborting", current);
 throw new RuntimeException("ETL failed: " + current);
 }

 current = current.plusDays(1);
 }

 logger.info("Complete date range processing: {} ~ {}", startDate, endDate);
 }
}

Scheduling (Spring)#

package com.example.etl;

import com.example.etl.job.SalesEtlJob;
import org.apache.spark.sql.SparkSession;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.time.LocalDate;

@Component
public class EtlScheduler {

 private final SparkSession spark;
 private final EtlRunner runner;
 private final String inputPath;
 private final String outputPath;

 public EtlScheduler(SparkSession spark) {
 this.spark = spark;
 this.runner = new EtlRunner(3, 60000); // 3 retries, 1 minute wait
 this.inputPath = System.getenv("ETL_INPUT_PATH");
 this.outputPath = System.getenv("ETL_OUTPUT_PATH");
 }

 @Scheduled(cron = "0 30 1 * * *") // Daily at 1:30 AM
 public void runDailyEtl() {
 LocalDate yesterday = LocalDate.now().minusDays(1);
 SalesEtlJob job = new SalesEtlJob(spark, yesterday, inputPath, outputPath);
 runner.runWithRetry(job);
 }
}

Basic Examples - DataFrame basic operations
Monitoring - Pipeline monitoring
Performance Tuning - Large-scale processing optimization