Performance Optimization and Production Deployment: Taking Scala Applications to Production

Introduction

Building high-performance Scala applications requires understanding both language-specific optimizations and JVM tuning techniques. This lesson covers performance optimization strategies, monitoring, profiling, and production deployment best practices to ensure your Scala applications perform excellently in real-world environments.

You'll learn to identify performance bottlenecks, optimize critical code paths, configure the JVM for optimal performance, and deploy applications with confidence using modern deployment strategies.

Performance Analysis and Profiling

JVM Performance Monitoring

// Performance monitoring utilities
import java.lang.management.ManagementFactory
import java.util.concurrent.TimeUnit
import scala.concurrent.duration._

object PerformanceMonitor {
  private val runtime = Runtime.getRuntime
  private val memoryMX = ManagementFactory.getMemoryMXBean
  private val gcMX = ManagementFactory.getGarbageCollectorMXBeans

  case class MemoryStats(
    heapUsed: Long,
    heapMax: Long,
    nonHeapUsed: Long,
    nonHeapMax: Long,
    heapUtilization: Double
  )

  case class GCStats(
    collectorName: String,
    collectionCount: Long,
    collectionTime: Long
  )

  case class SystemStats(
    availableProcessors: Int,
    maxMemory: Long,
    totalMemory: Long,
    freeMemory: Long,
    usedMemory: Long
  )

  def getMemoryStats: MemoryStats = {
    val heapMemory = memoryMX.getHeapMemoryUsage
    val nonHeapMemory = memoryMX.getNonHeapMemoryUsage

    MemoryStats(
      heapUsed = heapMemory.getUsed,
      heapMax = heapMemory.getMax,
      nonHeapUsed = nonHeapMemory.getUsed,
      nonHeapMax = nonHeapMemory.getMax,
      heapUtilization = heapMemory.getUsed.toDouble / heapMemory.getMax * 100
    )
  }

  def getGCStats: List[GCStats] = {
    import scala.jdk.CollectionConverters._
    gcMX.asScala.map { gc =>
      GCStats(
        collectorName = gc.getName,
        collectionCount = gc.getCollectionCount,
        collectionTime = gc.getCollectionTime
      )
    }.toList
  }

  def getSystemStats: SystemStats = {
    SystemStats(
      availableProcessors = runtime.availableProcessors(),
      maxMemory = runtime.maxMemory(),
      totalMemory = runtime.totalMemory(),
      freeMemory = runtime.freeMemory(),
      usedMemory = runtime.totalMemory() - runtime.freeMemory()
    )
  }

  def printStats(): Unit = {
    val memory = getMemoryStats
    val gc = getGCStats
    val system = getSystemStats

    println("=== System Performance Stats ===")
    println(f"Heap: ${memory.heapUsed / 1024 / 1024}%,d MB / ${memory.heapMax / 1024 / 1024}%,d MB (${memory.heapUtilization}%.1f%%)")
    println(f"Non-Heap: ${memory.nonHeapUsed / 1024 / 1024}%,d MB / ${memory.nonHeapMax / 1024 / 1024}%,d MB")
    println(f"Used Memory: ${system.usedMemory / 1024 / 1024}%,d MB")
    println(f"Available Processors: ${system.availableProcessors}")

    println("\n=== Garbage Collection ===")
    gc.foreach { stats =>
      println(f"${stats.collectorName}: ${stats.collectionCount} collections, ${stats.collectionTime} ms total")
    }
    println("=" * 35)
  }

  // Memory pressure test
  def simulateMemoryPressure(): Unit = {
    println("Simulating memory pressure...")
    val lists = scala.collection.mutable.ListBuffer[List[String]]()

    try {
      for (i <- 1 to 1000) {
        lists += List.fill(10000)(s"String_$i")
        if (i % 100 == 0) {
          println(s"Created $i lists...")
          printStats()
        }
      }
    } catch {
      case _: OutOfMemoryError =>
        println("Out of memory!")
        printStats()
    } finally {
      lists.clear()
      System.gc()  // Suggest garbage collection
      Thread.sleep(1000)
      println("After cleanup:")
      printStats()
    }
  }
}

// Benchmark utilities
object BenchmarkUtils {

  def time[T](description: String)(block: => T): T = {
    val start = System.nanoTime()
    val result = block
    val elapsed = System.nanoTime() - start
    println(f"$description: ${elapsed / 1000000.0}%.3f ms")
    result
  }

  def timeWithWarmup[T](description: String, warmupRuns: Int = 5, measureRuns: Int = 10)(block: => T): T = {
    // Warmup runs
    for (_ <- 1 to warmupRuns) block

    // Measured runs
    val times = for (_ <- 1 to measureRuns) yield {
      val start = System.nanoTime()
      val result = block
      val elapsed = System.nanoTime() - start
      (result, elapsed)
    }

    val (results, durations) = times.unzip
    val avgTime = durations.sum / durations.length / 1000000.0
    val minTime = durations.min / 1000000.0
    val maxTime = durations.max / 1000000.0

    println(f"$description: avg=${avgTime}%.3f ms, min=${minTime}%.3f ms, max=${maxTime}%.3f ms")
    results.head
  }

  // Throughput benchmark
  def throughput[T](description: String, durationSeconds: Int = 5)(operation: => T): Double = {
    val startTime = System.currentTimeMillis()
    val endTime = startTime + (durationSeconds * 1000)
    var operations = 0L

    while (System.currentTimeMillis() < endTime) {
      operation
      operations += 1
    }

    val actualDuration = (System.currentTimeMillis() - startTime) / 1000.0
    val opsPerSecond = operations / actualDuration

    println(f"$description: ${operations} operations in ${actualDuration}%.1f seconds = ${opsPerSecond}%.0f ops/sec")
    opsPerSecond
  }

  // Memory allocation benchmark
  def measureAllocations[T](description: String)(block: => T): T = {
    val beforeGC = PerformanceMonitor.getGCStats
    System.gc()
    Thread.sleep(100)
    val beforeMemory = PerformanceMonitor.getMemoryStats

    val result = block

    val afterMemory = PerformanceMonitor.getMemoryStats
    val afterGC = PerformanceMonitor.getGCStats

    val allocatedMemory = afterMemory.heapUsed - beforeMemory.heapUsed
    val gcCollections = afterGC.map(_.collectionCount).sum - beforeGC.map(_.collectionCount).sum

    println(f"$description: ${allocatedMemory / 1024 / 1024}%,d MB allocated, $gcCollections GC collections")
    result
  }
}

// Performance testing examples
object PerformanceTests {

  // Collection performance comparison
  def testCollectionPerformance(): Unit = {
    val size = 1000000
    val data = (1 to size).toList

    println("=== Collection Performance Tests ===")

    // List vs Vector vs Array performance
    BenchmarkUtils.timeWithWarmup("List creation") {
      (1 to size).toList
    }

    BenchmarkUtils.timeWithWarmup("Vector creation") {
      (1 to size).toVector
    }

    BenchmarkUtils.timeWithWarmup("Array creation") {
      (1 to size).toArray
    }

    val list = data
    val vector = data.toVector
    val array = data.toArray

    // Access patterns
    val indices = List.fill(10000)(scala.util.Random.nextInt(size))

    BenchmarkUtils.timeWithWarmup("List random access") {
      indices.map(list(_)).sum
    }

    BenchmarkUtils.timeWithWarmup("Vector random access") {
      indices.map(vector(_)).sum
    }

    BenchmarkUtils.timeWithWarmup("Array random access") {
      indices.map(array(_)).sum
    }

    // Append operations
    BenchmarkUtils.timeWithWarmup("List prepend") {
      (1 to 10000).foldLeft(List.empty[Int])(_ +: _)
    }

    BenchmarkUtils.timeWithWarmup("Vector append") {
      (1 to 10000).foldLeft(Vector.empty[Int])(_ :+ _)
    }

    // Map operations
    BenchmarkUtils.timeWithWarmup("List map") {
      list.map(_ * 2)
    }

    BenchmarkUtils.timeWithWarmup("Vector map") {
      vector.map(_ * 2)
    }

    BenchmarkUtils.timeWithWarmup("Array map") {
      array.map(_ * 2)
    }
  }

  // String concatenation performance
  def testStringPerformance(): Unit = {
    val iterations = 10000

    println("\n=== String Performance Tests ===")

    BenchmarkUtils.timeWithWarmup("String concatenation") {
      (1 to iterations).foldLeft("")(_ + _)
    }

    BenchmarkUtils.timeWithWarmup("StringBuilder") {
      val sb = new StringBuilder()
      for (i <- 1 to iterations) sb.append(i)
      sb.toString
    }

    BenchmarkUtils.timeWithWarmup("String interpolation") {
      (1 to iterations).map(i => s"Item $i").mkString
    }

    BenchmarkUtils.timeWithWarmup("String formatting") {
      (1 to iterations).map(i => f"Item $i%05d").mkString
    }
  }

  // Functional vs imperative performance
  def testFunctionalVsImperative(): Unit = {
    val data = (1 to 1000000).toList

    println("\n=== Functional vs Imperative Performance ===")

    // Functional approach
    BenchmarkUtils.timeWithWarmup("Functional filter/map/reduce") {
      data.filter(_ % 2 == 0).map(_ * 2).sum
    }

    // Imperative approach
    BenchmarkUtils.timeWithWarmup("Imperative loop") {
      var sum = 0
      var i = 0
      while (i < data.length) {
        val value = data(i)
        if (value % 2 == 0) {
          sum += value * 2
        }
        i += 1
      }
      sum
    }

    // View for lazy evaluation
    BenchmarkUtils.timeWithWarmup("Lazy view") {
      data.view.filter(_ % 2 == 0).map(_ * 2).sum
    }

    // Iterator for memory efficiency
    BenchmarkUtils.timeWithWarmup("Iterator") {
      data.iterator.filter(_ % 2 == 0).map(_ * 2).sum
    }
  }

  // Concurrency performance
  def testConcurrencyPerformance(): Unit = {
    import scala.concurrent.{Future, ExecutionContext}
    import scala.concurrent.duration._
    import scala.util.{Success, Failure}

    implicit val ec: ExecutionContext = ExecutionContext.global

    val data = (1 to 1000000).toList
    val cpuCount = Runtime.getRuntime.availableProcessors()

    println(s"\n=== Concurrency Performance Tests (${cpuCount} CPUs) ===")

    // Sequential processing
    BenchmarkUtils.timeWithWarmup("Sequential processing") {
      data.map(x => x * x + math.sqrt(x)).sum
    }

    // Parallel processing
    BenchmarkUtils.timeWithWarmup("Parallel processing") {
      data.par.map(x => x * x + math.sqrt(x)).sum
    }

    // Future-based parallel processing
    BenchmarkUtils.timeWithWarmup("Future-based parallel") {
      val chunkSize = data.length / cpuCount
      val chunks = data.grouped(chunkSize).toList

      val futures = chunks.map { chunk =>
        Future {
          chunk.map(x => x * x + math.sqrt(x)).sum
        }
      }

      val result = Future.sequence(futures).map(_.sum)
      scala.concurrent.Await.result(result, 10.seconds)
    }
  }
}

// Run performance tests
println("Starting performance analysis...")
PerformanceMonitor.printStats()

PerformanceTests.testCollectionPerformance()
PerformanceTests.testStringPerformance()
PerformanceTests.testFunctionalVsImperative()
PerformanceTests.testConcurrencyPerformance()

println("\nFinal stats after tests:")
PerformanceMonitor.printStats()

Memory Management and Optimization

// Memory-efficient data structures and patterns
object MemoryOptimization {

  // Avoid memory leaks with proper resource management
  trait Resource {
    def close(): Unit
  }

  def withResource[R <: Resource, T](resource: R)(operation: R => T): T = {
    try {
      operation(resource)
    } finally {
      resource.close()
    }
  }

  // Memory-efficient streaming
  def processLargeFile(filename: String): Long = {
    import scala.io.Source

    val source = Source.fromFile(filename)
    try {
      source.getLines()
        .filter(_.nonEmpty)
        .map(_.length)
        .sum
    } finally {
      source.close()
    }
  }

  // Object pooling for frequent allocations
  class ObjectPool[T](factory: () => T, reset: T => Unit, maxSize: Int = 100) {
    private val pool = scala.collection.mutable.Queue[T]()
    private var created = 0

    def acquire(): T = {
      pool.synchronized {
        if (pool.nonEmpty) {
          pool.dequeue()
        } else {
          created += 1
          factory()
        }
      }
    }

    def release(obj: T): Unit = {
      pool.synchronized {
        if (pool.size < maxSize) {
          reset(obj)
          pool.enqueue(obj)
        }
      }
    }

    def stats: (Int, Int) = pool.synchronized((created, pool.size))
  }

  // Example usage of object pool
  case class DataProcessor(var data: List[String] = List.empty) {
    def process(input: String): String = {
      data = input :: data
      data.mkString(",")
    }
  }

  val processorPool = new ObjectPool[DataProcessor](
    factory = () => DataProcessor(),
    reset = processor => processor.data = List.empty,
    maxSize = 50
  )

  def processWithPool(inputs: List[String]): List[String] = {
    inputs.map { input =>
      val processor = processorPool.acquire()
      try {
        processor.process(input)
      } finally {
        processorPool.release(processor)
      }
    }
  }

  // Memory-efficient caching with weak references
  import java.lang.ref.WeakReference
  import scala.collection.mutable

  class WeakCache[K, V] {
    private val cache = mutable.Map[K, WeakReference[V]]()

    def get(key: K): Option[V] = {
      cache.get(key).flatMap(ref => Option(ref.get()))
    }

    def put(key: K, value: V): Unit = {
      cache(key) = new WeakReference(value)
    }

    def cleanup(): Unit = {
      cache.retain((_, ref) => ref.get() != null)
    }

    def size: Int = cache.size
  }

  // Flyweight pattern for immutable data
  object StringPool {
    private val pool = mutable.Map[String, String]()

    def intern(s: String): String = {
      pool.getOrElseUpdate(s, s)
    }

    def size: Int = pool.size
  }

  // Memory-efficient builders
  class EfficientStringBuilder(initialCapacity: Int = 16) {
    private var buffer = new Array[Char](initialCapacity)
    private var length = 0

    def append(s: String): this.type = {
      val newLength = length + s.length
      if (newLength > buffer.length) {
        val newCapacity = math.max(newLength, buffer.length * 2)
        val newBuffer = new Array[Char](newCapacity)
        Array.copy(buffer, 0, newBuffer, 0, length)
        buffer = newBuffer
      }

      s.getChars(0, s.length, buffer, length)
      length = newLength
      this
    }

    def result(): String = new String(buffer, 0, length)

    def clear(): this.type = {
      length = 0
      this
    }
  }

  // Compact data structures
  case class CompactUser(
    id: Int,
    name: String,  // Use String interning for common names
    email: String,
    flags: Byte    // Pack boolean flags into a single byte
  ) {
    def isActive: Boolean = (flags & 1) != 0
    def isVerified: Boolean = (flags & 2) != 0
    def isPremium: Boolean = (flags & 4) != 0

    def withActive(active: Boolean): CompactUser = {
      val newFlags = if (active) (flags | 1).toByte else (flags & ~1).toByte
      copy(flags = newFlags)
    }

    def withVerified(verified: Boolean): CompactUser = {
      val newFlags = if (verified) (flags | 2).toByte else (flags & ~2).toByte
      copy(flags = newFlags)
    }

    def withPremium(premium: Boolean): CompactUser = {
      val newFlags = if (premium) (flags | 4).toByte else (flags & ~4).toByte
      copy(flags = newFlags)
    }
  }

  object CompactUser {
    def apply(id: Int, name: String, email: String, active: Boolean = false, 
              verified: Boolean = false, premium: Boolean = false): CompactUser = {
      var flags: Byte = 0
      if (active) flags = (flags | 1).toByte
      if (verified) flags = (flags | 2).toByte
      if (premium) flags = (flags | 4).toByte

      CompactUser(id, StringPool.intern(name), email, flags)
    }
  }

  // Memory usage monitoring
  def measureMemoryUsage[T](description: String)(block: => T): T = {
    System.gc()
    Thread.sleep(100)
    val memBefore = PerformanceMonitor.getMemoryStats

    val result = block

    System.gc()
    Thread.sleep(100)
    val memAfter = PerformanceMonitor.getMemoryStats

    val allocated = memAfter.heapUsed - memBefore.heapUsed
    println(f"$description: ${allocated / 1024 / 1024}%,d MB allocated")

    result
  }

  // Demonstration of memory optimization techniques
  def demonstrateOptimizations(): Unit = {
    println("=== Memory Optimization Demonstrations ===")

    // Object pooling
    val inputs = List.fill(1000)("test input")

    measureMemoryUsage("Without object pooling") {
      inputs.map { input =>
        val processor = DataProcessor()
        processor.process(input)
      }
    }

    measureMemoryUsage("With object pooling") {
      processWithPool(inputs)
    }

    val (created, pooled) = processorPool.stats
    println(s"Object pool stats: $created created, $pooled pooled")

    // String interning
    val names = List.fill(1000)("John") ++ List.fill(1000)("Jane") ++ List.fill(1000)("Bob")

    measureMemoryUsage("Without string interning") {
      names.map(name => s"User: $name")
    }

    measureMemoryUsage("With string interning") {
      names.map(name => s"User: ${StringPool.intern(name)}")
    }

    println(s"String pool size: ${StringPool.size}")

    // Compact data structures
    measureMemoryUsage("Standard data structure") {
      (1 to 10000).map { i =>
        (i, s"User$i", s"user$i@example.com", i % 2 == 0, i % 3 == 0, i % 5 == 0)
      }
    }

    measureMemoryUsage("Compact data structure") {
      (1 to 10000).map { i =>
        CompactUser(i, s"User$i", s"user$i@example.com", i % 2 == 0, i % 3 == 0, i % 5 == 0)
      }
    }
  }
}

// Run memory optimization demonstrations
MemoryOptimization.demonstrateOptimizations()

JVM Tuning and Configuration

JVM Performance Tuning

// JVM configuration examples and monitoring
object JVMTuning {

  // JVM arguments for different scenarios
  object JVMArgs {

    // Development environment (laptop/desktop)
    val development = List(
      "-Xms512m",                    // Initial heap size
      "-Xmx2g",                      // Maximum heap size
      "-XX:+UseG1GC",                // Use G1 garbage collector
      "-XX:MaxGCPauseMillis=200",    // Target GC pause time
      "-XX:+UnlockExperimentalVMOptions",
      "-XX:+UseCGroupMemoryLimitForHeap",  // Container awareness
      "-XX:+PrintGC",                // Enable GC logging
      "-XX:+PrintGCDetails",
      "-XX:+PrintGCTimeStamps",
      "-XX:+UseStringDeduplication", // Deduplicate strings
      "-server"                      // Server mode
    )

    // Production high-throughput application
    val highThroughput = List(
      "-Xms8g",                      // Large initial heap
      "-Xmx8g",                      // Fixed heap size
      "-XX:+UseG1GC",
      "-XX:MaxGCPauseMillis=100",    // Low latency target
      "-XX:G1HeapRegionSize=16m",    // Larger regions for large heap
      "-XX:+UseStringDeduplication",
      "-XX:+OptimizeStringConcat",
      "-XX:+UseFastUnorderedTimeStamps",
      "-XX:+AggressiveOpts",         // Aggressive optimizations
      "-XX:+TieredCompilation",      // Enable tiered compilation
      "-server"
    )

    // Production low-latency application
    val lowLatency = List(
      "-Xms4g",
      "-Xmx4g",
      "-XX:+UnlockExperimentalVMOptions",
      "-XX:+UseZGC",                 // Ultra-low latency GC (Java 11+)
      "-XX:+UseLargePages",          // Large memory pages
      "-XX:+UnlockDiagnosticVMOptions",
      "-XX:+LogVMOutput",
      "-XX:+TraceClassLoading",
      "-XX:+PrintCompilation",
      "-server"
    )

    // Memory-constrained environment (containers)
    val containerOptimized = List(
      "-Xms256m",
      "-Xmx1g",
      "-XX:+UseSerialGC",            // Simple GC for small heaps
      "-XX:+UseCGroupMemoryLimitForHeap",
      "-XX:+UnlockExperimentalVMOptions",
      "-XX:+UseCGroupCPUCount",      // Container CPU awareness
      "-Djava.security.egd=file:/dev/./urandom",  // Faster random
      "-server"
    )

    // Debugging and profiling
    val debugging = List(
      "-Xms1g",
      "-Xmx2g",
      "-XX:+UseG1GC",
      "-XX:+PrintGC",
      "-XX:+PrintGCDetails",
      "-XX:+PrintGCApplicationStoppedTime",
      "-XX:+PrintStringDeduplicationStatistics",
      "-XX:+FlightRecorder",         // Enable Java Flight Recorder
      "-XX:+UnlockCommercialFeatures",
      "-XX:StartFlightRecording=duration=60s,filename=app.jfr",
      "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005",  // Remote debugging
      "-server"
    )
  }

  // GC analysis utilities
  object GCAnalysis {
    import java.lang.management.{GarbageCollectorMXBean, ManagementFactory}
    import scala.jdk.CollectionConverters._

    case class GCInfo(
      name: String,
      collectionCount: Long,
      collectionTime: Long,
      averageTime: Double
    )

    def analyzeGC(): List[GCInfo] = {
      ManagementFactory.getGarbageCollectorMXBeans.asScala.toList.map { gc =>
        val count = gc.getCollectionCount
        val time = gc.getCollectionTime
        val avgTime = if (count > 0) time.toDouble / count else 0.0

        GCInfo(gc.getName, count, time, avgTime)
      }
    }

    def printGCAnalysis(): Unit = {
      val gcInfo = analyzeGC()
      println("=== Garbage Collection Analysis ===")
      gcInfo.foreach { info =>
        println(f"${info.name}:")
        println(f"  Collections: ${info.collectionCount}")
        println(f"  Total time: ${info.collectionTime} ms")
        println(f"  Average time: ${info.averageTime}%.2f ms")
      }
    }

    // GC pressure simulation
    def simulateGCPressure(rounds: Int = 10): Unit = {
      println(s"Simulating GC pressure with $rounds rounds...")

      for (round <- 1 to rounds) {
        val beforeGC = analyzeGC()

        // Allocate and discard memory
        val data = (1 to 100000).map(i => s"String number $i with extra data").toList
        val processed = data.map(_.toUpperCase).filter(_.contains("5"))

        val afterGC = analyzeGC()

        // Calculate GC activity in this round
        val gcActivity = (beforeGC zip afterGC).map { case (before, after) =>
          (after.name, after.collectionCount - before.collectionCount, after.collectionTime - before.collectionTime)
        }.filter(_._2 > 0)

        if (gcActivity.nonEmpty) {
          println(f"Round $round: ${gcActivity.map(a => s"${a._1}:${a._2}collections/${a._3}ms").mkString(", ")}")
        }

        // Keep some references to simulate memory pressure
        if (round % 3 == 0) {
          Thread.sleep(100)  // Allow GC to catch up
        }
      }
    }
  }

  // JIT compilation monitoring
  object JITAnalysis {
    import java.lang.management.ManagementFactory

    def getCompilationInfo(): Unit = {
      val compilationMX = ManagementFactory.getCompilationMXBean
      if (compilationMX.isCompilationTimeMonitoringSupported) {
        println(s"JIT Compilation time: ${compilationMX.getTotalCompilationTime} ms")
      }
    }

    // Hot spot identification through repeated execution
    def identifyHotSpots(): Unit = {
      println("=== JIT Hot Spot Analysis ===")

      // Method that will become a hot spot
      def computeIntensive(n: Int): Long = {
        var sum = 0L
        for (i <- 1 to n) {
          sum += i * i + math.sqrt(i).toLong
        }
        sum
      }

      val iterations = 50000
      val warmupRuns = 10
      val measuredRuns = 5

      // Warmup phase to trigger JIT compilation
      println("Warming up JIT compiler...")
      getCompilationInfo()

      for (_ <- 1 to warmupRuns) {
        computeIntensive(iterations)
      }

      getCompilationInfo()

      // Measure performance after JIT compilation
      println("Measuring performance after JIT compilation...")
      val times = for (_ <- 1 to measuredRuns) yield {
        val start = System.nanoTime()
        val result = computeIntensive(iterations)
        val elapsed = System.nanoTime() - start
        (result, elapsed)
      }

      val avgTime = times.map(_._2).sum / times.length / 1000000.0
      println(f"Average execution time after JIT: $avgTime%.3f ms")

      getCompilationInfo()
    }
  }

  // Memory pool analysis
  object MemoryPoolAnalysis {
    import java.lang.management.{ManagementFactory, MemoryPoolMXBean, MemoryType}
    import scala.jdk.CollectionConverters._

    case class PoolInfo(
      name: String,
      poolType: MemoryType,
      used: Long,
      committed: Long,
      max: Long,
      utilization: Double
    )

    def analyzeMemoryPools(): List[PoolInfo] = {
      ManagementFactory.getMemoryPoolMXBeans.asScala.toList.map { pool =>
        val usage = pool.getUsage
        val utilization = if (usage.getMax > 0) {
          usage.getUsed.toDouble / usage.getMax * 100
        } else 0.0

        PoolInfo(
          name = pool.getName,
          poolType = pool.getType,
          used = usage.getUsed,
          committed = usage.getCommitted,
          max = usage.getMax,
          utilization = utilization
        )
      }
    }

    def printMemoryPoolAnalysis(): Unit = {
      val pools = analyzeMemoryPools()
      println("=== Memory Pool Analysis ===")

      val heapPools = pools.filter(_.poolType == MemoryType.HEAP)
      val nonHeapPools = pools.filter(_.poolType == MemoryType.NON_HEAP)

      println("Heap Pools:")
      heapPools.foreach { pool =>
        println(f"  ${pool.name}:")
        println(f"    Used: ${pool.used / 1024 / 1024}%,d MB")
        println(f"    Committed: ${pool.committed / 1024 / 1024}%,d MB")
        println(f"    Max: ${if (pool.max > 0) s"${pool.max / 1024 / 1024} MB" else "Unlimited"}")
        println(f"    Utilization: ${pool.utilization}%.1f%%")
      }

      println("\nNon-Heap Pools:")
      nonHeapPools.foreach { pool =>
        println(f"  ${pool.name}:")
        println(f"    Used: ${pool.used / 1024 / 1024}%,d MB")
        println(f"    Committed: ${pool.committed / 1024 / 1024}%,d MB")
        println(f"    Max: ${if (pool.max > 0) s"${pool.max / 1024 / 1024} MB" else "Unlimited"}")
        println(f"    Utilization: ${pool.utilization}%.1f%%")
      }
    }
  }

  // Class loading analysis
  object ClassLoadingAnalysis {
    import java.lang.management.ManagementFactory

    def analyzeClassLoading(): Unit = {
      val classLoadingMX = ManagementFactory.getClassLoadingMXBean

      println("=== Class Loading Analysis ===")
      println(s"Current loaded classes: ${classLoadingMX.getLoadedClassCount}")
      println(s"Total loaded classes: ${classLoadingMX.getTotalLoadedClassCount}")
      println(s"Unloaded classes: ${classLoadingMX.getUnloadedClassCount}")
      println(s"Verbose class loading: ${classLoadingMX.isVerbose}")
    }
  }

  // Comprehensive JVM analysis
  def performJVMAnalysis(): Unit = {
    println("=== Comprehensive JVM Performance Analysis ===")

    MemoryPoolAnalysis.printMemoryPoolAnalysis()
    println()

    GCAnalysis.printGCAnalysis()
    println()

    ClassLoadingAnalysis.analyzeClassLoading()
    println()

    JITAnalysis.identifyHotSpots()
    println()

    // Simulate some load and analyze GC behavior
    GCAnalysis.simulateGCPressure(5)

    println("=== Final Analysis ===")
    GCAnalysis.printGCAnalysis()
  }
}

// Run JVM analysis
JVMTuning.performJVMAnalysis()

Production Deployment Strategies

Containerization and Orchestration

// Docker configuration for Scala applications
object DockerConfiguration {

  // Dockerfile generation
  val baseDockerfile = """
    |FROM openjdk:11-jre-slim
    |
    |# Install additional packages if needed
    |RUN apt-get update && apt-get install -y \
    |    curl \
    |    && rm -rf /var/lib/apt/lists/*
    |
    |# Create non-root user
    |RUN groupadd -r appuser && useradd -r -g appuser appuser
    |
    |# Set working directory
    |WORKDIR /app
    |
    |# Copy application jar
    |COPY target/scala-*/myapp-assembly-*.jar app.jar
    |
    |# Copy configuration files
    |COPY conf/ /app/conf/
    |
    |# Set JVM options for containers
    |ENV JAVA_OPTS="-Xms256m -Xmx1g -XX:+UseContainerSupport -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap"
    |
    |# Expose port
    |EXPOSE 8080
    |
    |# Health check
    |HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
    |    CMD curl -f http://localhost:8080/health || exit 1
    |
    |# Switch to non-root user
    |USER appuser
    |
    |# Run application
    |ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar app.jar"]
    |""".stripMargin

  // Multi-stage Docker build
  val multiStageDockerfile = """
    |# Build stage
    |FROM sbtscala/scala-sbt:openjdk-11.0.2_9_1.3.8_2.13.1 as builder
    |
    |WORKDIR /app
    |
    |# Copy build files
    |COPY build.sbt .
    |COPY project/ project/
    |
    |# Download dependencies
    |RUN sbt update
    |
    |# Copy source code
    |COPY src/ src/
    |
    |# Build application
    |RUN sbt assembly
    |
    |# Runtime stage
    |FROM openjdk:11-jre-slim
    |
    |RUN groupadd -r appuser && useradd -r -g appuser appuser
    |
    |WORKDIR /app
    |
    |# Copy only the built JAR from build stage
    |COPY --from=builder /app/target/scala-*/myapp-assembly-*.jar app.jar
    |
    |# Runtime configuration
    |ENV JAVA_OPTS="-Xms256m -Xmx1g -XX:+UseContainerSupport"
    |
    |EXPOSE 8080
    |
    |HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
    |    CMD curl -f http://localhost:8080/health || exit 1
    |
    |USER appuser
    |
    |ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar app.jar"]
    |""".stripMargin

  // Kubernetes deployment configuration
  val kubernetesDeployment = """
    |apiVersion: apps/v1
    |kind: Deployment
    |metadata:
    |  name: scala-app
    |  labels:
    |    app: scala-app
    |spec:
    |  replicas: 3
    |  selector:
    |    matchLabels:
    |      app: scala-app
    |  template:
    |    metadata:
    |      labels:
    |        app: scala-app
    |    spec:
    |      containers:
    |      - name: scala-app
    |        image: myregistry/scala-app:latest
    |        ports:
    |        - containerPort: 8080
    |        env:
    |        - name: JAVA_OPTS
    |          value: "-Xms512m -Xmx2g -XX:+UseG1GC -XX:+UseContainerSupport"
    |        - name: DATABASE_URL
    |          valueFrom:
    |            secretKeyRef:
    |              name: app-secrets
    |              key: database-url
    |        resources:
    |          requests:
    |            memory: "1Gi"
    |            cpu: "500m"
    |          limits:
    |            memory: "2Gi"
    |            cpu: "1000m"
    |        livenessProbe:
    |          httpGet:
    |            path: /health
    |            port: 8080
    |          initialDelaySeconds: 60
    |          periodSeconds: 30
    |        readinessProbe:
    |          httpGet:
    |            path: /ready
    |            port: 8080
    |          initialDelaySeconds: 30
    |          periodSeconds: 10
    |---
    |apiVersion: v1
    |kind: Service
    |metadata:
    |  name: scala-app-service
    |spec:
    |  selector:
    |    app: scala-app
    |  ports:
    |  - port: 80
    |    targetPort: 8080
    |  type: LoadBalancer
    |""".stripMargin

  // Horizontal Pod Autoscaler
  val hpaConfiguration = """
    |apiVersion: autoscaling/v2
    |kind: HorizontalPodAutoscaler
    |metadata:
    |  name: scala-app-hpa
    |spec:
    |  scaleTargetRef:
    |    apiVersion: apps/v1
    |    kind: Deployment
    |    name: scala-app
    |  minReplicas: 2
    |  maxReplicas: 10
    |  metrics:
    |  - type: Resource
    |    resource:
    |      name: cpu
    |      target:
    |        type: Utilization
    |        averageUtilization: 70
    |  - type: Resource
    |    resource:
    |      name: memory
    |      target:
    |        type: Utilization
    |        averageUtilization: 80
    |  behavior:
    |    scaleUp:
    |      stabilizationWindowSeconds: 60
    |      policies:
    |      - type: Percent
    |        value: 100
    |        periodSeconds: 60
    |    scaleDown:
    |      stabilizationWindowSeconds: 300
    |      policies:
    |      - type: Percent
    |        value: 10
    |        periodSeconds: 60
    |""".stripMargin
}

// Application health checks and monitoring
object HealthMonitoring {

  // Health check implementation
  trait HealthChecker {
    def name: String
    def check(): HealthStatus
  }

  sealed trait HealthStatus
  case object Healthy extends HealthStatus
  case class Unhealthy(reason: String) extends HealthStatus

  // Database health checker
  class DatabaseHealthChecker(connectionPool: String) extends HealthChecker {
    def name: String = "database"

    def check(): HealthStatus = {
      try {
        // Simulate database connection check
        if (connectionPool.nonEmpty) {
          Healthy
        } else {
          Unhealthy("No database connection pool")
        }
      } catch {
        case e: Exception => Unhealthy(s"Database connection failed: ${e.getMessage}")
      }
    }
  }

  // External service health checker
  class ExternalServiceHealthChecker(serviceUrl: String) extends HealthChecker {
    def name: String = "external-service"

    def check(): HealthStatus = {
      try {
        // Simulate HTTP health check
        if (serviceUrl.startsWith("http")) {
          Healthy
        } else {
          Unhealthy("Invalid service URL")
        }
      } catch {
        case e: Exception => Unhealthy(s"External service check failed: ${e.getMessage}")
      }
    }
  }

  // Disk space health checker
  class DiskSpaceHealthChecker(thresholdPercent: Double = 90.0) extends HealthChecker {
    def name: String = "disk-space"

    def check(): HealthStatus = {
      val file = new java.io.File(".")
      val totalSpace = file.getTotalSpace
      val freeSpace = file.getFreeSpace
      val usedPercent = ((totalSpace - freeSpace).toDouble / totalSpace) * 100

      if (usedPercent > thresholdPercent) {
        Unhealthy(f"Disk usage too high: $usedPercent%.1f%%")
      } else {
        Healthy
      }
    }
  }

  // Memory health checker
  class MemoryHealthChecker(thresholdPercent: Double = 85.0) extends HealthChecker {
    def name: String = "memory"

    def check(): HealthStatus = {
      val memStats = PerformanceMonitor.getMemoryStats
      if (memStats.heapUtilization > thresholdPercent) {
        Unhealthy(f"Memory usage too high: ${memStats.heapUtilization}%.1f%%")
      } else {
        Healthy
      }
    }
  }

  // Comprehensive health service
  class HealthService(checkers: List[HealthChecker]) {

    case class OverallHealth(
      status: String,
      checks: Map[String, String],
      timestamp: Long
    )

    def checkHealth(): OverallHealth = {
      val results = checkers.map { checker =>
        val status = try {
          checker.check() match {
            case Healthy => "UP"
            case Unhealthy(reason) => s"DOWN: $reason"
          }
        } catch {
          case e: Exception => s"ERROR: ${e.getMessage}"
        }
        checker.name -> status
      }.toMap

      val overallStatus = if (results.values.forall(_.startsWith("UP"))) "UP" else "DOWN"

      OverallHealth(
        status = overallStatus,
        checks = results,
        timestamp = System.currentTimeMillis()
      )
    }

    def printHealthReport(): Unit = {
      val health = checkHealth()
      println(s"=== Health Check Report - ${health.status} ===")
      println(s"Timestamp: ${new java.util.Date(health.timestamp)}")

      health.checks.foreach { case (name, status) =>
        val icon = if (status.startsWith("UP")) "✓" else "✗"
        println(s"  $icon $name: $status")
      }
    }
  }

  // Application metrics
  class ApplicationMetrics {
    private var requestCount = 0L
    private var errorCount = 0L
    private val responseTimeHistory = scala.collection.mutable.ListBuffer[Long]()
    private val maxHistorySize = 1000

    def recordRequest(responseTimeMs: Long, isError: Boolean = false): Unit = {
      synchronized {
        requestCount += 1
        if (isError) errorCount += 1

        responseTimeHistory += responseTimeMs
        if (responseTimeHistory.size > maxHistorySize) {
          responseTimeHistory.remove(0)
        }
      }
    }

    def getMetrics: Map[String, Any] = synchronized {
      val avgResponseTime = if (responseTimeHistory.nonEmpty) {
        responseTimeHistory.sum.toDouble / responseTimeHistory.size
      } else 0.0

      val errorRate = if (requestCount > 0) {
        (errorCount.toDouble / requestCount) * 100
      } else 0.0

      Map(
        "total_requests" -> requestCount,
        "error_count" -> errorCount,
        "error_rate_percent" -> f"$errorRate%.2f",
        "avg_response_time_ms" -> f"$avgResponseTime%.2f",
        "memory_usage" -> PerformanceMonitor.getMemoryStats,
        "system_stats" -> PerformanceMonitor.getSystemStats
      )
    }

    def printMetrics(): Unit = {
      val metrics = getMetrics
      println("=== Application Metrics ===")
      metrics.foreach { case (key, value) =>
        println(s"$key: $value")
      }
    }
  }

  // Create health service with all checkers
  val healthCheckers = List(
    new DatabaseHealthChecker("mock-pool"),
    new ExternalServiceHealthChecker("https://api.example.com"),
    new DiskSpaceHealthChecker(90.0),
    new MemoryHealthChecker(85.0)
  )

  val healthService = new HealthService(healthCheckers)
  val applicationMetrics = new ApplicationMetrics()

  // Simulate application activity and monitoring
  def simulateApplicationLoad(): Unit = {
    println("=== Simulating Application Load ===")

    val random = new scala.util.Random()

    // Simulate 100 requests
    for (i <- 1 to 100) {
      val responseTime = 50 + random.nextInt(200)  // 50-250ms
      val isError = random.nextDouble() < 0.05      // 5% error rate

      applicationMetrics.recordRequest(responseTime, isError)

      if (i % 20 == 0) {
        println(s"Processed $i requests...")
        healthService.printHealthReport()
        applicationMetrics.printMetrics()
        println()
      }

      Thread.sleep(10)  // Small delay between requests
    }

    println("=== Final Application State ===")
    healthService.printHealthReport()
    applicationMetrics.printMetrics()
  }
}

// CI/CD Pipeline configuration
object CIPipeline {

  // GitHub Actions workflow
  val githubActionsWorkflow = """
    |name: Scala CI/CD Pipeline
    |
    |on:
    |  push:
    |    branches: [ main, develop ]
    |  pull_request:
    |    branches: [ main ]
    |
    |jobs:
    |  test:
    |    runs-on: ubuntu-latest
    |    
    |    services:
    |      postgres:
    |        image: postgres:13
    |        env:
    |          POSTGRES_PASSWORD: postgres
    |        options: >-
    |          --health-cmd pg_isready
    |          --health-interval 10s
    |          --health-timeout 5s
    |          --health-retries 5
    |
    |    steps:
    |    - uses: actions/checkout@v2
    |    
    |    - name: Set up JDK 11
    |      uses: actions/setup-java@v2
    |      with:
    |        java-version: '11'
    |        distribution: 'adopt'
    |        
    |    - name: Cache SBT
    |      uses: actions/cache@v2
    |      with:
    |        path: |
    |          ~/.sbt
    |          ~/.ivy2/cache
    |          ~/.coursier
    |        key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt') }}
    |        
    |    - name: Run tests
    |      run: sbt test
    |      
    |    - name: Generate coverage report
    |      run: sbt coverage test coverageReport
    |      
    |    - name: Upload coverage to Codecov
    |      uses: codecov/codecov-action@v1
    |
    |  build-and-deploy:
    |    needs: test
    |    runs-on: ubuntu-latest
    |    if: github.ref == 'refs/heads/main'
    |    
    |    steps:
    |    - uses: actions/checkout@v2
    |    
    |    - name: Set up JDK 11
    |      uses: actions/setup-java@v2
    |      with:
    |        java-version: '11'
    |        distribution: 'adopt'
    |        
    |    - name: Build application
    |      run: sbt assembly
    |      
    |    - name: Build Docker image
    |      run: |
    |        docker build -t ${{ secrets.DOCKER_REGISTRY }}/scala-app:${{ github.sha }} .
    |        docker tag ${{ secrets.DOCKER_REGISTRY }}/scala-app:${{ github.sha }} ${{ secrets.DOCKER_REGISTRY }}/scala-app:latest
    |        
    |    - name: Login to Docker Registry
    |      run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login ${{ secrets.DOCKER_REGISTRY }} -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
    |      
    |    - name: Push Docker image
    |      run: |
    |        docker push ${{ secrets.DOCKER_REGISTRY }}/scala-app:${{ github.sha }}
    |        docker push ${{ secrets.DOCKER_REGISTRY }}/scala-app:latest
    |        
    |    - name: Deploy to Kubernetes
    |      run: |
    |        echo "${{ secrets.KUBE_CONFIG }}" | base64 -d > kubeconfig
    |        export KUBECONFIG=kubeconfig
    |        kubectl set image deployment/scala-app scala-app=${{ secrets.DOCKER_REGISTRY }}/scala-app:${{ github.sha }}
    |        kubectl rollout status deployment/scala-app
    |""".stripMargin

  // SBT release configuration
  val sbtReleaseConfig = """
    |// project/plugins.sbt
    |addSbtPlugin("com.github.sbt" % "sbt-release" % "1.1.0")
    |addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.1")
    |addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.9.0")
    |
    |// build.sbt release settings
    |import ReleaseTransformations._
    |
    |releaseProcess := Seq[ReleaseStep](
    |  checkSnapshotDependencies,
    |  inquireVersions,
    |  runClean,
    |  runTest,
    |  setReleaseVersion,
    |  commitReleaseVersion,
    |  tagRelease,
    |  publishArtifacts,
    |  setNextVersion,
    |  commitNextVersion,
    |  pushChanges
    |)
    |
    |// Docker configuration
    |enablePlugins(DockerPlugin)
    |docker / dockerfile := {
    |  val jarFile = (Compile / packageBin).value
    |  val classpath = (Compile / managedClasspath).value
    |  val mainclass = (Compile / packageBin / mainClass).value.getOrElse(sys.error("Expected exactly one main class"))
    |  val jarTarget = s"/app/${jarFile.getName}"
    |
    |  new Dockerfile {
    |    from("openjdk:11-jre-slim")
    |    add(jarFile, jarTarget)
    |    entryPoint("java", "-jar", jarTarget)
    |  }
    |}
    |""".stripMargin
}

// Run production deployment demonstrations
println("=== Production Deployment Strategy Demonstration ===")

// Health monitoring simulation
HealthMonitoring.simulateApplicationLoad()

println("\n=== Docker Configuration ===")
println("Multi-stage Dockerfile for optimized builds:")
println(DockerConfiguration.multiStageDockerfile.take(500) + "...")

println("\n=== CI/CD Pipeline ===")
println("GitHub Actions workflow configured for:")
println("- Automated testing")
println("- Docker image building")
println("- Kubernetes deployment")
println("- Coverage reporting")

println("\n=== Performance Analysis Complete ===")
println("Ready for production deployment with:")
println("✓ JVM optimization")
println("✓ Memory management")
println("✓ Health monitoring")
println("✓ Container orchestration")
println("✓ CI/CD automation")

Summary

In this lesson, you've mastered production-ready Scala development:

✅ Performance Analysis: JVM monitoring, profiling, and bottleneck identification
✅ Memory Management: Efficient data structures, object pooling, and leak prevention
✅ JVM Tuning: Garbage collection optimization and configuration for different scenarios
✅ Deployment Strategies: Containerization, Kubernetes orchestration, and scaling
✅ Monitoring: Health checks, metrics collection, and observability
✅ CI/CD: Automated testing, building, and deployment pipelines
✅ Production Best Practices: Configuration management and operational excellence

These skills ensure your Scala applications perform optimally in production environments and can scale to meet real-world demands.

Congratulations!

You've completed an advanced journey through Scala, from advanced types and implicits to production deployment. You now have the knowledge and tools to build high-performance, scalable Scala applications that excel in professional environments. Continue practicing these concepts and exploring the rich Scala ecosystem to become a Scala expert!