Production Deployment and DevOps: Enterprise Scala Application Lifecycle

Successfully deploying and managing Scala applications in production requires comprehensive understanding of modern DevOps practices, containerization, monitoring, and infrastructure automation. This lesson covers enterprise-grade deployment strategies, monitoring solutions, and operational best practices.

Containerization and Application Packaging

Docker Integration for Scala Applications

// Build configuration for containerized applications
// build.sbt
ThisBuild / version := "1.0.0"
ThisBuild / scalaVersion := "2.13.12"

lazy val root = (project in file("."))
  .enablePlugins(JavaAppPackaging, DockerPlugin)
  .settings(
    name := "scala-production-app",
    libraryDependencies ++= Seq(
      "com.typesafe.akka" %% "akka-http" % "10.5.0",
      "com.typesafe.akka" %% "akka-stream" % "2.8.0",
      "com.typesafe.akka" %% "akka-actor-typed" % "2.8.0",
      "ch.qos.logback" % "logback-classic" % "1.4.11",
      "net.logstash.logback" % "logstash-logback-encoder" % "7.4",
      "io.prometheus" % "simpleclient" % "0.16.0",
      "io.prometheus" % "simpleclient_hotspot" % "0.16.0",
      "io.prometheus" % "simpleclient_httpserver" % "0.16.0",
      "com.typesafe" % "config" % "1.4.2"
    ),

    // Docker configuration
    Docker / packageName := "scala-app",
    Docker / version := version.value,
    Docker / daemonUser := "appuser",
    dockerBaseImage := "openjdk:17-jre-slim",
    dockerExposedPorts := Seq(8080, 9090), // App and metrics ports
    dockerRepository := Some("your-registry.com"),

    // Multi-stage Docker build
    dockerCommands := {
      val appDir = "/opt/app"
      val configDir = s"$appDir/conf"
      val logsDir = s"$appDir/logs"

      Seq(
        Cmd("FROM", "openjdk:17-jre-slim"),
        Cmd("LABEL", "maintainer=\"Your Team <team@company.com>\""),

        // Install system dependencies
        Cmd("RUN", "apt-get update && apt-get install -y curl dumb-init && rm -rf /var/lib/apt/lists/*"),

        // Create app user and directories
        Cmd("RUN", s"useradd -m -u 1001 appuser"),
        Cmd("RUN", s"mkdir -p $appDir $configDir $logsDir"),
        Cmd("RUN", s"chown -R appuser:appuser $appDir"),

        // Copy application
        Cmd("COPY", "--chown=appuser:appuser opt/docker/lib /opt/app/lib"),
        Cmd("COPY", "--chown=appuser:appuser opt/docker/bin /opt/app/bin"),
        Cmd("COPY", "--chown=appuser:appuser opt/docker/conf /opt/app/conf"),

        // Health check
        Cmd("HEALTHCHECK", "--interval=30s --timeout=3s --start-period=5s --retries=3",
            "CMD curl -f http://localhost:8080/health || exit 1"),

        Cmd("USER", "appuser"),
        Cmd("WORKDIR", appDir),
        Cmd("ENTRYPOINT", "[\"dumb-init\", \"--\"]"),
        Cmd("CMD", "[\"bin/scala-production-app\"]")
      )
    }
  )

// Multi-stage Dockerfile template
val dockerfileTemplate = 
"""
# Multi-stage build for Scala applications
FROM sbt:1.9.6-openjdk-17 as builder

WORKDIR /app
COPY build.sbt .
COPY project/ project/
RUN sbt update

COPY src/ src/
RUN sbt stage

FROM openjdk:17-jre-slim as runtime

# Install runtime dependencies
RUN apt-get update && \
    apt-get install -y curl dumb-init netcat-openbsd && \
    rm -rf /var/lib/apt/lists/*

# Create application user
RUN useradd -m -u 1001 appuser

# Set up application directory
WORKDIR /opt/app
COPY --from=builder --chown=appuser:appuser /app/target/universal/stage/ .

# Create required directories
RUN mkdir -p logs conf data && \
    chown -R appuser:appuser /opt/app

# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
    CMD curl -f http://localhost:8080/health || exit 1

USER appuser
EXPOSE 8080 9090

ENTRYPOINT ["dumb-init", "--"]
CMD ["bin/scala-production-app"]
"""

Production-Ready Application Structure

// Production application configuration
import com.typesafe.config.{Config, ConfigFactory}
import akka.actor.typed.ActorSystem
import akka.actor.typed.scaladsl.Behaviors
import akka.http.scaladsl.Http
import akka.http.scaladsl.server.Route
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success}
import java.util.concurrent.TimeUnit
import io.prometheus.client.CollectorRegistry
import io.prometheus.client.hotspot.DefaultExports

// Main application entry point
object ProductionApp {

  def main(args: Array[String]): Unit = {
    // Load configuration with environment-specific overrides
    val config = loadConfiguration()

    // Initialize metrics collection
    initializeMetrics()

    // Start the application
    implicit val system: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "production-app", config)
    implicit val ec: ExecutionContext = system.executionContext

    val app = new ProductionApp(config)

    app.start().onComplete {
      case Success(binding) =>
        val localAddress = binding.localAddress
        system.log.info(s"Server online at http://${localAddress.getHostString}:${localAddress.getPort}/")

        // Register shutdown hook
        sys.addShutdownHook {
          system.log.info("Shutting down application...")
          app.shutdown().onComplete(_ => system.terminate())
        }

      case Failure(exception) =>
        system.log.error("Failed to start server", exception)
        system.terminate()
    }
  }

  private def loadConfiguration(): Config = {
    val environment = sys.env.getOrElse("ENVIRONMENT", "development")

    // Load configuration with environment-specific overrides
    val baseConfig = ConfigFactory.load()
    val envConfig = ConfigFactory.load(s"application-$environment.conf")

    envConfig.withFallback(baseConfig).resolve()
  }

  private def initializeMetrics(): Unit = {
    // Register JVM metrics
    DefaultExports.initialize()

    // Initialize custom metrics
    ApplicationMetrics.initialize()
  }
}

class ProductionApp(config: Config)(implicit system: ActorSystem[Nothing], ec: ExecutionContext) {

  import akka.http.scaladsl.server.Directives._

  private val httpConfig = config.getConfig("app.http")
  private val host = httpConfig.getString("host")
  private val port = httpConfig.getInt("port")
  private val metricsPort = httpConfig.getInt("metrics-port")

  // Application components
  private val healthCheck = new HealthCheckService(config)
  private val businessLogic = new BusinessLogicService(config)
  private val metricsService = new MetricsService()

  // Application routes
  private val routes: Route = {
    pathPrefix("api" / "v1") {
      businessLogic.routes
    } ~
    path("health") {
      healthCheck.route
    } ~
    path("ready") {
      healthCheck.readinessRoute
    } ~
    path("metrics") {
      metricsService.route
    }
  }

  def start(): Future[Http.ServerBinding] = {
    // Start metrics server
    startMetricsServer()

    // Start main application server
    Http().newServerAt(host, port).bind(routes)
  }

  private def startMetricsServer(): Future[Http.ServerBinding] = {
    val metricsRoute = path("metrics") {
      metricsService.route
    }

    Http().newServerAt(host, metricsPort).bind(metricsRoute)
  }

  def shutdown(): Future[Unit] = {
    for {
      _ <- businessLogic.shutdown()
      _ <- healthCheck.shutdown()
    } yield ()
  }
}

// Health check implementation
class HealthCheckService(config: Config)(implicit system: ActorSystem[Nothing], ec: ExecutionContext) {

  import akka.http.scaladsl.server.Directives._
  import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes}
  import spray.json._
  import DefaultJsonProtocol._

  case class HealthStatus(
    status: String,
    timestamp: Long,
    uptime: Long,
    version: String,
    checks: Map[String, CheckResult]
  )

  case class CheckResult(status: String, message: String, responseTime: Long)

  implicit val checkResultFormat: RootJsonFormat[CheckResult] = jsonFormat3(CheckResult)
  implicit val healthStatusFormat: RootJsonFormat[HealthStatus] = jsonFormat5(HealthStatus)

  private val startTime = System.currentTimeMillis()
  private val version = config.getString("app.version")
  private val databaseChecker = new DatabaseHealthChecker(config)
  private val externalServiceChecker = new ExternalServiceHealthChecker(config)

  val route: Route = {
    get {
      complete {
        performHealthCheck().map { status =>
          val httpStatus = if (status.status == "healthy") StatusCodes.OK else StatusCodes.ServiceUnavailable
          val entity = HttpEntity(ContentTypes.`application/json`, status.toJson.prettyPrint)
          (httpStatus, entity)
        }
      }
    }
  }

  val readinessRoute: Route = {
    get {
      complete {
        performReadinessCheck().map { isReady =>
          if (isReady) StatusCodes.OK else StatusCodes.ServiceUnavailable
        }
      }
    }
  }

  private def performHealthCheck(): Future[HealthStatus] = {
    val checksF = Future.sequence(Seq(
      checkDatabase(),
      checkExternalServices(),
      checkMemoryUsage(),
      checkDiskSpace()
    ))

    checksF.map { checks =>
      val checkMap = checks.toMap
      val overallStatus = if (checks.forall(_._2.status == "healthy")) "healthy" else "unhealthy"

      HealthStatus(
        status = overallStatus,
        timestamp = System.currentTimeMillis(),
        uptime = System.currentTimeMillis() - startTime,
        version = version,
        checks = checkMap
      )
    }
  }

  private def performReadinessCheck(): Future[Boolean] = {
    // Readiness check is simpler - just check if essential services are available
    for {
      dbReady <- databaseChecker.isReady()
      servicesReady <- externalServiceChecker.areEssentialServicesReady()
    } yield dbReady && servicesReady
  }

  private def checkDatabase(): Future[(String, CheckResult)] = {
    val startTime = System.nanoTime()
    databaseChecker.check().map { isHealthy =>
      val responseTime = (System.nanoTime() - startTime) / 1000000 // Convert to milliseconds
      val result = CheckResult(
        status = if (isHealthy) "healthy" else "unhealthy",
        message = if (isHealthy) "Database connection OK" else "Database connection failed",
        responseTime = responseTime
      )
      ("database", result)
    }.recover {
      case ex =>
        val responseTime = (System.nanoTime() - startTime) / 1000000
        ("database", CheckResult("unhealthy", s"Database check failed: ${ex.getMessage}", responseTime))
    }
  }

  private def checkExternalServices(): Future[(String, CheckResult)] = {
    val startTime = System.nanoTime()
    externalServiceChecker.checkAll().map { results =>
      val responseTime = (System.nanoTime() - startTime) / 1000000
      val allHealthy = results.forall(_._2)
      val message = if (allHealthy) "All external services OK" else s"Some services failing: ${results.filter(!_._2).map(_._1).mkString(", ")}"

      val result = CheckResult(
        status = if (allHealthy) "healthy" else "degraded",
        message = message,
        responseTime = responseTime
      )
      ("external-services", result)
    }
  }

  private def checkMemoryUsage(): Future[(String, CheckResult)] = {
    Future {
      val runtime = Runtime.getRuntime
      val maxMemory = runtime.maxMemory()
      val usedMemory = runtime.totalMemory() - runtime.freeMemory()
      val memoryUsagePercent = (usedMemory.toDouble / maxMemory.toDouble) * 100

      val status = if (memoryUsagePercent < 80) "healthy" else if (memoryUsagePercent < 90) "warning" else "critical"
      val message = f"Memory usage: ${memoryUsagePercent}%.1f%% (${usedMemory / 1024 / 1024}MB / ${maxMemory / 1024 / 1024}MB)"

      ("memory", CheckResult(status, message, 0))
    }
  }

  private def checkDiskSpace(): Future[(String, CheckResult)] = {
    Future {
      val file = new java.io.File(".")
      val totalSpace = file.getTotalSpace
      val freeSpace = file.getFreeSpace
      val usedSpace = totalSpace - freeSpace
      val diskUsagePercent = (usedSpace.toDouble / totalSpace.toDouble) * 100

      val status = if (diskUsagePercent < 80) "healthy" else if (diskUsagePercent < 90) "warning" else "critical"
      val message = f"Disk usage: ${diskUsagePercent}%.1f%% (${usedSpace / 1024 / 1024 / 1024}GB / ${totalSpace / 1024 / 1024 / 1024}GB)"

      ("disk", CheckResult(status, message, 0))
    }
  }

  def shutdown(): Future[Unit] = {
    for {
      _ <- databaseChecker.shutdown()
      _ <- externalServiceChecker.shutdown()
    } yield ()
  }
}

// Database health checker
class DatabaseHealthChecker(config: Config)(implicit ec: ExecutionContext) {
  // Implementation would depend on your database setup

  def check(): Future[Boolean] = {
    Future {
      // Perform actual database health check
      // Example: execute a simple query
      true
    }
  }

  def isReady(): Future[Boolean] = check()

  def shutdown(): Future[Unit] = Future.successful(())
}

// External service health checker
class ExternalServiceHealthChecker(config: Config)(implicit system: ActorSystem[Nothing], ec: ExecutionContext) {

  import akka.http.scaladsl.Http
  import akka.http.scaladsl.model.{HttpRequest, StatusCodes}
  import scala.concurrent.duration._

  private val externalServices = config.getConfigList("app.external-services").asScala.map { serviceConfig =>
    serviceConfig.getString("name") -> serviceConfig.getString("health-url")
  }.toMap

  def checkAll(): Future[Map[String, Boolean]] = {
    val checks = externalServices.map { case (name, url) =>
      checkService(url).map(name -> _)
    }

    Future.sequence(checks).map(_.toMap)
  }

  def areEssentialServicesReady(): Future[Boolean] = {
    checkAll().map(_.values.forall(identity))
  }

  private def checkService(url: String): Future[Boolean] = {
    Http().singleRequest(HttpRequest(uri = url))
      .map(_.status == StatusCodes.OK)
      .recover(_ => false)
  }

  def shutdown(): Future[Unit] = Future.successful(())
}

Monitoring and Observability

Comprehensive Metrics Collection

// Prometheus metrics integration
import io.prometheus.client._
import io.prometheus.client.hotspot.DefaultExports
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.model.{ContentTypes, HttpEntity}

object ApplicationMetrics {

  // Business metrics
  val requestsTotal: Counter = Counter.build()
    .name("http_requests_total")
    .help("Total number of HTTP requests")
    .labelNames("method", "endpoint", "status")
    .register()

  val requestDuration: Histogram = Histogram.build()
    .name("http_request_duration_seconds")
    .help("HTTP request duration in seconds")
    .labelNames("method", "endpoint")
    .buckets(0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)
    .register()

  val activeConnections: Gauge = Gauge.build()
    .name("http_active_connections")
    .help("Number of active HTTP connections")
    .register()

  val businessOperationsTotal: Counter = Counter.build()
    .name("business_operations_total")
    .help("Total number of business operations")
    .labelNames("operation", "result")
    .register()

  val businessOperationDuration: Histogram = Histogram.build()
    .name("business_operation_duration_seconds")
    .help("Business operation duration in seconds")
    .labelNames("operation")
    .register()

  // Database metrics
  val databaseConnectionsActive: Gauge = Gauge.build()
    .name("database_connections_active")
    .help("Number of active database connections")
    .register()

  val databaseQueriesTotal: Counter = Counter.build()
    .name("database_queries_total")
    .help("Total number of database queries")
    .labelNames("query_type", "result")
    .register()

  val databaseQueryDuration: Histogram = Histogram.build()
    .name("database_query_duration_seconds")
    .help("Database query duration in seconds")
    .labelNames("query_type")
    .register()

  // Cache metrics
  val cacheOperationsTotal: Counter = Counter.build()
    .name("cache_operations_total")
    .help("Total number of cache operations")
    .labelNames("operation", "result")
    .register()

  val cacheHitRatio: Gauge = Gauge.build()
    .name("cache_hit_ratio")
    .help("Cache hit ratio")
    .labelNames("cache_name")
    .register()

  // Error metrics
  val errorsTotal: Counter = Counter.build()
    .name("errors_total")
    .help("Total number of errors")
    .labelNames("error_type", "component")
    .register()

  def initialize(): Unit = {
    // Register JVM metrics
    DefaultExports.initialize()

    // Initialize custom metrics
    System.setProperty("io.prometheus.simpleclient.defaultMetrics", "true")
  }

  // Utility methods for timing operations
  def timeOperation[T](histogram: Histogram, labels: String*)(operation: => T): T = {
    val timer = histogram.labels(labels: _*).startTimer()
    try {
      operation
    } finally {
      timer.observeDuration()
    }
  }

  def timeOperationAsync[T](histogram: Histogram, labels: String*)(operation: => Future[T])(implicit ec: ExecutionContext): Future[T] = {
    val timer = histogram.labels(labels: _*).startTimer()
    operation.andThen {
      case _ => timer.observeDuration()
    }
  }
}

// Metrics service for Prometheus scraping
class MetricsService {

  import akka.http.scaladsl.server.Directives._

  val route = {
    get {
      complete {
        val writer = new java.io.StringWriter()
        TextFormat.write004(writer, CollectorRegistry.defaultRegistry.metricFamilySamples())
        HttpEntity(ContentTypes.`text/plain(UTF-8)`, writer.toString)
      }
    }
  }
}

// Custom metrics collector for business logic
class BusinessMetricsCollector extends Collector {

  import scala.jdk.CollectionConverters._

  override def collect(): java.util.List[Collector.MetricFamilySamples] = {
    val samples = List(
      new Collector.MetricFamilySamples(
        "business_custom_metric",
        Collector.Type.GAUGE,
        "Custom business metric",
        List(
          new Collector.MetricFamilySamples.Sample(
            "business_custom_metric",
            List("label1").asJava,
            List("value1").asJava,
            42.0
          )
        ).asJava
      )
    )

    samples.asJava
  }
}

// Application monitoring with structured logging
import ch.qos.logback.classic.Logger
import net.logstash.logback.marker.Markers
import org.slf4j.{LoggerFactory, MDC}

object ApplicationLogging {

  private val logger = LoggerFactory.getLogger("application").asInstanceOf[Logger]

  case class LogContext(
    requestId: String,
    userId: Option[String] = None,
    operation: String,
    component: String,
    additionalFields: Map[String, String] = Map.empty
  ) {
    def toMDC(): Unit = {
      MDC.put("requestId", requestId)
      userId.foreach(MDC.put("userId", _))
      MDC.put("operation", operation)
      MDC.put("component", component)
      additionalFields.foreach { case (k, v) => MDC.put(k, v) }
    }

    def clearMDC(): Unit = {
      MDC.clear()
    }
  }

  def withContext[T](context: LogContext)(operation: => T): T = {
    try {
      context.toMDC()
      operation
    } finally {
      context.clearMDC()
    }
  }

  def logBusinessEvent(
    event: String,
    level: String = "INFO",
    details: Map[String, Any] = Map.empty
  ): Unit = {
    val marker = Markers.appendEntries(details.asJava)

    level.toUpperCase match {
      case "DEBUG" => logger.debug(marker, event)
      case "INFO" => logger.info(marker, event)
      case "WARN" => logger.warn(marker, event)
      case "ERROR" => logger.error(marker, event)
      case _ => logger.info(marker, event)
    }
  }

  def logError(
    message: String,
    throwable: Throwable,
    context: Map[String, Any] = Map.empty
  ): Unit = {
    val marker = Markers.appendEntries(context.asJava)
    logger.error(marker, message, throwable)

    // Update error metrics
    ApplicationMetrics.errorsTotal.labels(
      throwable.getClass.getSimpleName,
      context.getOrElse("component", "unknown").toString
    ).inc()
  }
}

// Distributed tracing integration
import brave.{Tracing, Span}
import brave.sampler.Sampler
import zipkin2.reporter.AsyncReporter
import zipkin2.reporter.okhttp3.OkHttpSender

object DistributedTracing {

  private lazy val sender = OkHttpSender.create("http://zipkin:9411/api/v2/spans")
  private lazy val spanReporter = AsyncReporter.create(sender)

  private lazy val tracing = Tracing.newBuilder()
    .localServiceName("scala-production-app")
    .spanReporter(spanReporter)
    .sampler(Sampler.create(0.1f)) // Sample 10% of traces
    .build()

  def tracer = tracing.tracer()

  def withSpan[T](operationName: String, tags: Map[String, String] = Map.empty)(operation: Span => T): T = {
    val span = tracer.nextSpan().name(operationName).start()
    try {
      tags.foreach { case (key, value) => span.tag(key, value) }
      operation(span)
    } catch {
      case ex: Throwable =>
        span.tag("error", ex.getMessage)
        throw ex
    } finally {
      span.end()
    }
  }

  def shutdown(): Unit = {
    tracing.close()
    spanReporter.close()
    sender.close()
  }
}

CI/CD Pipeline Implementation

GitHub Actions Workflow

# .github/workflows/ci-cd.yml
name: CI/CD Pipeline