Skip to main content

Serverless & Infrastructure as Code

Table of Contents​

  1. Serverless Architecture
  2. AWS Lambda
  3. API Gateway
  4. Infrastructure as Code (Terraform)
  5. Kubernetes & Helm
  6. CloudFormation
  7. Serverless Architecture Patterns
  8. Infrastructure as Code Fundamentals
  9. Terraform State, Refactoring and Governance
  10. Senior and Principal Serverless and IaC Scenarios

Serverless Architecture​

┌─────────────────────────────────────────────────────────────┐
│ SERVERLESS ARCHITECTURE │
├─────────────────────────────────────────────────────────────┤
│ │
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
│ │ Client │───▶│API Gateway│───▶│ Lambda │ │
│ │ (Web) │ │ │ │ Function │ │
│ └──────────┘ └──────────┘ └─────┬─────┘ │
│ │ │
│ ▼ │
│ ┌──────────┐ │
│ │ DynamoDB│ │
│ │ (DB) │ │
│ └──────────┘ │
│ │
│ ┌──────────────────────────────────────────────────────┐ │
│ │ SERVERLESS BENEFITS │ │
│ ├──────────────────────────────────────────────────────┤ │
│ │ • No server management │ │
│ │ • Auto-scaling (zero to millions) │ │
│ │ • Pay per invocation (cost effective) │ │
│ │ • Built-in high availability │ │
│ │ • Event-driven architecture │ │
│ └──────────────────────────────────────────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────┘

AWS Lambda​

Lambda Functions​

# ============================================
# AWS LAMBDA FUNCTION
# ============================================
import json
import boto3
from datetime import datetime

# Initialize clients
dynamodb = boto3.resource('dynamodb')
sns = boto3.client('sns')

def lambda_handler(event, context):
"""
AWS Lambda handler function
- event: dict containing request data
- context: runtime information
"""

# Extract data from event
http_method = event.get('httpMethod')
path = event.get('path')
body = event.get('body')

# Route based on HTTP method
if http_method == 'GET':
return handle_get(event)
elif http_method == 'POST':
return handle_post(event)
elif http_method == 'DELETE':
return handle_delete(event)
else:
return {
'statusCode': 405,
'body': json.dumps({'error': 'Method not allowed'})
}

def handle_get(event):
"""Handle GET requests"""
user_id = event.get('pathParameters', {}).get('userId')

if user_id:
# Get single user
table = dynamodb.Table('Users')
response = table.get_item(Key={'userId': user_id})
item = response.get('Item')

if item:
return {
'statusCode': 200,
'body': json.dumps(item)
}
else:
return {
'statusCode': 404,
'body': json.dumps({'error': 'User not found'})
}
else:
# Get all users
table = dynamodb.Table('Users')
response = table.scan()
return {
'statusCode': 200,
'body': json.dumps(response.get('Items', []))
}

def handle_post(event):
"""Handle POST requests"""
body = json.loads(event.get('body', '{}'))

# Validate required fields
if 'email' not in body or 'name' not in body:
return {
'statusCode': 400,
'body': json.dumps({'error': 'Missing required fields'})
}

# Create user
table = dynamodb.Table('Users')
user_id = f"user_{datetime.now().timestamp()}"

item = {
'userId': user_id,
'name': body['name'],
'email': body['email'],
'createdAt': datetime.now().isoformat()
}

table.put_item(Item=item)

return {
'statusCode': 201,
'body': json.dumps(item)
}

def handle_delete(event):
"""Handle DELETE requests"""
user_id = event.get('pathParameters', {}).get('userId')

if not user_id:
return {
'statusCode': 400,
'body': json.dumps({'error': 'User ID required'})
}

table = dynamodb.Table('Users')
table.delete_item(Key={'userId': user_id})

return {
'statusCode': 204,
'body': json.dumps({})
}

Lambda with DynamoDB​

# Lambda with DynamoDB Streams
def process_stream_handler(event, context):
"""Process DynamoDB Stream events"""
for record in event['Records']:
event_name = record['eventName']

if event_name in ['INSERT', 'MODIFY']:
new_image = record['dynamodb'].get('NewImage', {})

# Process new/modified items
user_id = new_image.get('userId', {}).get('S')
email = new_image.get('email', {}).get('S')

# Send welcome email, create audit log, etc.
send_welcome_email(email)
create_audit_log(f"User {user_id} created")

elif event_name == 'REMOVE':
old_image = record['dynamodb'].get('OldImage', {})
user_id = old_image.get('userId', {}).get('S')
create_audit_log(f"User {user_id} deleted")

def send_welcome_email(email):
"""Send welcome email via SES"""
ses = boto3.client('ses')
ses.send_email(
Source='noreply@example.com',
Destination={'ToAddresses': [email]},
Message={
'Subject': {'Data': 'Welcome!'},
'Body': {'Text': {'Data': 'Thanks for signing up!'}}
}
)

Lambda Layers and Dependencies​

# Using shared dependencies via Lambda Layer
# Layer structure:
# /python/
# ├── requests/
# ├── pandas/
# └── common/ <- shared utilities

# In Lambda function
import common.utils as utils
import requests
import pandas as pd

def handler(event, context):
# Use shared utilities
validated_data = utils.validate(event)
processed = utils.transform(validated_data)
return processed

API Gateway​

API Gateway Integration​

# API Gateway configuration (Terraform)
resource "aws_api_gateway_rest_api" "api" {
name = "my-api"
description = "My API Gateway"

endpoint_configuration {
types = ["REGIONAL"]
}
}

resource "aws_api_gateway_resource" "users" {
rest_api_id = aws_api_gateway_rest_api.api.id
parent_id = aws_api_gateway_rest_api.api.root_resource_id
path_part = "users"
}

resource "aws_api_gateway_method" "get_users" {
rest_api_id = aws_api_gateway_rest_api.api.id
resource_id = aws_api_gateway_resource.users.id
http_method = "GET"
authorization = "NONE"
}

resource "aws_api_gateway_integration" "lambda" {
rest_api_id = aws_api_gateway_rest_api.api.id
resource_id = aws_api_gateway_method.get_users.resource_id
http_method = aws_api_gateway_method.get_users.http_method

integration_http_method = "POST"
type = "AWS_PROXY"
uri = aws_lambda_function.users.invoke_arn
}

API Gateway Caching and Throttling​

# API Gateway settings in Terraform
resource "aws_api_gateway_stage" "prod" {
deployment_id = aws_api_gateway_deployment.deployment.id
rest_api_id = aws_api_gateway_rest_api.api.id
stage_name = "prod"

settings {
# Caching
cache_data_encrypted = true
cache_ttl_in_seconds = 300
caching_enabled = true

# Throttling
throttling_burst_limit = 1000
throttling_rate_limit = 500

# Metrics
metrics_enabled = true
logging_level = "INFO"
}
}

Infrastructure as Code (Terraform)​

Terraform Basics​

# ============================================
# TERRAFORM CONFIGURATION
# ============================================

terraform {
required_version = ">= 1.0"

required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 5.0"
}
}

backend "s3" {
bucket = "my-terraform-state"
key = "prod/terraform.tfstate"
region = "us-east-1"
use_lockfile = true # S3-native locking (Terraform 1.10+); older setups use dynamodb_table
encrypt = true
}
}

provider "aws" {
region = var.aws_region

default_tags {
tags = {
Environment = var.environment
ManagedBy = "Terraform"
Project = "my-project"
}
}
}

variable "aws_region" {
description = "AWS region"
type = string
default = "us-east-1"
}

variable "environment" {
description = "Environment name"
type = string
default = "production"
}

output "lambda_function_name" {
value = aws_lambda_function.api.function_name
}

Complete Infrastructure​

# ============================================
# COMPLETE INFRASTRUCTURE
# ============================================

# VPC
resource "aws_vpc" "main" {
cidr_block = "10.0.0.0/16"
enable_dns_hostnames = true
enable_dns_support = true

tags = {
Name = "main-vpc"
}
}

# Subnets
resource "aws_subnet" "private" {
count = 3

vpc_id = aws_vpc.main.id
cidr_block = "10.0.${count.index + 1}.0/24"
availability_zone = data.aws_availability_zones.available.names[count.index]
map_public_ip_on_launch = false

tags = {
Name = "private-subnet-${count.index + 1}"
}
}

resource "aws_subnet" "public" {
count = 3

vpc_id = aws_vpc.main.id
cidr_block = "10.0.${count.index + 10}.0/24"
availability_zone = data.aws_availability_zones.available.names[count.index]
map_public_ip_on_launch = true

tags = {
Name = "public-subnet-${count.index + 1}"
}
}

# Internet Gateway
resource "aws_internet_gateway" "main" {
vpc_id = aws_vpc.main.id

tags = {
Name = "main-igw"
}
}

# Route Table
resource "aws_route_table" "public" {
vpc_id = aws_vpc.main.id

route {
cidr_block = "0.0.0.0/0"
gateway_id = aws_internet_gateway.main.id
}

tags = {
Name = "public-route-table"
}
}

# Lambda Function
resource "aws_lambda_function" "api" {
filename = "lambda_function.zip"
function_name = "api-function"
role = aws_iam_role.lambda_exec.arn
handler = "handler.lambda_handler"
source_code_hash = filebase64sha256("lambda_function.zip")

runtime = "python3.12"
timeout = 30
memory_size = 256

environment {
variables = {
DYNAMODB_TABLE = aws_dynamodb_table.users.name
ENVIRONMENT = var.environment
}
}

vpc_config {
subnet_ids = aws_subnet.private[*].id
security_group_ids = [aws_security_group.lambda.id]
}

depends_on = [
aws_iam_role_policy_attachment.lambda_basic,
aws_iam_role_policy_attachment.lambda_vpc
]
}

# DynamoDB Table
resource "aws_dynamodb_table" "users" {
name = "users"
billing_mode = "PAY_PER_REQUEST"
hash_key = "userId"

attribute {
name = "userId"
type = "S"
}

attribute {
name = "email"
type = "S"
}

global_secondary_index {
name = "email-index"
hash_key = "email"
projection_type = "ALL"
}

tags = {
Name = "users-table"
}
}

# IAM Role for Lambda
resource "aws_iam_role" "lambda_exec" {
name = "lambda_exec_role"

assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [{
Action = "sts:AssumeRole"
Effect = "Allow"
Principal = {
Service = "lambda.amazonaws.com"
}
}]
})
}

resource "aws_iam_role_policy_attachment" "lambda_basic" {
role = aws_iam_role.lambda_exec.name
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}

resource "aws_iam_role_policy_attachment" "lambda_vpc" {
role = aws_iam_role.lambda_exec.name
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
}

Terraform Modules​

# ============================================
# TERRAFORM MODULES
# ============================================

# modules/lambda/main.tf
variable "function_name" {}
variable "handler" {}
variable "runtime" {}
variable "memory_size" {}
variable "timeout" {}
variable "source_code_path" {}
variable "environment_variables" {}

resource "aws_lambda_function" "this" {
filename = var.source_code_path
function_name = var.function_name
handler = var.handler
runtime = var.runtime
memory_size = var.memory_size
timeout = var.timeout
role = aws_iam_role.this.arn

source_code_hash = filebase64sha256(var.source_code_path)

environment {
variables = var.environment_variables
}
}

resource "aws_iam_role" "this" {
name = "${var.function_name}-role"

assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [{
Action = "sts:AssumeRole"
Effect = "Allow"
Principal = {
Service = "lambda.amazonaws.com"
}
}]
})
}

output "function_arn" {
value = aws_lambda_function.this.arn
}

# modules/lambda/variables.tf
variable "function_name" {
type = string
}

variable "handler" {
type = string
}

variable "runtime" {
type = string
default = "python3.12"
}

variable "memory_size" {
type = number
default = 128
}

variable "timeout" {
type = number
default = 30
}

variable "source_code_path" {
type = string
}

variable "environment_variables" {
type = map(string)
default = {}
}

# Usage
module "api_lambda" {
source = "./modules/lambda"

function_name = "api-function"
handler = "handler.lambda_handler"
runtime = "python3.12"
memory_size = 256
timeout = 30
source_code_path = "lambda_function.zip"

environment_variables = {
DYNAMODB_TABLE = "users"
ENVIRONMENT = "production"
}
}

Kubernetes & Helm​

Helm Charts​

# ============================================
# HELM CHART STRUCTURE
# ============================================

# Chart.yaml
apiVersion: v2
name: web-application
description: A web application Helm chart
version: 1.0.0
appVersion: "1.0.0"

# values.yaml
replicaCount: 3

image:
repository: myregistry/webapp
pullPolicy: IfNotPresent
tag: "latest"

service:
type: ClusterIP
port: 80

ingress:
enabled: true
className: "nginx"
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
hosts:
- host: app.example.com
paths:
- path: /
pathType: Prefix

resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 100m
memory: 128Mi

autoscaling:
enabled: true
minReplicas: 2
maxReplicas: 10
targetCPUUtilizationPercentage: 70

# templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "web-application.fullname" . }}
labels:
{{- include "web-application.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
{{- include "web-application.selectorLabels" . | nindent 6 }}
template:
metadata:
labels:
{{- include "web-application.selectorLabels" . | nindent 8 }}
spec:
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
containerPort: 80
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: http
readinessProbe:
httpGet:
path: /ready
port: http
resources:
{{- toYaml .Values.resources | nindent 12 }}
env:
{{- range $key, $value := .Values.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}

CloudFormation​

# ============================================
# CLOUDFORMATION TEMPLATE
# ============================================
AWSTemplateFormatVersion: "2010-09-09"
Description: "Web Application Stack"

Parameters:
Environment:
Type: String
Default: production
AllowedValues:
- production
- staging
- development

Resources:
# Lambda Function
ApiFunction:
Type: AWS::Lambda::Function
Properties:
FunctionName: !Sub "${Environment}-api-function"
Runtime: python3.12
Handler: handler.lambda_handler
MemorySize: 256
Timeout: 30
Code:
S3Bucket: !Ref SourceCodeBucket
S3Key: lambda_function.zip
Environment:
Variables:
ENVIRONMENT: !Ref Environment
Role: !GetAtt LambdaExecutionRole.Arn
VpcConfig:
SubnetIds: !Ref SubnetIds
SecurityGroupIds:
- !Ref LambdaSecurityGroup

# Lambda Execution Role
LambdaExecutionRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Principal:
Service: lambda.amazonaws.com
Action: sts:AssumeRole
Policies:
- PolicyName: LambdaPolicy
PolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: "*"

# DynamoDB Table
UsersTable:
Type: AWS::DynamoDB::Table
Properties:
TableName: !Sub "${Environment}-users"
BillingMode: PAY_PER_REQUEST
AttributeDefinitions:
- AttributeName: userId
AttributeType: S
KeySchema:
- AttributeName: userId
KeyType: HASH

# API Gateway
ApiGateway:
Type: AWS::ApiGatewayV2::Api
Properties:
Name: !Sub "${Environment}-api"
ProtocolType: HTTP
Target: !GetAtt ApiFunction.Arn

# API Gateway Route
ApiRoute:
Type: AWS::ApiGatewayV2::Route
Properties:
ApiId: !Ref ApiGateway
RouteKey: "ANY /{proxy+}"
AuthorizationType: NONE
Target: !Sub "integrations/${ApiIntegration}"

Outputs:
ApiEndpoint:
Description: API Gateway endpoint
Value: !GetAtt ApiGateway.ApiEndpoint

Interview Questions​

Q1: What is serverless?​

Serverless is a cloud execution model where the cloud provider manages infrastructure, automatically allocates resources, and bills only for actual usage rather than pre-provisioned capacity.

Q2: What are the advantages of serverless?​

  • No server management required
  • Automatic scaling from zero to millions
  • Cost-effective (pay per invocation)
  • Built-in high availability
  • Event-driven architecture support

Q3: What are the limitations of Lambda?​

  • Cold starts (initial latency)
  • Execution time limits (15 minutes max)
  • Limited filesystem access
  • Package size limits (50 MB zipped for direct upload, 250 MB unzipped including layers, 10 GB for container images)
  • Stateless nature requires external storage

Q4: What is Infrastructure as Code?​

IaC is the practice of managing and provisioning infrastructure through code rather than manual processes, enabling version control, reproducibility, and automation.

Q5: Terraform vs CloudFormation?​

  • Terraform: Multi-cloud, declarative, larger community, state management
  • CloudFormation: AWS-native, integrated with AWS services, managed by AWS

Q6: What causes Lambda cold starts and how do you reduce them?​

Cold start = time to provision a sandbox and load the runtime before your handler runs. Causes: new instances, infrequent traffic, heavy dependencies, large deployments. Reduction: keep dependencies lean, use lighter runtimes (Python > Java), provisioned concurrency for latency-critical paths, SnapStart (Java), and warm keep-alives where cost allows.

Q7: DynamoDB vs RDS for serverless workloads?​

DynamoDB: NoSQL, pay-per-request, auto-scales to any traffic, good for key-value/JSON access patterns. RDS: relational SQL, joins, transactions, complex queries — but you pay for always-on capacity. Choose by query pattern; many apps combine both via a service layer (RDS for OLTP truth, DynamoDB for hot lookup).

Q8: What is an Event Source Mapping (ESM)?​

AWS Lambda's integration that polls a source (SQS, DynamoDB Streams, Kinesis) and invokes your function in batches. ESM handles retries, partial batch failures, and checkpoints. It lets Lambda act as a consumer without running a custom poller.

Q9: How does Lambda handle retries and dead letter queues (DLQ)?​

Synchronous invocations: client retries. Asynchronous (EventBridge, S3): Lambda retries 2 times with backoff, then sends the event to a configured DLQ (SQS/SNS) for later inspection and replay. Idempotent handlers are essential because events can arrive more than once.

Q10: What is Terraform state and why is it important?​

Terraform state records what resources it manages (ID, attributes, metadata) so it can diff, update, and delete them. It must be stored safely (e.g. S3 with versioning) and locked against concurrent writes (S3-native locking with use_lockfile in Terraform 1.10+, or a DynamoDB table in older setups). Losing state orphans resources; editing state manually causes drift.

Q11: What is OpenTofu and why does it exist?​

OpenTofu is an open-source fork of Terraform, created after HashiCorp switched Terraform to BUSL license. It keeps the same HCL/config syntax and providers. Teams choose it for an open-source (MPL) license while staying compatible with existing tooling. HCL and Terraform configs remain transferable.

Q12: Terraform vs Pulumi?​

Terraform: HCL, declarative, largest provider ecosystem, state-focused. Pulumi: real programming languages (Python/TS/Go), more expressive logic and loops, same declarative result. Choose Terraform for ops-standardization, Pulumi when you need code-level conditionals and reuse.

Q13: Serverless vs containers on ECS/Fargate?​

Serverless: no infrastructure management, scales to zero, pay per invocation — best for event-driven and bursty workloads. Fargate: run containers without managing clusters, good for long-running services, custom runtimes, or steady workloads needing predictable capacity. Not either/or — many apps run both.

Q14: What is IaC drift?​

Drift = the difference between declared infrastructure and what actually exists (someone changed a resource via console). Drift makes future plans wrong and destroys changes. Detect with terraform plan in CI, scheduled terraform plan -detailed-exitcode runs, or AWS Config (driftctl is no longer maintained); fix via reapply or adopt the change intentionally.

Q15: What is the Shared Responsibility Model in serverless?​

AWS secures the cloud (physical infra, Lambda runtime, platform). You secure what you run in the cloud: function code, IAM roles and permissions, environment variables (secrets), event sources, VPC config, and API access. Blast radius of misconfig is yours, not AWS's.

Q16: How do you handle secrets in serverless apps?​

Use Parameter Store / Secrets Manager, retrieve at runtime (or via Lambda extensions), never hardcode or commit them. Use IAM roles instead of keys where possible. Rotate secrets and version them. Check env vars into the console only for non-secret config.

Q17: What is Lambda SnapStart?​

SnapStart takes a snapshot of the initialized execution environment before the handler runs, then restores it on cold start — cutting cold start latency substantially, often from seconds to sub-second. It supports Java, Python and .NET runtimes. Trade-off: random-number state must be re-seeded; in-memory state is frozen at snapshot time.

Q18: Cost optimization levers for serverless?​

Match concurrency to traffic (avoid overrun waste), use reserved/provisioned concurrency only when needed, enable Compute Savings Plans, consolidate short functions, stream logs to cheaper storage, and watch for hot-loop infinite invocations. Setting billing alarms is non-negotiable.

Q19: How do you define infrastructure per environment?​

Use Terraform workspaces or per-env directories with variable files (dev.tfvars, prod.tfvars), keep environment-specific values in config not code, use separate state files per environment, and gate prod applies behind approval in CI.

Q20: What are serverless observability challenges?​

No fixed hosts means agents don't work; you rely on Lambda's built-in CloudWatch + X-Ray or OpenTelemetry extension. Cold starts create latency spikes that skew p99. Ensure correlation IDs flow through handlers, and add structured logging for queryability.


Serverless Architecture Patterns​

Q21: When is serverless a good fit, and when is it the wrong choice?​

Serverless here means functions-as-a-service (AWS Lambda, Google Cloud Run functions, Azure Functions) plus managed building blocks (queues, event buses, managed databases), where you pay per use and the provider handles scaling and servers.

Good fit:

  • Spiky or unpredictable traffic, including long idle periods (scale to zero).
  • Event-driven glue: reacting to file uploads, queue messages, webhooks, database change streams, schedules.
  • Short, stateless units of work that finish in seconds.
  • Small teams that want to minimize operations work, and prototypes where time-to-market matters more than unit cost.

Poor fit:

  • Steady, high-throughput traffic: per-invocation pricing can cost several times more than always-on containers at sustained load.
  • Long-running or stateful work: hard execution time limits (15 minutes on Lambda), no durable local disk, connections that must stay open for a long time.
  • Strict, consistent low latency where cold starts are unacceptable and pre-warming costs erase the savings.
  • Heavy use of connection-based resources, such as a relational database with a small connection limit, unless you add pooling (for example RDS Proxy).
  • Specialized hardware or runtimes (GPUs, custom kernels, large in-memory models).

A senior answer avoids "serverless everywhere" and "serverless never": many systems use containers for the steady core API and functions for asynchronous, bursty side work.

Q22: How do cold starts influence architecture, beyond tuning a single function?​

A cold start happens when the platform must create a new execution environment: allocate the sandbox, download code, start the runtime, and run your initialization code. It hits the first request after idle time and every request that triggers scale-out, so bursty traffic produces many cold starts at once.

Architectural levers, in rough order of preference:

  1. Keep cold starts off the critical path: put latency-insensitive work behind a queue so the user-facing request only enqueues.
  2. Reduce initialization cost: smaller deployment packages, lazy imports, lighter runtimes and frameworks, create SDK clients once outside the handler and reuse them.
  3. Keep capacity warm when latency matters and traffic justifies it: Lambda provisioned concurrency, Cloud Run minimum instances (gcloud run deploy orders-api --min-instances=1), Azure Functions Premium plan pre-warmed instances. This trades the pay-per-use model for a baseline cost.
  4. Choose a different compute model for consistently latency-critical, steady endpoints (containers), keeping functions for asynchronous work.

Measure before optimizing: look at the percentage of invocations that are cold and their p99 latency contribution. Scheduled "ping to keep warm" hacks only keep one instance warm and do not help with concurrent scale-out.

Q23: Why must event-driven handlers be idempotent, and how do you implement idempotency?​

Most event sources deliver at least once: SQS standard queues, SNS, EventBridge, S3 notifications and asynchronous Lambda invocations can all deliver the same event more than once, and retries after timeouts re-run work that may have partly succeeded. A handler is idempotent if processing the same event twice has the same effect as processing it once.

Techniques:

  • Idempotency key: a unique ID per business operation (order ID, payment intent ID, or the event's own ID). Record it atomically with the work; skip duplicates.
  • Conditional writes: "insert only if not exists", "update only if version = N".
  • Naturally idempotent operations: PUT the full state instead of incrementing counters.

DynamoDB conditional write as a deduplication guard:

import json
import os
import time

import boto3
from botocore.exceptions import ClientError

dynamodb = boto3.resource("dynamodb")
table = dynamodb.Table(os.environ["IDEMPOTENCY_TABLE"])


def claim(idempotency_key: str, ttl_seconds: int = 86400) -> bool:
"""Return True if this is the first time we see the key."""
try:
table.put_item(
Item={"pk": idempotency_key, "expires_at": int(time.time()) + ttl_seconds},
ConditionExpression="attribute_not_exists(pk)",
)
return True
except ClientError as err:
if err.response["Error"]["Code"] == "ConditionalCheckFailedException":
return False
raise


def handler(event, context):
for record in event["Records"]: # SQS batch
body = json.loads(record["body"])
if not claim(f"charge#{body['payment_id']}"):
continue # duplicate delivery, already processed
charge_customer(body) # placeholder for your business logic


def charge_customer(body):
...

Caveat: claiming the key before doing the work means a crash after the claim loses the work; claiming after means a crash in between causes a duplicate. Robust implementations store a status (IN_PROGRESS with an expiry, then COMPLETED with the stored result), which is what Powertools for AWS Lambda's idempotency utility does. Where possible, also pass the idempotency key to downstream APIs (payment providers commonly accept an idempotency key header).

Q24: How do you design fan-out with queues, and why put a queue in front of each consumer?​

Fan-out means one event triggers several independent consumers (send email, update search index, start fulfillment). A common AWS shape is SNS topic (or EventBridge bus) to one SQS queue per consumer to a function.

Why a queue per consumer instead of direct invocation:

  • Isolation: a slow or failing consumer does not affect the others; each has its own backlog, retry policy and dead-letter queue (DLQ).
  • Buffering and backpressure: the queue absorbs bursts; you cap consumer concurrency to protect downstream systems.
  • Replay: messages that failed go to a DLQ and can be redriven after a fix.
resource "aws_sns_topic" "order_events" {
name = "order-events"
}

resource "aws_sqs_queue" "email_dlq" {
name = "order-email-dlq"
message_retention_seconds = 1209600 # 14 days
}

resource "aws_sqs_queue" "email" {
name = "order-email"
visibility_timeout_seconds = 180 # at least 6x the function timeout is a common guideline
redrive_policy = jsonencode({
deadLetterTargetArn = aws_sqs_queue.email_dlq.arn
maxReceiveCount = 5
})
}

resource "aws_sns_topic_subscription" "email" {
topic_arn = aws_sns_topic.order_events.arn
protocol = "sqs"
endpoint = aws_sqs_queue.email.arn
raw_message_delivery = true
}

data "aws_iam_policy_document" "email_queue" {
statement {
actions = ["sqs:SendMessage"]
resources = [aws_sqs_queue.email.arn]

principals {
type = "Service"
identifiers = ["sns.amazonaws.com"]
}

condition {
test = "ArnEquals"
variable = "aws:SourceArn"
values = [aws_sns_topic.order_events.arn]
}
}
}

resource "aws_sqs_queue_policy" "email" {
queue_url = aws_sqs_queue.email.id
policy = data.aws_iam_policy_document.email_queue.json
}

SNS vs EventBridge for the fan-out point: SNS has very high throughput and simple subscription filters; EventBridge offers content-based routing rules, schema registry, archive and replay, and cross-account buses, with lower default throughput quotas. Either way, consumers must be idempotent and must not assume ordering unless you use FIFO variants.

Q25: Orchestration vs choreography: how do you coordinate multi-step serverless workflows?​

  • Choreography: services emit events and react to each other's events; there is no central coordinator. OrderPlaced leads payment to emit PaymentCaptured, which leads inventory to emit StockReserved, and so on.
  • Orchestration: a central workflow (AWS Step Functions, Azure Durable Functions, Google Workflows, Temporal) explicitly calls each step, handles retries, timeouts and compensation, and holds the state of each execution.
ChoreographyOrchestration
CouplingLoose; producers do not know consumersOrchestrator knows every step
VisibilityFlow is implicit, spread across servicesFlow is explicit; each execution's state is inspectable
Error handlingEach service handles its own failures; compensation is distributedCentralized retries, timeouts, compensation
ChangeEasy to add new consumersChanging the flow means changing the workflow definition
Best forBroadcasting facts to many independent consumersBusiness processes with ordering, deadlines and rollback

Rule of thumb: choreograph between bounded contexts, orchestrate within a business process. For example, the order service orchestrates "reserve stock, charge card, create shipment" with Step Functions, then publishes a single OrderCompleted event that analytics, email and loyalty services consume independently.

Q26: How do you implement a saga with compensation in a serverless workflow?​

A saga replaces a distributed transaction with a sequence of local transactions, each with a compensating action that semantically undoes it (refund instead of rollback). If step N fails, the workflow runs the compensations for steps N-1 down to 1.

AWS Step Functions (Amazon States Language) expresses this with Retry for transient errors and Catch to route permanent failures into compensation states:

{
"Comment": "Order saga: reserve stock, charge payment, create shipment",
"StartAt": "ReserveStock",
"States": {
"ReserveStock": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": { "FunctionName": "reserve-stock", "Payload.$": "$" },
"ResultPath": "$.reservation",
"Retry": [
{ "ErrorEquals": ["States.TaskFailed"], "IntervalSeconds": 2, "MaxAttempts": 3, "BackoffRate": 2.0 }
],
"Catch": [
{ "ErrorEquals": ["States.ALL"], "ResultPath": "$.error", "Next": "OrderFailed" }
],
"Next": "ChargePayment"
},
"ChargePayment": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": { "FunctionName": "charge-payment", "Payload.$": "$" },
"ResultPath": "$.payment",
"Retry": [
{ "ErrorEquals": ["States.TaskFailed"], "IntervalSeconds": 2, "MaxAttempts": 3, "BackoffRate": 2.0 }
],
"Catch": [
{ "ErrorEquals": ["States.ALL"], "ResultPath": "$.error", "Next": "ReleaseStock" }
],
"Next": "CreateShipment"
},
"CreateShipment": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": { "FunctionName": "create-shipment", "Payload.$": "$" },
"ResultPath": "$.shipment",
"Catch": [
{ "ErrorEquals": ["States.ALL"], "ResultPath": "$.error", "Next": "RefundPayment" }
],
"End": true
},
"RefundPayment": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": { "FunctionName": "refund-payment", "Payload.$": "$" },
"ResultPath": null,
"Retry": [
{ "ErrorEquals": ["States.ALL"], "IntervalSeconds": 5, "MaxAttempts": 5, "BackoffRate": 2.0 }
],
"Next": "ReleaseStock"
},
"ReleaseStock": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"Parameters": { "FunctionName": "release-stock", "Payload.$": "$" },
"ResultPath": null,
"Retry": [
{ "ErrorEquals": ["States.ALL"], "IntervalSeconds": 5, "MaxAttempts": 5, "BackoffRate": 2.0 }
],
"Next": "OrderFailed"
},
"OrderFailed": {
"Type": "Fail",
"Error": "OrderFailed",
"Cause": "A saga step failed and compensations were executed"
}
}
}

Key points:

  • Compensations must be idempotent and retried aggressively; they may run more than once and must eventually succeed. If they cannot, alert a human (send to a DLQ or a manual review queue).
  • ResultPath keeps each step's output in its own field so later compensations still see the original input (for example, the payment ID needed to refund).
  • Sagas give eventual consistency, not isolation: other requests can observe intermediate states (stock reserved but payment not yet charged), which the business process must tolerate.

Q27: How does the serverless cost model work, and when does it flip against you?​

Function pricing is roughly requests x price-per-request + (memory x duration) x price-per-GB-second, plus every managed service in the path (API gateway per request, queue requests, log ingestion, data transfer). Idle time costs nothing; busy time costs a premium compared with raw compute.

Worked example with AWS Lambda list prices for x86 in us-east-1 at the time of writing ($0.20 per million requests, about $0.0000166667 per GB-second; check current pricing and free tier):

Steady load: 100 requests/second, 200 ms average duration, 512 MB memory, 30-day month

Requests/month : 100 x 2,592,000 s = 259.2 million
Request cost : 259.2 x $0.20 = ~$52
Compute : 259.2M x 0.2 s x 0.5 GB = 25.92 million GB-s
Compute cost : 25.92M x $0.0000166667 = ~$432
Lambda total = ~$484/month (before API gateway, logs, transfer)

The same steady 100 req/s might be served by a few small always-on containers, often for a fraction of that, so at sustained high utilization containers usually win. At 1 request/second average with long idle periods, the Lambda bill would be about 1/100 of the figure above and containers would sit mostly idle, so serverless wins.

Where costs flip or surprise:

  • High, steady utilization (as above).
  • Over-provisioned memory or slow downstream calls: you pay for wall-clock time spent waiting on I/O.
  • Hidden per-request services: API gateway, NAT gateway data processing, CloudWatch Logs ingestion of verbose logs can each exceed the compute bill.
  • Runaway invocations: recursive triggers or retry storms scale to the concurrency limit and bill accordingly.

Levers: right-size memory with measurements (more memory also means more CPU and may reduce duration), use Arm (Graviton) where supported, batch messages, reduce log volume, set budgets and anomaly alerts, and re-evaluate the compute model when traffic becomes steady.

Q28: How real is vendor lock-in with serverless, and how do you manage it?​

Lock-in is real but often misjudged. The function code is rarely the problem; the coupling is in the managed services and event formats around it: DynamoDB data model and access patterns, Step Functions definitions, IAM policies, EventBridge rules, event payload shapes, and operational tooling.

Ways to manage it without giving up the benefits:

  • Thin handlers, portable core: the handler translates the provider's event into a domain call; business logic does not import provider SDKs (hexagonal architecture, ports and adapters).
  • Standard interfaces where they are cheap: containers for compute (Cloud Run, Lambda container images, Fargate), OpenTelemetry for tracing, SQL databases, CloudEvents for event envelopes.
  • Accept lock-in deliberately where the managed service saves significant effort, and record the decision and the exit cost (an architecture decision record).
# domain/orders.py - no cloud SDK imports
from dataclasses import dataclass


@dataclass
class PlaceOrder:
order_id: str
customer_id: str
amount_cents: int


class OrderService:
def __init__(self, repo, publisher):
self.repo = repo # port: implemented by DynamoDB, Postgres, or an in-memory fake
self.publisher = publisher # port: implemented by EventBridge, Kafka, or a test double

def place(self, cmd: PlaceOrder) -> None:
self.repo.save(cmd)
self.publisher.publish("OrderPlaced", {"order_id": cmd.order_id})
# adapters/lambda_handler.py - the only file that knows about the Lambda event format
import json

from adapters.dynamo_repo import DynamoOrderRepo
from adapters.eventbridge_publisher import EventBridgePublisher
from domain.orders import OrderService, PlaceOrder

service = OrderService(DynamoOrderRepo(), EventBridgePublisher())


def handler(event, context):
body = json.loads(event["body"])
service.place(PlaceOrder(**body))
return {"statusCode": 202, "body": json.dumps({"status": "accepted"})}

The honest framing in an interview: portability has a cost too (lowest-common-denominator designs, extra abstraction layers). Optimize for the probability and cost of actually moving, not for a theoretical multi-cloud future.

Infrastructure as Code Fundamentals​

Q29: What is the difference between declarative and imperative infrastructure as code?​

  • Imperative: you write the steps to reach a state ("create a VPC, then create two subnets, then attach a gateway"). Examples: shell scripts with the AWS CLI, Ansible playbooks used procedurally, SDK scripts. Re-running a script may fail or create duplicates unless every step checks current state first.
  • Declarative: you describe the desired end state ("a VPC with these two subnets exists") and the tool computes the difference from current state and the actions to close it. Examples: Terraform/OpenTofu, CloudFormation, Kubernetes manifests.
# Imperative: running this twice creates two VPCs
aws ec2 create-vpc --cidr-block 10.0.0.0/16
# Declarative: applying this twice converges to one VPC
resource "aws_vpc" "main" {
cidr_block = "10.0.0.0/16"
}

Declarative tools give you idempotency, a preview of changes (plan), and drift detection, which is why they dominate infrastructure provisioning. Note that "declarative" is about the model, not the language: Pulumi and AWS CDK let you write TypeScript or Python, but the program builds a desired-state graph that an engine diffs and applies, so they are still declarative in outcome.

Q30: How do you choose between Terraform, OpenTofu, Pulumi, AWS CDK and CloudFormation?​

ToolLanguageStateCloudsNotes
TerraformHCLState file you manage (S3, HCP Terraform, and others)Any with a providerLargest provider ecosystem; Business Source License since 2023
OpenTofuHCL (Terraform-compatible)Same as Terraform, plus optional client-side state encryptionAny with a providerOpen-source (MPL) fork under the Linux Foundation
PulumiTypeScript, Python, Go, .NET, Java, YAMLPulumi Cloud or self-managed backend (for example S3)Any with a providerReal languages: loops, functions, unit tests, package managers
AWS CDKTypeScript, Python, Java, .NET, GoSynthesizes CloudFormation; AWS stores stack stateAWSHigh-level constructs; deploys via CloudFormation
CloudFormationYAML or JSONManaged by AWS (stacks)AWSNo state file to secure; change sets; drift detection; native rollback on failure

Decision factors:

  • Multi-cloud or SaaS providers (Datadog, GitHub, Cloudflare, PagerDuty) point to Terraform/OpenTofu or Pulumi.
  • AWS-only teams that want no state management often pick CloudFormation or CDK.
  • Team skills: platform/ops teams often prefer HCL's constrained, reviewable syntax; application teams may be more productive with Pulumi or CDK in a language they already use.
  • Licensing and governance: some organizations moved to OpenTofu because of Terraform's license change.
  • Existing estate: migrating tools is expensive; consistency across the organization usually beats marginal tool benefits.

Q31: How do you set up remote state and locking for Terraform, and why does it matter?​

Local state on a laptop fails as soon as two people or a CI pipeline manage the same infrastructure: no shared view, no protection against concurrent apply, and the file contains sensitive values. A remote backend stores state centrally, and locking ensures only one operation writes it at a time.

S3 backend with S3-native locking (Terraform 1.10 and later support use_lockfile; older setups use a DynamoDB table via dynamodb_table, which newer versions deprecate):

terraform {
required_version = ">= 1.10"

backend "s3" {
bucket = "acme-tfstate-prod"
key = "network/terraform.tfstate"
region = "us-east-1"
encrypt = true
use_lockfile = true
}
}

Hardening the state bucket itself (managed in a separate bootstrap configuration):

resource "aws_s3_bucket" "tfstate" {
bucket = "acme-tfstate-prod"
}

resource "aws_s3_bucket_versioning" "tfstate" {
bucket = aws_s3_bucket.tfstate.id
versioning_configuration {
status = "Enabled" # lets you recover an earlier state version
}
}

resource "aws_s3_bucket_server_side_encryption_configuration" "tfstate" {
bucket = aws_s3_bucket.tfstate.id
rule {
apply_server_side_encryption_by_default {
sse_algorithm = "aws:kms"
}
}
}

resource "aws_s3_bucket_public_access_block" "tfstate" {
bucket = aws_s3_bucket.tfstate.id
block_public_acls = true
block_public_policy = true
ignore_public_acls = true
restrict_public_buckets = true
}

Also: restrict IAM access to the bucket (state can contain secrets), use one state file per component and environment to limit blast radius, and never edit state by hand; use terraform state subcommands, moved, import and removed blocks.

Q32: How do you run terraform plan and apply safely in CI/CD?​

Goals: every change is reviewed as a plan, the apply executes exactly the reviewed plan, applies are serialized, and production applies need approval.

name: terraform-prod
on:
push:
branches: [main]
paths: ["infra/prod/**"]

permissions:
contents: read
id-token: write

concurrency:
group: terraform-prod
cancel-in-progress: false

jobs:
plan:
runs-on: ubuntu-latest
defaults:
run:
working-directory: infra/prod
steps:
- uses: actions/checkout@v4
- uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::123456789012:role/terraform-plan-prod
aws-region: us-east-1
- run: terraform init -input=false
- run: terraform plan -input=false -out=tfplan
- run: terraform show -no-color tfplan > plan.txt
- uses: actions/upload-artifact@v4
with:
name: tfplan
path: |
infra/prod/tfplan
infra/prod/plan.txt
retention-days: 1

apply:
needs: plan
runs-on: ubuntu-latest
environment: production # required reviewers read plan.txt before approving
defaults:
run:
working-directory: infra/prod
steps:
- uses: actions/checkout@v4
- uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::123456789012:role/terraform-apply-prod
aws-region: us-east-1
- uses: actions/download-artifact@v4
with:
name: tfplan
path: infra/prod
- run: terraform init -input=false
- run: terraform apply -input=false tfplan

Practices behind this workflow:

  • Saved plan files: apply tfplan applies exactly what was planned; if state changed since the plan, Terraform rejects the stale plan instead of doing something different.
  • Separate roles: the plan role is read-only; only the apply job (behind environment approval) can assume the write role.
  • Plan files contain sensitive values; keep artifact retention short and access restricted.
  • Pin versions: required_version, provider constraints and the committed .terraform.lock.hcl make plan and apply use identical tooling.
  • Run fmt -check, validate, linting and policy checks on pull requests, and post the plan to the PR for review. Tools such as Atlantis, HCP Terraform, Spacelift or env0 package this workflow.

Q33: How do you structure and version Terraform modules?​

A module is a directory of Terraform files with input variables and outputs. Good modules encapsulate one concept (a VPC, a service with its IAM role and alarms) with a small, stable interface.

Versioning rules:

  • Consume shared modules from a registry or a git tag, never from a moving branch, so every environment pins a known version.
  • Follow semantic versioning: breaking input or output changes, or changes that replace resources, are major versions.
  • Promote module versions through environments like application releases: bump in dev, then staging, then prod.
# Public registry module with a version constraint
module "vpc" {
source = "terraform-aws-modules/vpc/aws"
version = "~> 5.0"

name = "prod"
cidr = "10.0.0.0/16"
}

# Internal module pinned to a git tag (version argument is not supported for git sources)
module "orders_service" {
source = "git::https://github.com/acme/terraform-modules.git//modules/ecs-service?ref=v2.3.0"

name = "orders-api"
desired_count = 3
}

Module design tips: expose only the variables consumers need, with type, description and validation blocks; output IDs and ARNs other stacks need; avoid configuring providers inside reusable modules; and include a moved block when you refactor resource addresses so consumers do not get destroy-and-recreate plans on upgrade.

Q34: Terraform workspaces vs separate state per environment: which do you use for dev, staging and prod?​

CLI workspaces (terraform workspace new staging) give one configuration multiple state files in the same backend. They are convenient for short-lived, identical copies (for example, an ephemeral environment per feature branch).

For long-lived environments, most teams prefer separate root modules (directories) with separate state and separate credentials:

infra/
modules/
service/
envs/
dev/
main.tf # module "service" { source = "../../modules/service" ... }
backend.tf # key = "dev/service.tfstate", dev account
staging/
main.tf
backend.tf # key = "staging/service.tfstate", staging account
prod/
main.tf
backend.tf # key = "prod/service.tfstate", prod account

Why separate directories for environments:

  • Isolation: workspaces share the backend and usually the same credentials, so someone with access to dev state can reach prod state, and the current workspace is invisible in the code (easy to run apply in the wrong one).
  • Differences are explicit: prod may have multi-AZ databases or extra alarms; in a directory they are visible in code review, not hidden behind terraform.workspace conditionals.
  • Promotion is reviewable: upgrading a module version in prod is a diff in envs/prod/.

The cost is some duplication of root-module wiring, which tools like Terragrunt or a thin root module calling shared modules keep small.

Terraform State, Refactoring and Governance​

Q35: How do you bring existing, manually created resources under Terraform management?​

Use import, which records an existing resource in state so Terraform manages it from then on, instead of trying to create a duplicate.

Since Terraform 1.5, the preferred way is a declarative import block, which goes through normal plan and review, and Terraform can generate starting configuration for it:

# imports.tf
import {
to = aws_s3_bucket.logs
id = "acme-prod-logs"
}
# Generate HCL for imported resources that have no resource block yet
terraform plan -generate-config-out=generated.tf

# Review and clean generated.tf, then plan again until it shows "1 to import, 0 to change"
terraform plan
terraform apply

The older CLI form, terraform import aws_s3_bucket.logs acme-prod-logs, modifies state immediately without a plan, which is why the block form is better for teams and CI.

Tips:

  • The goal after import is a plan with no changes; any diff means your configuration does not yet match reality.
  • Generated configuration is a starting point: replace hard-coded IDs with references and remove attributes that are just defaults.
  • For large estates, tools that generate import blocks in bulk exist, but review everything they produce.

Q36: What are moved and removed blocks, and why are they safer than terraform state mv?​

Renaming a resource or moving it into a module changes its address. Without help, Terraform plans to destroy the old address and create the new one, which for a database means data loss.

A moved block (Terraform 1.1+) records the refactor in code, so every environment and every module consumer gets the same rename during their next plan:

# Renamed a resource
moved {
from = aws_db_instance.db
to = aws_db_instance.orders
}

# Moved a resource into a module
moved {
from = aws_sqs_queue.email
to = module.notifications.aws_sqs_queue.email
}

A removed block (Terraform 1.7+) stops managing a resource without destroying it, for example when handing it over to another state:

removed {
from = aws_s3_bucket.legacy_reports

lifecycle {
destroy = false
}
}

Why blocks beat terraform state mv and terraform state rm:

  • They are reviewed in a pull request and show up in plan ("has moved to"), instead of being one-off commands someone ran against one environment.
  • They apply consistently to dev, staging and prod, and to every consumer of a shared module.
  • They are safe to leave in place for a while; remove old moved blocks only once all states have been updated.

Q37: Why can Terraform state contain secrets, and how do you protect them?​

State stores every attribute of every managed resource so Terraform can compute diffs. That includes values marked sensitive: sensitive = true only hides values from CLI output, it does not encrypt or omit them in state. Database passwords, generated keys (tls_private_key, random_password), and access keys created by Terraform all end up in the state file in plain text.

Mitigations, in order of effectiveness:

  1. Do not let secrets pass through Terraform at all. Let the service generate and store them. For example, RDS can manage the master password in Secrets Manager:
resource "aws_db_instance" "orders" {
identifier = "orders-prod"
engine = "postgres"
instance_class = "db.t4g.medium"
allocated_storage = 50
username = "orders_admin"
manage_master_user_password = true # password generated and rotated in Secrets Manager, never in state
deletion_protection = true
skip_final_snapshot = false
final_snapshot_identifier = "orders-prod-final"
}
  1. Reference secrets, do not create them: create the secret container with Terraform, but set its value out-of-band or with a rotation function.
  2. Encrypt and restrict the backend: KMS encryption on the bucket, tight IAM on who can read state, access logging. OpenTofu additionally supports client-side state encryption.
  3. Newer Terraform releases add ephemeral values that are never persisted to plan or state; use them where your Terraform version and providers support them.
  4. Treat anyone with state read access as having access to those secrets, and design permissions accordingly.

Q38: What is policy as code for infrastructure, and how do OPA, Sentinel and Checkov differ?​

Policy as code turns rules such as "no public S3 buckets", "all resources tagged with an owner" or "no destroying databases without an override" into automated checks that run in CI before apply.

ToolPolicy languageChecksTypical use
Checkov, tfsec/TrivyBuilt-in rules (Python/YAML custom rules)Terraform source code (and plan JSON for some tools)Fast static scan for common misconfigurations
OPA / ConftestRegoAny JSON/YAML, including terraform show -json plan outputCustom organization rules on the actual planned changes
SentinelSentinelPlans in HCP Terraform / Terraform EnterpriseEnforced policy sets with advisory, soft-mandatory and hard-mandatory levels

Checking the plan is more accurate than checking source code because variables, modules and count are already resolved. Conftest example that blocks deletion of data stores:

# policy/terraform.rego
package main

import rego.v1

protected_types := {"aws_db_instance", "aws_rds_cluster", "aws_dynamodb_table", "aws_s3_bucket"}

deny contains msg if {
some rc in input.resource_changes
protected_types[rc.type]
"delete" in rc.change.actions
msg := sprintf("%s would be deleted or replaced; destructive changes to data stores require a manual override", [rc.address])
}

deny contains msg if {
some rc in input.resource_changes
rc.type == "aws_s3_bucket"
"create" in rc.change.actions
not rc.change.after.tags.owner
msg := sprintf("%s is missing the required 'owner' tag", [rc.address])
}
terraform plan -out=tfplan
terraform show -json tfplan > tfplan.json
conftest test tfplan.json --policy policy/
checkov -d . --quiet

A replacement appears in plan JSON as actions ["delete", "create"] or ["create", "delete"], so the first rule catches replacements too. Start policies in warn mode, fix existing violations, then enforce; and provide a documented, audited override path, or teams will route around the checks.

Q39: What is immutable infrastructure, and how does it change operations?​

Immutable infrastructure means servers and containers are never modified after deployment. To change anything (OS patch, config, app version) you build a new image and replace the instances. The opposite is mutable infrastructure, where long-lived servers are patched and reconfigured in place (SSH, configuration management runs).

Benefits:

  • No configuration drift: every instance is identical to the tested image.
  • Reliable rollback: redeploy the previous image.
  • Reproducibility and security: images are built by a pipeline, scanned and signed; no manual changes on hosts, so SSH access can often be removed.

How it is implemented:

  • Containers: build a new image per change, deploy by digest.
  • VMs: bake machine images (for example with Packer), roll them out through autoscaling group instance refresh or blue/green.
  • IaC: prefer replacing resources over patching, using create_before_destroy to avoid downtime.
resource "aws_launch_template" "app" {
name_prefix = "app-"
image_id = var.ami_id # new AMI per release, built by the pipeline
instance_type = "t3.medium"
}

# A new AMI creates a new launch template version; the instance refresh
# then replaces instances in batches instead of patching them in place.

resource "aws_autoscaling_group" "app" {
name = "app"
min_size = 3
max_size = 6
vpc_zone_identifier = var.private_subnet_ids

launch_template {
id = aws_launch_template.app.id
version = aws_launch_template.app.latest_version
}

instance_refresh {
strategy = "Rolling"
preferences {
min_healthy_percentage = 90
}
}
}

Trade-offs: you need a fast image pipeline, state must live outside instances (databases, object storage), and debugging moves from "SSH in and look" to logs, metrics and reproducing with the same image.

Q40: What do Terraform lifecycle arguments do, and what are their pitfalls?​

ArgumentEffectPitfall
create_before_destroyWhen a change forces replacement, create the new object before destroying the oldFails if the resource needs a unique name; use name_prefix or generated names
prevent_destroyAny plan that would destroy the resource errors outOnly protects while the block exists in config; deleting the resource block removes the protection too
ignore_changesIgnore differences in listed attributes after creationHides real drift; ignore_changes = all turns Terraform into a create-only tool for that resource
replace_triggered_byReplace this resource when a referenced resource or attribute changesEasy to cause unexpected replacements
precondition / postconditionAssert assumptions during plan/apply with a custom error messageOveruse makes plans fail for reasons unrelated to the change
resource "aws_ecs_service" "orders" {
name = "orders-api"
cluster = var.cluster_arn
task_definition = var.task_definition_arn
desired_count = 3

lifecycle {
# Autoscaling changes desired_count at runtime; do not fight it on every apply
ignore_changes = [desired_count]
}
}

resource "aws_dynamodb_table" "orders" {
name = "orders"
billing_mode = "PAY_PER_REQUEST"
hash_key = "pk"

attribute {
name = "pk"
type = "S"
}

point_in_time_recovery {
enabled = true
}

lifecycle {
prevent_destroy = true
}
}

Because prevent_destroy disappears with the block, combine it with protections that live outside Terraform: provider-level deletion protection (deletion_protection on RDS, deletion_protection_enabled on DynamoDB), backups and point-in-time recovery, and plan-time policy checks.


Senior and Principal Serverless and IaC Scenarios​

Q41: Terraform state lock stuck after a CI runner was killed​

Level: Senior

Scenario: A production terraform apply in CI is cancelled midway when the runner is preempted. Every subsequent plan fails with "Error acquiring the state lock", showing a lock ID, the old run as the holder, and a timestamp from an hour ago. A fix for an ongoing incident is waiting on this pipeline.

Impact: All infrastructure changes for that state are blocked. Worse, the interrupted apply may have created or modified some resources without recording all of them in state.

Root Cause:

  • The process holding the lock died without releasing it (locks are released by the client at the end of the operation).
  • The apply was interrupted, so state may be behind reality.
  • Long-running applies on preemptible runners with no protection against cancellation.

Fix:

  1. Confirm the holder is really dead: check the CI run, and that no other job or person is running against this state. Force-unlocking a live operation lets two writers corrupt state.
  2. Back up current state, then force-unlock with the lock ID from the error.
  3. Run a plan (not apply) and review it carefully: resources created by the interrupted run but not recorded in state will show up as conflicts on create ("already exists"); import them rather than deleting them.
  4. Prevent recurrence: serialize runs with a CI concurrency group, run applies on non-preemptible runners, give the apply job a timeout longer than the slowest apply, and avoid cancelling apply jobs.
cd infra/prod
terraform init -input=false

# 1. Back up the current state before touching anything
terraform state pull > "backup-$(date +%Y%m%d-%H%M%S).tfstate"

# 2. Release the orphaned lock (ID from the error message)
terraform force-unlock -force 6f1c2a9e-3b7d-4c55-9a10-2f8e7d4b1c03

# 3. See how reality differs from state after the interrupted apply
terraform plan -input=false

Workflow Diagram:

apply running --(runner killed)--> lock never released
|
v
next plan: "Error acquiring the state lock"
|
v
Verify holder is dead (CI run, team) --- still running? --> wait, do NOT unlock
|
v dead
state pull (backup) -> force-unlock LOCK_ID -> plan
|
v
Reconcile partial apply: import orphaned resources, re-apply
|
v
Prevent: concurrency group, stable runners, no cancelling applies

Interviewer Notes: The key signal is caution: verify before force-unlocking, back up state, and understand that an interrupted apply can leave real resources untracked. Candidates should explain what the lock protects against and how pipelines should serialize Terraform runs.

Red Flags: Force-unlocking immediately without checking for a live holder; deleting the lock entry or object directly in the backend as a first move; running apply right after unlocking without reviewing a plan.


Q42: The Terraform state file was corrupted or overwritten​

Level: Senior

Scenario: An engineer runs an old Terraform version locally against the production backend, hits an error, and "fixes" it with terraform state push of a file from their laptop. The next CI plan wants to create 140 resources that already exist, and destroy nothing, because the pushed state is from months ago.

Impact: Any apply now would fail with "already exists" errors or, for resources without unique names, create duplicates. Teams are blocked; there is risk of someone "fixing" it by deleting resources.

Root Cause:

  • Humans with write access to production state and the ability to state push.
  • No version pinning (required_version), so mixed Terraform versions touch the same state.
  • Reliance on a single copy of state with no tested restore procedure.

Fix:

  1. Freeze: stop all pipelines and announce a change freeze for that state.
  2. Restore a previous version from the versioned S3 bucket (choose the last version written by CI before the incident).
  3. Run a plan and expect "no changes" or only the small, known diff since that version; reconcile remaining differences with import or apply.
  4. Prevent: humans get read-only state access; only the CI role writes production state. Pin required_version and providers; enable bucket versioning and replication; document and rehearse the restore.
BUCKET=acme-tfstate-prod
KEY=network/terraform.tfstate

# List historical versions of the state object
aws s3api list-object-versions --bucket "$BUCKET" --prefix "$KEY" \
--query 'Versions[].{Id:VersionId,Modified:LastModified,Size:Size}' --output table

# Download the chosen good version
aws s3api get-object --bucket "$BUCKET" --key "$KEY" \
--version-id 3HL4kqtJlcpXroDTDmJ.rmSpXd3dIbrHY restored.tfstate

# Inspect it, then restore it as the current version
terraform show restored.tfstate | head -50
aws s3 cp restored.tfstate "s3://${BUCKET}/${KEY}"

terraform init -input=false
terraform plan -input=false
terraform {
required_version = "~> 1.10.0"

required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 5.80"
}
}
}

Copying the object back directly (rather than terraform state push) is deliberate here: state push refuses states with an older serial or a different lineage unless forced, which is a useful safety check in normal operation. Whichever method you use, make sure no process is running against the state while restoring. If the backend still uses DynamoDB locking, Terraform also stores a checksum of the state in that table; after copying an object back manually, init or plan can fail with a checksum mismatch until the stored digest is updated to match the restored file (the error message shows the expected value).

Workflow Diagram:

Bad state pushed -> plan wants to create 140 existing resources
|
v
FREEZE all pipelines for this state
|
v
S3 versioning: find last good version (written by CI)
|
v
Restore -> plan -> expect ~no changes -> reconcile small diff (import/apply)
|
v
Prevent: CI-only write access, pinned versions, versioned + replicated bucket, rehearsed restore

Interviewer Notes: Look for: stop the bleeding first (freeze), restore from versioned storage, validate with a plan, then fix the access model. Senior candidates also mention lineage and serial as Terraform's own protection against this mistake, and that "recreate everything from scratch" is rarely acceptable for production.

Red Flags: Applying while the plan shows mass creation; rebuilding state by mass-importing as the first option when backups exist; no bucket versioning; everyone having write access to production state.


Q43: A manual console change caused drift and the next apply reverted it​

Level: Senior

Scenario: During an incident, an engineer adds an inbound rule to a production security group in the AWS console so a new internal service can reach the database. Two weeks later, an unrelated Terraform change is applied; the plan included "~ update in-place" for the security group, nobody noticed among 30 other changes, and the rule was removed. The internal service loses database access and fails.

Impact: Outage caused by a routine apply, confusion about root cause ("we only changed tags"), and declining trust in IaC.

Root Cause:

  • Manual change in a Terraform-managed resource without a follow-up commit.
  • No drift detection, so the difference was discovered only by the next apply.
  • Large plans reviewed superficially; nothing highlighted the unexpected change.

Fix:

  1. Restore service: re-add the rule through Terraform (code change, plan, apply), so it is durable.
  2. Detect drift continuously: a scheduled pipeline runs terraform plan -detailed-exitcode (exit code 2 means changes) and alerts the owning team.
  3. Use -refresh-only to accept intentional out-of-band changes into state, then codify them.
  4. Make console write access to Terraform-managed production accounts exceptional (read-only by default, audited break-glass roles), and alert on changes to security-sensitive resources made outside the pipeline (for example, CloudTrail events from non-CI principals).
  5. Keep plans small (one change per PR, smaller states) so unexpected diffs stand out.
name: drift-detection
on:
schedule:
- cron: "0 6 * * *"

permissions:
contents: read
id-token: write

jobs:
drift:
runs-on: ubuntu-latest
defaults:
run:
working-directory: infra/prod
steps:
- uses: actions/checkout@v4
- uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::123456789012:role/terraform-plan-prod
aws-region: us-east-1
- run: terraform init -input=false
- name: Plan and detect drift
run: |
set +e
terraform plan -input=false -lock=false -detailed-exitcode -no-color > plan.txt
code=$?
set -e
if [ "$code" -eq 2 ]; then
echo "Drift detected in infra/prod"
cat plan.txt
exit 1
fi
exit "$code"

Workflow Diagram:

Console change (rule added) Terraform code (rule absent)
\ /
v v
Reality != Code (drift, unnoticed)
|
2 weeks later: unrelated apply
|
v
Rule removed -> outage

With drift detection:
nightly plan -detailed-exitcode -> exit 2 -> alert owner
-> codify the change (PR) or revert it intentionally

Interviewer Notes: Good answers treat drift as a process problem: changes outside the pipeline must be either prevented, detected or reconciled quickly. Candidates should know -detailed-exitcode, -refresh-only, and the idea of read-only console access with break-glass. Principal candidates discuss reducing blast radius with smaller states and making large plans reviewable.

Red Flags: "Never use the console" with no break-glass plan for incidents; ignore_changes on the security group rules to stop the diff; no ownership of drift alerts.


Q44: terraform apply destroyed and recreated a production database​

Level: Principal

Scenario: A PR renames aws_db_instance.db to aws_db_instance.orders for readability and, in the same PR, changes the instance identifier to match a new naming convention. The plan says 1 to add, 0 to change, 1 to destroy; the reviewer reads it as a rename. Apply deletes the production database, and the final snapshot is skipped because skip_final_snapshot = true was copied from a dev example.

Impact: Production outage and data loss back to the last automated backup; recovery takes hours; potential regulatory reporting.

Root Cause:

  • Changing a resource address without a moved block plans a destroy and create.
  • Changing a force-new attribute (identifier) requires replacement even with the same address.
  • No guardrails: no prevent_destroy, deletion protection off, final snapshot skipped, no policy check on destructive plans.
  • Plan review relied on humans reading summaries.

Fix:

  1. Recover: restore from automated backups or point-in-time recovery to a new instance, then point the configuration at it (import) instead of letting Terraform create a fresh empty one.
  2. Use moved blocks for renames; never combine refactors with force-new attribute changes.
  3. Layered protections for data stores: prevent_destroy in config, deletion_protection at the AWS level, final snapshots and backup retention, AWS Backup with a separate vault for critical data.
  4. Policy check in CI that fails any plan deleting or replacing a data store unless an explicit, audited override is present.
  5. Make the plan output impossible to skim: post a summary to the PR that highlights deletions and replacements in red.
moved {
from = aws_db_instance.db
to = aws_db_instance.orders
}

resource "aws_db_instance" "orders" {
identifier = "orders-prod" # unchanged: changing it forces replacement
engine = "postgres"
instance_class = "db.r6g.large"
allocated_storage = 200
username = "orders_admin"
manage_master_user_password = true
multi_az = true

backup_retention_period = 14
deletion_protection = true
skip_final_snapshot = false
final_snapshot_identifier = "orders-prod-final"

lifecycle {
prevent_destroy = true
}
}
# CI gate: count deletions/replacements in the plan and fail on protected types
terraform show -json tfplan > tfplan.json
jq -r '.resource_changes[]
| select(.change.actions | index("delete"))
| "\(.address) \(.change.actions | join(","))"' tfplan.json
conftest test tfplan.json --policy policy/

Workflow Diagram:

PR: rename address + change identifier
|
v
Plan: -/+ destroy and create aws_db_instance (looked like a rename)
|
v
Apply -> DB deleted, no final snapshot -> data loss

Guardrails, each able to stop it independently:
moved block ........ rename becomes a no-op
prevent_destroy .... plan errors out
policy check ....... CI fails on delete of protected type
deletion_protection AWS API refuses the delete
final snapshot ..... data recoverable even if all else fails

Interviewer Notes: Principal candidates think in layers (defense in depth) and explain which layer catches which failure: prevent_destroy disappears if the block is deleted, deletion protection lives in AWS, policy checks work regardless of config. They also address the human factor: plan review needs tooling that surfaces destructive actions, and the recovery plan must be rehearsed.

Red Flags: "Reviewers should read plans more carefully" as the only fix; not knowing what forces replacement; relying solely on prevent_destroy; dev defaults (skip_final_snapshot = true) in production modules.


Q45: A recursive trigger produced a runaway serverless bill​

Level: Senior

Scenario: A function is triggered by s3:ObjectCreated:* on an uploads bucket. It generates a thumbnail and writes it back to the same bucket under thumbnails/. That write triggers the function again, which creates a thumbnail of the thumbnail, and so on. Lambda scales to the account concurrency limit; within hours there are millions of invocations, millions of S3 PUTs, and large log volume. Other functions in the account are throttled because the loop consumes all concurrency.

Impact: A large, unexpected bill (Lambda, S3 requests, CloudWatch Logs), throttling of unrelated production functions sharing the account concurrency, and millions of junk objects to clean up.

Root Cause:

  • The function writes to the same resource that triggers it, with no prefix or suffix filter separating inputs from outputs.
  • No reserved concurrency limiting the function's blast radius.
  • No budget or anomaly alerts; nobody noticed until the bill or throttling did.

Fix:

  1. Stop the loop now: set the function's reserved concurrency to 0 (throttles all invocations), or remove the event notification.
  2. Break the cycle by design: write outputs to a separate bucket, or at minimum filter the trigger on an input prefix the function never writes to.
  3. Cap blast radius with reserved concurrency on each function.
  4. Alert early: AWS Budgets and Cost Anomaly Detection, plus CloudWatch alarms on invocation rate and concurrency.
  5. Lambda's recursive loop detection can stop some loops automatically, but only for the event sources it supports; check the current list for your trigger and treat it as a safety net, not the design.
# Emergency stop: throttle every invocation of the function
aws lambda put-function-concurrency --function-name thumbnailer --reserved-concurrent-executions 0
resource "aws_s3_bucket" "uploads" {
bucket = "acme-uploads"
}

resource "aws_s3_bucket" "thumbnails" {
bucket = "acme-thumbnails" # outputs go to a different bucket
}

resource "aws_lambda_function" "thumbnailer" {
function_name = "thumbnailer"
role = aws_iam_role.thumbnailer.arn # execution role defined elsewhere
runtime = "python3.12"
handler = "app.handler"
filename = "build/thumbnailer.zip"
timeout = 30
memory_size = 1024
reserved_concurrent_executions = 50 # blast-radius cap

environment {
variables = {
OUTPUT_BUCKET = aws_s3_bucket.thumbnails.bucket
}
}
}

resource "aws_lambda_permission" "s3_invoke" {
statement_id = "AllowS3Invoke"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.thumbnailer.function_name
principal = "s3.amazonaws.com"
source_arn = aws_s3_bucket.uploads.arn
}

resource "aws_s3_bucket_notification" "uploads" {
bucket = aws_s3_bucket.uploads.id

lambda_function {
lambda_function_arn = aws_lambda_function.thumbnailer.arn
events = ["s3:ObjectCreated:*"]
filter_prefix = "incoming/"
}

depends_on = [aws_lambda_permission.s3_invoke]
}

resource "aws_cloudwatch_metric_alarm" "thumbnailer_invocations" {
alarm_name = "thumbnailer-invocation-spike"
namespace = "AWS/Lambda"
metric_name = "Invocations"
dimensions = { FunctionName = aws_lambda_function.thumbnailer.function_name }
statistic = "Sum"
period = 300
evaluation_periods = 1
threshold = 50000
comparison_operator = "GreaterThanThreshold"
alarm_actions = [var.oncall_sns_topic_arn]
}

Workflow Diagram:

Loop:
upload -> S3 (acme-uploads) -> Lambda -> PUT thumbnails/x.png into SAME bucket
^ |
+------------------- ObjectCreated ----------------+
=> invocations grow until concurrency limit, bill grows every minute

Fixed:
upload -> acme-uploads/incoming/ -> Lambda (reserved concurrency 50)
|
v
acme-thumbnails (no trigger)
Alarms: invocation spike + budget/anomaly alerts -> on-call

Interviewer Notes: Candidates should immediately know how to stop the bleeding (reserved concurrency 0 or remove the trigger), explain why the loop happened, and design it out (separate buckets, prefix filters). Good answers include cost guardrails (budgets, anomaly detection, alarms) and the shared account concurrency pool, which turns a cost problem into an availability problem for other functions.

Red Flags: "Add a check in the code to skip thumbnails" as the only fix (the function still gets invoked and billed for every event); no emergency stop plan; no concurrency limits; finding out from the monthly bill.


Q46: Function scale-out exhausted the relational database's connections​

Level: Senior

Scenario: A marketing campaign sends a traffic spike to an API backed by Lambda and a PostgreSQL RDS instance. Lambda scales from 20 to 800 concurrent executions within minutes. Each execution environment opens its own database connection; PostgreSQL hits max_connections, new connections fail with "too many clients", and the API returns 500s. Retries from clients make it worse.

Impact: Full API outage at the most valuable moment; the database is healthy on CPU but unusable; other services sharing the database fail too.

Root Cause:

  • Function concurrency scales far faster and higher than the database's connection capacity.
  • One connection per execution environment, opened outside a pool, and kept while idle.
  • No backpressure: nothing limits how much concurrency reaches the database.

Fix:

  1. Connection pooling outside the function: RDS Proxy (or PgBouncer) multiplexes many client connections onto a bounded pool of database connections.
  2. Cap concurrency to what the database can serve: reserved concurrency on synchronous functions; for queue consumers, the SQS event source mapping's maximum concurrency setting.
  3. Buffer writes through a queue so spikes become backlog instead of errors, where the API semantics allow asynchronous processing (return 202).
  4. Reuse one connection per execution environment (create it outside the handler), set short connect and statement timeouts, and use client-side retries with jittered backoff.
  5. Load test with realistic concurrency, and consider whether a store with request-based scaling (for example DynamoDB) fits the hottest access paths better.
resource "aws_db_proxy" "orders" {
name = "orders-proxy"
engine_family = "POSTGRESQL"
role_arn = aws_iam_role.rds_proxy.arn # role allowed to read the DB secret, defined elsewhere
vpc_subnet_ids = var.private_subnet_ids
require_tls = true

auth {
auth_scheme = "SECRETS"
iam_auth = "REQUIRED"
secret_arn = var.db_secret_arn
}
}

resource "aws_db_proxy_default_target_group" "orders" {
db_proxy_name = aws_db_proxy.orders.name

connection_pool_config {
max_connections_percent = 70 # leave headroom for admin and other clients
}
}

resource "aws_db_proxy_target" "orders" {
db_proxy_name = aws_db_proxy.orders.name
target_group_name = aws_db_proxy_default_target_group.orders.name
db_instance_identifier = aws_db_instance.orders.identifier
}

# Queue consumer: never more than 20 concurrent executions hit the database
resource "aws_lambda_event_source_mapping" "orders_writer" {
event_source_arn = aws_sqs_queue.orders.arn
function_name = aws_lambda_function.orders_writer.arn
batch_size = 10

scaling_config {
maximum_concurrency = 20
}
}

Workflow Diagram:

Before:
spike -> 800 Lambda envs -> 800 DB connections -> max_connections hit -> 500s

After:
spike -> API Lambda (reserved concurrency) -> RDS Proxy (pool, 70% of max) -> PostgreSQL
-> async writes: SQS -> consumer Lambda (max concurrency 20) -> RDS Proxy -> PostgreSQL
Backlog grows in SQS instead of errors at the database.

Interviewer Notes: Candidates should identify the mismatch between elastic compute and a fixed-capacity dependency, and propose pooling plus concurrency limits plus buffering. Strong answers discuss that reserved concurrency also throttles callers (so the API must handle 429s gracefully) and that RDS Proxy pinning (session state such as prepared statements or SET commands) can reduce multiplexing.

Red Flags: "Increase max_connections" as the only fix (memory per connection limits this); opening a new connection per invocation; no load testing at realistic concurrency.


Q47: One poison message kept a whole SQS batch retrying​

Level: Senior

Scenario: A Lambda function consumes an SQS queue with batch size 10. One message has a malformed payload that always raises an exception. Every time, the whole batch fails, so the nine valid messages are also returned to the queue and reprocessed: emails are sent repeatedly and payments are re-attempted. Eventually all ten messages hit maxReceiveCount and land in the DLQ together, valid ones included.

Impact: Duplicate side effects (repeat emails, duplicate charge attempts), growing latency for the queue, and a DLQ full of good messages that someone must sort through.

Root Cause:

  • By default, if the function returns an error, the entire batch is considered failed and becomes visible again.
  • Handler does not isolate failures per message and is not idempotent.
  • No distinction between permanent errors (bad payload) and transient errors (downstream timeout).

Fix:

  1. Enable partial batch responses (ReportBatchItemFailures) and return only the failed message IDs; successful messages are deleted.
  2. Validate messages first; send permanently invalid messages straight to a DLQ or quarantine store instead of retrying them.
  3. Make side effects idempotent (idempotency keys) so retries are harmless.
  4. Alarm on DLQ depth and on the age of the oldest message; build a redrive process after fixes.
resource "aws_lambda_event_source_mapping" "notifications" {
event_source_arn = aws_sqs_queue.notifications.arn
function_name = aws_lambda_function.notifications.arn
batch_size = 10
function_response_types = ["ReportBatchItemFailures"]
}
import json
import logging

logger = logging.getLogger()
logger.setLevel(logging.INFO)


class PermanentError(Exception):
"""The message can never succeed (bad payload); do not retry it."""


def handler(event, context):
failures = []
for record in event["Records"]:
try:
process(json.loads(record["body"]))
except (PermanentError, json.JSONDecodeError):
# Quarantine instead of retrying forever; here we just log and drop it
logger.error("Permanent failure, dropping message %s", record["messageId"])
except Exception:
logger.exception("Transient failure for message %s", record["messageId"])
failures.append({"itemIdentifier": record["messageId"]})
return {"batchItemFailures": failures}


def process(message: dict) -> None:
if "recipient" not in message:
raise PermanentError("missing recipient")
send_notification(message) # idempotent by message["notification_id"]


def send_notification(message: dict) -> None:
...

In production, "drop" should mean "write to a quarantine queue or table with the error", so nothing is silently lost.

Workflow Diagram:

Before:
batch [m1..m10], m7 malformed -> handler raises -> all 10 back on queue
-> m1..m6, m8..m10 reprocessed (duplicates) -> eventually all 10 in DLQ

After (ReportBatchItemFailures):
batch [m1..m10]
m1..m6, m8..m10 OK -> deleted
m7 permanent error -> quarantined, not retried
transient failures only -> returned in batchItemFailures -> retried -> DLQ after N

Interviewer Notes: Candidates should know the default all-or-nothing batch semantics, partial batch responses, the difference between permanent and transient errors, and that idempotency is still required (partial responses reduce duplicates but do not eliminate at-least-once delivery). For FIFO queues, strong candidates note that processing should stop at the first failure in a message group to preserve ordering.

Red Flags: Setting batch size to 1 as the fix (throughput and cost suffer); catching and swallowing all exceptions (data silently lost); no DLQ monitoring.


Q48: One giant Terraform state makes every change slow and risky​

Level: Principal

Scenario: A company manages its entire AWS estate (networking, IAM, 60 services, databases, DNS) in one Terraform root module with about 4,000 resources. A plan takes 25 minutes, applies frequently hit API rate limits, any team's change locks everyone else out, and a typo in one service's module once produced a plan touching networking.

Impact: Slow delivery, teams bypassing Terraform for "quick" changes (causing drift), huge blast radius for every apply, and API throttling incidents.

Root Cause: A single state as the unit of locking, planning, permissions and failure. Everything refreshes on every plan, every apply needs permissions for everything, and every reviewer has to reason about everything.

Fix: Split state along ownership and rate of change:

  1. Define layers with clear dependencies: foundation (accounts, networking), shared platform (clusters, DNS zones), data stores, and per-service stacks. Lower layers change rarely; services change often.
  2. Pass values between states through explicit outputs: data sources, SSM parameters, or terraform_remote_state with read-only access.
  3. Migrate incrementally, one component at a time, without destroying anything: removed blocks with destroy = false in the old configuration, import blocks in the new one, and verify both plans show no changes.
  4. Give each state its own CI pipeline, role with least privilege, and owners (CODEOWNERS).
  5. Use orchestration tools (Terragrunt, Terraform Stacks, Spacelift or similar) if managing many states becomes the new bottleneck.
# Old monolith: stop managing the orders table without deleting it
removed {
from = aws_dynamodb_table.orders

lifecycle {
destroy = false
}
}
# New stack: infra/services/orders
import {
to = aws_dynamodb_table.orders
id = "orders"
}

resource "aws_dynamodb_table" "orders" {
name = "orders"
billing_mode = "PAY_PER_REQUEST"
hash_key = "pk"

attribute {
name = "pk"
type = "S"
}
}

# Consume networking outputs published by the foundation layer
data "aws_ssm_parameter" "private_subnets" {
name = "/platform/network/private-subnet-ids"
}

locals {
private_subnet_ids = split(",", data.aws_ssm_parameter.private_subnets.value)
}

Workflow Diagram:

Before:
[ one state: network + IAM + 60 services + DBs + DNS ] -> 25 min plan, global lock

After:
foundation/network (rare changes, platform team)
| outputs via SSM
v
platform/eks, dns (platform team)
|
v
data/orders-db (service team, protected)
|
v
services/orders (service team, many changes per day, 2 min plan)

Migration per component: removed{destroy=false} in old -> import{} in new -> both plans "No changes"

Interviewer Notes: Principal candidates explain the criteria for splitting (ownership, change frequency, blast radius, permissions), how cross-state dependencies are expressed, and a zero-downtime migration path. They also mention the trade-off: more states mean more coordination and dependency ordering, which needs conventions and tooling.

Red Flags: Splitting by resource type ("all S3 buckets in one state") rather than ownership; migrating by deleting and recreating resources; hand-editing state files to move resources; cyclic dependencies between states.


Q49: An unpinned provider upgrade broke a production apply​

Level: Senior

Scenario: A Terraform configuration declares the AWS provider with version = ">= 4.0" and the team never committed .terraform.lock.hcl (it is in .gitignore). A new major provider version is released. The next CI run downloads it, and the plan suddenly shows changes on resources nobody touched, plus errors for removed arguments. The pipeline is blocked on a day with an urgent infrastructure fix.

Impact: Blocked deployments, risk of applying unintended changes caused by changed provider defaults, and different results on laptops and in CI.

Root Cause:

  • Open-ended version constraint allows major upgrades with breaking changes.
  • No dependency lock file, so every init resolves the newest matching provider.
  • Upgrades happen implicitly instead of as reviewed changes.

Fix:

  1. Unblock: commit a lock file pinned to the last working provider version (or tighten the constraint) and re-run.
  2. Use pessimistic constraints (~> 5.80 allows 5.x from 5.80 upward but not 6.0) and commit .terraform.lock.hcl.
  3. Generate lock file hashes for every platform that runs Terraform (CI Linux, developer macOS), so init does not fail or rewrite the file on different machines.
  4. Upgrade providers deliberately: Renovate or Dependabot PR, read the upgrade guide, review the plan in non-production first.
terraform {
required_version = "~> 1.10.0"

required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 5.80"
}
}
}
# Record checksums for all platforms used by the team and CI, then commit the lock file
terraform providers lock -platform=linux_amd64 -platform=darwin_arm64 -platform=darwin_amd64
git add .terraform.lock.hcl
git commit -m "chore(infra): lock AWS provider versions"

# Deliberate upgrade later, in its own PR
terraform init -upgrade
terraform plan

Workflow Diagram:

Before:
version ">= 4.0" + no lock file
-> new major provider released -> CI init picks it -> unexpected plan diff/errors

After:
version "~> 5.80" + committed .terraform.lock.hcl
-> CI always uses locked version
-> upgrades arrive as PRs (Renovate) -> plan in dev -> merge -> promote

Interviewer Notes: Candidates should distinguish version constraints (what is allowed) from the lock file (what was selected), know that the lock file belongs in version control for root modules, and treat provider upgrades like any dependency upgrade. Reusable modules should declare minimum provider versions while root modules pin exact selections via the lock file.

Red Flags: Ignoring the lock file in git; pinning exact versions in shared modules (causes conflicts for consumers); upgrading providers and making resource changes in the same PR.


Q50: An event-driven order flow became impossible to reason about​

Level: Principal

Scenario: An e-commerce platform built its order process with pure choreography across 12 functions and 9 event types on a shared event bus. Over two years, teams added consumers that emit further events. Now an order sometimes ends up "paid but never shipped", support cannot tell where an order is stuck, a change to one event's payload broke two unknown consumers, and nobody can draw the flow from memory.

Impact: Lost and stuck orders, long investigations, fear of changing event schemas, and increasing incident rate as the system grows.

Root Cause:

  • Choreography used for a business process that needs ordering, deadlines and compensation.
  • No single place that knows the state of an order.
  • Event schemas without ownership, versioning or a catalog; consumers not discoverable.
  • Missing correlation IDs and distributed tracing across asynchronous hops.

Fix:

  1. Orchestrate the core process: model "order placed to shipped" as an explicit workflow (Step Functions or similar) with timeouts, retries and compensation, so each order's state is visible and queryable.
  2. Keep choreography at the edges: the workflow emits a few well-defined domain events (OrderPaid, OrderShipped) for independent consumers such as email, analytics and loyalty.
  3. Govern events as APIs: schema registry or catalog, an owner per event, backward-compatible evolution (add optional fields, never remove or repurpose), explicit versioning for breaking changes, and contract tests between producers and consumers.
  4. Trace everything: propagate a correlation ID (order ID plus trace context) on every event; use OpenTelemetry or X-Ray to see the full path.
  5. Detect stuck work: alarms on workflow executions exceeding expected duration and on orders that remain in a state longer than the SLA.
{
"version": "0",
"source": "com.acme.orders",
"detail-type": "OrderPaid",
"detail": {
"schema_version": "2",
"order_id": "ord_8f2c1a",
"correlation_id": "ord_8f2c1a",
"traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
"amount_cents": 4599,
"currency": "EUR"
}
}
# Emit a versioned domain event with correlation data via EventBridge
import json

import boto3

events = boto3.client("events")


def publish_order_paid(order_id: str, amount_cents: int, currency: str, traceparent: str) -> None:
events.put_events(
Entries=[
{
"Source": "com.acme.orders",
"DetailType": "OrderPaid",
"EventBusName": "orders",
"Detail": json.dumps(
{
"schema_version": "2",
"order_id": order_id,
"correlation_id": order_id,
"traceparent": traceparent,
"amount_cents": amount_cents,
"currency": currency,
}
),
}
]
)

put_events can partially fail: production code should check FailedEntryCount in the response and retry the failed entries.

Workflow Diagram:

Before (implicit chain, nobody owns the whole flow):
OrderPlaced -> fn A -> PaymentRequested -> fn B -> PaymentCaptured -> fn C -> StockReserved
\-> fn D -> FraudChecked -> fn E -> ... (9 event types, 12 functions)

After:
+----------------- Order workflow (orchestrated) ----------------+
OrderPlaced ---> | fraud check -> reserve stock -> charge -> create shipment |
| retries, timeouts, compensation, visible state per order |
+-------------------------------+---------------------------------+
| emits few, versioned events
v
OrderPaid / OrderShipped on event bus (choreography)
| | |
email analytics loyalty

Interviewer Notes: Principal-level answers recognize the trade-off between autonomy (choreography) and visibility and control (orchestration), and draw the boundary deliberately. They treat events as public contracts with owners and versioning, insist on observability across asynchronous boundaries, and plan an incremental migration (wrap the core flow in a workflow while keeping existing consumers working) rather than a rewrite.

Red Flags: "Just add more logging"; replacing every event with synchronous calls; breaking schema changes without versioning; no way to find all consumers of an event.