Systems and Infrastructure Experience Questions

#!/usr/bin/env bash
set -euo pipefail

# CONFIG
LOG_DIR="/var/log/myapp"
ROTATE_DIR="/var/log/myapp/rotated"
S3_BUCKET="s3://myapp-logs"
RETENTION_DAYS=30
AWS_PROFILE="default"   # or unset to use env creds
RETRIES=3
BACKOFF=2

timestamp=$(date -u +"%Y-%m-%d")
lockfile="/var/lock/logrotate_myapp.lock"

# simple lock for idempotency
exec 9>"$lockfile"
if ! flock -n 9; then
  echo "Another instance is running; exiting."
  exit 0
fi

mkdir -p "$ROTATE_DIR"

# rotate: move current logs to timestamped file and recreate empty log
for f in "$LOG_DIR"/*.log; do
  [ -e "$f" ] || continue
  base=$(basename "$f")
  rotated="$ROTATE_DIR/${base}.${timestamp}"
  if [ ! -e "$rotated.gz" ]; then
    mv "$f" "$rotated"
    # recreate original file with same permissions
    touch "$f"
    chown --reference="$rotated" "$f" || true
    chmod --reference="$rotated" "$f" || true
  else
    echo "Rotated archive already exists for $base; skipping move."
  fi
done

# compress any uncompressed rotated files
find "$ROTATE_DIR" -type f ! -name "*.gz" -name "*.log.*" -print0 | while IFS= read -r -d '' file; do
  gzip -9 "$file"
done

# upload to S3 with retries and idempotency (use aws s3 cp which overwrites deterministically)
for obj in "$ROTATE_DIR"/*.gz; do
  [ -e "$obj" ] || continue
  key="$(basename "$obj")"
  attempt=0
  while [ $attempt -lt $RETRIES ]; do
    if aws s3 cp "$obj" "$S3_BUCKET/$key" ${AWS_PROFILE:+--profile $AWS_PROFILE}; then
      echo "Uploaded $key"
      break
    else
      attempt=$((attempt+1))
      sleep $((BACKOFF**attempt))
      echo "Retry $attempt for $key"
    fi
  done
  if [ $attempt -ge $RETRIES ]; then
    echo "Failed to upload $key after $RETRIES attempts" >&2
    exit 2
  fi
done

# enforce retention in S3: delete objects older than RETENTION_DAYS
cutoff=$(date -u -d "$RETENTION_DAYS days ago" +"%Y-%m-%dT%H:%M:%SZ")
# list objects and delete those with LastModified < cutoff
aws s3api list-objects-v2 --bucket "${S3_BUCKET#s3://}" --query "Contents[?LastModified<='${cutoff}'].{Key:Key}" --output json ${AWS_PROFILE:+--profile $AWS_PROFILE} |
  jq -r '.[].Key' | while read -r key; do
    echo "Deleting old object: $key"
    aws s3 rm "$S3_BUCKET/$key" ${AWS_PROFILE:+--profile $AWS_PROFILE}
  done

# verification: ensure today's files exist in S3
missing=0
for obj in "$ROTATE_DIR"/*"$timestamp".gz; do
  [ -e "$obj" ] || continue
  key=$(basename "$obj")
  if ! aws s3 ls "$S3_BUCKET/$key" ${AWS_PROFILE:+--profile $AWS_PROFILE} >/dev/null; then
    echo "Verification failed: $key not in S3" >&2
    missing=1
  fi
done

if [ $missing -ne 0 ]; then
  exit 3
fi

echo "Log rotation, upload, and retention enforcement completed successfully."