Deployment Monitoring and Verification

Custom Health Check Endpoint

<?php
namespace Vendor\HealthCheck\Controller\Health;

class Check implements \Magento\Framework\App\Action\HttpGetActionInterface
{
    private $resourceConnection;
    private $cacheFrontendPool;
    
    public function __construct(
        \Magento\Framework\App\ResourceConnection $resourceConnection,
        \Magento\Framework\Cache\FrontendPool $cacheFrontendPool
    ) {
        $this->resourceConnection = $resourceConnection;
        $this->cacheFrontendPool = $cacheFrontendPool;
    }
    
    public function execute()
    {
        $health = [
            'status' => 'healthy',
            'timestamp' => date('c'),
            'checks' => []
        ];
        
        // Database check
        try {
            $connection = $this->resourceConnection->getConnection();
            $connection->fetchOne('SELECT 1');
            $health['checks']['database'] = 'ok';
        } catch (\Exception $e) {
            $health['checks']['database'] = 'error: ' . $e->getMessage();
            $health['status'] = 'unhealthy';
        }
        
        // Cache check
        try {
            foreach ($this->cacheFrontendPool as $cache) {
                $cache->load('health_check_test');
            }
            $health['checks']['cache'] = 'ok';
        } catch (\Exception $e) {
            $health['checks']['cache'] = 'error: ' . $e->getMessage();
            $health['status'] = 'unhealthy';
        }
        
        $statusCode = $health['status'] === 'healthy' ? 200 : 503;
        
        $this->getResponse()->setStatusCode($statusCode);
        $this->getResponse()->setContent(json_encode($health));
        $this->getResponse()->setHeader('Content-Type', 'application/json');
        
        return $this->getResponse();
    }
}

Nginx Health Check Route

location /health {
    access_log off;
    fastcgi_pass unix:/var/run/php/php8.1-fpm.sock;
    fastcgi_param SCRIPT_FILENAME $document_root/health.php;
    include fastcgi_params;
}

Health Check Script

#!/bin/bash
# health-check.sh

BASE_URL="https://www.example.com"

check_endpoint() {
    local url=$1
    local expected=$2
    
    STATUS=$(curl -s -o /dev/null -w '%{http_code}' "$url")
    
    if [ $STATUS -eq $expected ]; then
        echo "OK: $url (HTTP $STATUS)"
        return 0
    else
        echo "FAIL: $url (HTTP $STATUS, expected $expected)"
        return 1
    fi
}

# Run checks
FAILURES=0

check_endpoint "$BASE_URL/" 200 || FAILURES=$((FAILURES + 1))
check_endpoint "$BASE_URL/health" 200 || FAILURES=$((FAILURES + 1))
check_endpoint "$BASE_URL/rest/V1/store/storeConfigs" 200 || FAILURES=$((FAILURES + 1))

if [ $FAILURES -gt 0 ]; then
    echo "Health check failed with $FAILURES failures"
    exit 1
fi

echo "All health checks passed"

Key Takeaway

Health checks verify database connectivity, cache availability, and critical endpoints. Implement custom endpoints and use curl-based scripts for verification.

Post-Deployment Smoke Tests

# GitHub Actions smoke tests
smoke-tests:
  needs: deploy-production
  runs-on: ubuntu-latest
  steps:
    - name: Wait for Deployment
      run: sleep 30
    
    - name: Homepage Check
      run: |
        STATUS=$(curl -s -o /dev/null -w '%{http_code}' https://www.example.com/)
        if [ $STATUS -ne 200 ]; then
          echo "Homepage returned $STATUS"
          exit 1
        fi
    
    - name: Product Page Check
      run: |
        STATUS=$(curl -s -o /dev/null -w '%{http_code}' https://www.example.com/catalog/product/view/id/1)
        if [ $STATUS -ne 200 ]; then
          echo "Product page returned $STATUS"
          exit 1
        fi
    
    - name: Category Page Check
      run: |
        STATUS=$(curl -s -o /dev/null -w '%{http_code}' https://www.example.com/catalog/category/view/id/3)
        if [ $STATUS -ne 200 ]; then
          echo "Category page returned $STATUS"
          exit 1
        fi
    
    - name: API Check
      run: |
        STATUS=$(curl -s -o /dev/null -w '%{http_code}' https://www.example.com/rest/V1/store/storeConfigs)
        if [ $STATUS -ne 200 ]; then
          echo "API returned $STATUS"
          exit 1
        fi
    
    - name: Cart Page Check
      run: |
        STATUS=$(curl -s -o /dev/null -w '%{http_code}' https://www.example.com/checkout/cart)
        if [ $STATUS -ne 200 ]; then
          echo "Cart page returned $STATUS"
          exit 1
        fi

Comprehensive Smoke Test Script

#!/bin/bash
# smoke-tests.sh

BASE_URL="https://www.example.com"
FAILED=0

check() {
    local name=$1
    local url=$2
    local expected=${3:-200}
    
    STATUS=$(curl -s -o /dev/null -w '%{http_code}' "$url")
    if [ $STATUS -eq $expected ]; then
        echo "✓ $name"
    else
        echo "✗ $name (HTTP $STATUS, expected $expected)"
        FAILED=$((FAILED + 1))
    fi
}

echo "Running smoke tests..."

check "Homepage" "$BASE_URL/"
check "Category Page" "$BASE_URL/catalog/category/view/id/3"
check "Product Page" "$BASE_URL/catalog/product/view/id/1"
check "CMS Page" "$BASE_URL/about-us"
check "API Store Config" "$BASE_URL/rest/V1/store/storeConfigs"
check "Health Check" "$BASE_URL/health"
check "Cart Page" "$BASE_URL/checkout/cart"
check "Search" "$BASE_URL/catalogsearch/result/?q=test"

echo ""
if [ $FAILED -gt 0 ]; then
    echo "FAILED: $FAILED smoke tests failed"
    exit 1
else
    echo "PASSED: All smoke tests passed"
fi

Key Takeaway

Smoke tests verify critical pages and API endpoints after deployment. Run homepage, product, category, cart, and API checks to ensure basic functionality.

Pre-Deployment Verification

#!/bin/bash
# pre-deploy-check.sh

echo "Pre-deployment checks..."

# Check staging is healthy
STATUS=$(curl -s -o /dev/null -w '%{http_code}' https://staging.example.com/health)
if [ $STATUS -ne 200 ]; then
    echo "Staging is unhealthy"
    exit 1
fi

# Check backup exists
BACKUP_EXISTS=$(ssh deployer@production "ls /var/backups/magento/$(date +%Y%m%d) 2>/dev/null")
if [ -z "$BACKUP_EXISTS" ]; then
    echo "No backup found for today"
    exit 1
fi

# Check disk space
DISK_USAGE=$(ssh deployer@production "df -h /var/www | tail -1 | awk '{print \$5}' | sed 's/%//'" )
if [ $DISK_USAGE -gt 90 ]; then
    echo "Disk usage too high: ${DISK_USAGE}%"
    exit 1
fi

echo "All pre-deployment checks passed"

Post-Deployment Verification

#!/bin/bash
# post-deploy-verify.sh

echo "Post-deployment verification..."

# Health check
HEALTH=$(curl -s https://www.example.com/health | jq -r '.status')
if [ "$HEALTH" != "healthy" ]; then
    echo "Health check failed: $HEALTH"
    exit 1
fi

# Response time check
RESPONSE_TIME=$(curl -s -o /dev/null -w '%{time_total}' https://www.example.com/)
if (( $(echo "$RESPONSE_TIME > 3.0" | bc -l) )); then
    echo "Response time too slow: ${RESPONSE_TIME}s"
    exit 1
fi

# Version check
VERSION=$(curl -s https://www.example.com/rest/V1/store/storeConfigs | jq -r '.[0].store_name')
echo "Store name: $VERSION"

# Error rate check
ERRORS=$(curl -s https://www.example.com/health | jq -r '.checks.error_count // 0')
if [ $ERRORS -gt 0 ]; then
    echo "Error count: $ERRORS"
    exit 1
fi

echo "All post-deployment checks passed"

Deployment Monitor Script

#!/bin/bash
# deployment-monitor.sh

DURATION=300  # 5 minutes
INTERVAL=10

start_time=$(date +%s)

while true; do
    current_time=$(date +%s)
    elapsed=$((current_time - start_time))
    
    if [ $elapsed -ge $DURATION ]; then
        echo "Monitoring complete"
        break
    fi
    
    # Check health
    STATUS=$(curl -s -o /dev/null -w '%{http_code}' https://www.example.com/)
    
    # Log to monitoring system
    echo "$(date): HTTP $STATUS" >> /var/log/deployment-monitor.log
    
    # Alert if unhealthy
    if [ $STATUS -ne 200 ]; then
        curl -X POST $SLACK_WEBHOOK \
            -d "{\"text\":\"Deployment alert: HTTP $STATUS\"}"
    fi
    
    sleep $INTERVAL
done

Key Takeaway

Verify deployment with pre-deployment checks (staging health, backups), post-deployment checks (health, response time), and continuous monitoring during deployment window.

Prometheus Metrics

<?php
// Custom metrics for deployment monitoring
class DeploymentMetrics
{
    private $prometheus;
    
    public function recordDeployment(string $version, string $status): void
    {
        $this->prometheus->counter(
            'magento_deployments_total',
            'Total deployments',
            ['version' => $version, 'status' => $status]
        );
    }
    
    public function recordDeploymentDuration(float $seconds): void
    {
        $this->prometheus->histogram(
            'magento_deployment_duration_seconds',
            'Deployment duration',
            $seconds
        );
    }
}

Grafana Dashboard Queries

# Request rate
rate(http_requests_total[5m])

# Error rate
rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m])

# Response time histogram
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))

# Deployment count
count(magento_deployments_total)

Alert Rules

# Prometheus alert rules
groups:
  - name: deployment-alerts
    rules:
      - alert: DeploymentFailed
        expr: increase(magento_deployments_total{status="failed"}[1h]) > 0
        for: 0m
        labels:
          severity: critical
        annotations:
          summary: "Deployment failed"
          description: "A deployment has failed"
      
      - alert: HighErrorRate
        expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High error rate detected"
          description: "Error rate is above 5%"
      
      - alert: SlowResponse
        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Slow response times"
          description: "95th percentile response time is above 2 seconds"

Slack Notification

# Send deployment notification
curl -X POST $SLACK_WEBHOOK \
    -H 'Content-type: application/json' \
    -d '{
        "text": "Deployment Complete",
        "blocks": [
            {
                "type": "section",
                "text": {
                    "type": "mrkdwn",
                    "text": "*Deployment Successful*\nVersion: v2.4.6\nTime: $(date)\nStatus: All checks passed"
                }
            }
        ]
    }'

Key Takeaway

Use Prometheus for metrics, Grafana for visualization, and alert rules for notifications. Monitor error rates, response times, and deployment counts.