Latency Increase Incident

Symptoms

Indicators:
├── Response time increased
├── Page load time increased
├── API response slow
├── User complaints
├── Timeout errors
└── Performance degradation

Impact:
├── Poor user experience
├── Cart abandonment
├── Lost sales
├── SEO ranking drop
└── Customer dissatisfaction

Detection

// Monitor response times
$metrics = [
    'response_time_p50' => $this->getResponseTime(50),
    'response_time_p95' => $this->getResponseTime(95),
    'response_time_p99' => $this->getResponseTime(99),
    'page_load_time' => $this->getPageLoadTime(),
    'api_response_time' => $this->getApiResponseTime()
];

// Alert thresholds
$thresholds = [
    'response_time_p95' => ['warning' => 2000, 'critical' => 5000],
    'page_load_time' => ['warning' => 3000, 'critical' => 5000],
    'api_response_time' => ['warning' => 1000, 'critical' => 3000]
];

Common Causes

1. Database slow queries
2. Redis cache misses
3. External service timeout
4. CPU/Memory exhaustion
5. Network issues
6. Code regression
7. Traffic spike
8. Resource contention

Application Profiling

// Enable profiling
$xhprof_dir = '/tmp/xhprof';
include_once $xhprof_dir . '/xhprof_lib/utils/xhprof_lib.php';
include_once $xhprof_dir . '/xhprof_lib/utils/xhprof_runs.php';

xhprof_enable(XHPROF_FLAGS_CPU | XHPROF_FLAGS_MEMORY);

// Application code
// ...

$xhprof_data = xhprof_disable();

$runs = new XHProfRuns_Default();
$run_id = $runs->save_run($xhprof_data, 'my_app');

// View report
// http://localhost/xhprof/index.php?run=$run_id&source=my_app

Database Profiling

-- Enable slow query log
SET GLOBAL slow_query_log = 'ON';
SET GLOBAL long_query_time = 1;

-- Check slow queries
SELECT * FROM mysql.slow_log
WHERE start_time > DATE_SUB(NOW(), INTERVAL 1 HOUR)
ORDER BY query_time DESC
LIMIT 10;

-- Analyze query performance
EXPLAIN ANALYZE SELECT * FROM catalog_product_entity
WHERE sku = '24-WB04';

-- Check query cache
SHOW STATUS LIKE 'Qcache%';

Redis Profiling

# Monitor Redis commands
redis-cli MONITOR

# Check slow commands
redis-cli SLOWLOG GET 10

# Check hit rate
redis-cli INFO stats | grep keyspace_hits
redis-cli INFO stats | grep keyspace_misses

# Check memory usage
redis-cli INFO memory

Network Profiling

# Check network latency
ping -c 10 localhost

# Check connection counts
netstat -an | grep :80 | wc -l
netstat -an | grep :443 | wc -l

# Check bandwidth
iftop -i eth0

# Check DNS resolution
time nslookup google.com

Identify Slow Components

// Profile each layer
$layers = [
    'web_server' => $this->profileWebServer(),
    'php' => $this->profilePhp(),
    'database' => $this->profileDatabase(),
    'redis' => $this->profileRedis(),
    'elasticsearch' => $this->profileElasticsearch(),
    'network' => $this->profileNetwork()
];

// Find bottleneck
usort($layers, function($a, $b) {
    return $b['time'] - $a['time'];
});

$biggestBottleneck = $layers[0];

Analyze Request Flow

// Profile request lifecycle
$requestFlow = [
    'routing' => 5,        // ms
    'authentication' => 10, // ms
    'controller' => 50,     // ms
    'model' => 100,         // ms
    'view' => 30,           // ms
    'response' => 5         // ms
];

// Total: 200ms
// Bottleneck: model (100ms = 50%)

// Focus on model layer
// - Check database queries
// - Check cache hits
// - Check external calls

Code Analysis

// Find slow code paths
$trace = $this->profiler->getTrace();

foreach ($trace as $entry) {
    if ($entry['time'] > 100) { // > 100ms
        echo "Slow: " . $entry['function'] . " (" . $entry['time'] . "ms)\n";
    }
}

// Common issues
$issues = [
    'N+1 queries' => $this->detectNPlusOne(),
    'missing_indexes' => $this->detectMissingIndexes(),
    'excessive_loops' => $this->detectExcessiveLoops(),
    'large_data_sets' => $this->detectLargeDataSets()
];

Immediate Fixes

// 1. Clear caches
$this->cache->flush();
$this->varnish->purge();

// 2. Kill slow queries
$this->db->query('KILL QUERY ' . $slowQueryId);

// 3. Scale resources
$this->deployment->scale('php-fpm', 5);
$this->database->addReadReplica();

// 4. Enable CDN
$this->cdn->enable();

// 5. Reduce traffic
$this->rateLimiter->setLimit(100);

Query Optimization

-- Add missing index
ALTER TABLE catalog_product_entity
ADD INDEX idx_sku (sku);

-- Rewrite slow query
-- Before: Subquery
SELECT * FROM orders WHERE customer_id IN (
    SELECT entity_id FROM customers WHERE group_id = 4
);

-- After: JOIN
SELECT o.* FROM orders o
JOIN customers c ON o.customer_id = c.entity_id
WHERE c.group_id = 4;

Caching Optimization

// Add caching for slow operations
public function getProducts($categoryId)
{
    $cacheKey = 'products_' . $categoryId;
    $cached = $this->cache->load($cacheKey);
    
    if ($cached) {
        return unserialize($cached);
    }
    
    $products = $this->slowDatabaseQuery($categoryId);
    $this->cache->save(serialize($products), $cacheKey, [], 3600);
    
    return $products;
}

Verification

// Test performance after fix
$before = $this->measurePerformance();

// Apply fix
$this->applyFix();

$after = $this->measurePerformance();

if ($after['response_time'] < $before['response_time'] * 0.5) {
    $this->logger->info('Performance improved by 50%');
} else {
    $this->logger->warning('Performance not improved significantly');
}