Search Engine Unavailable

Symptoms

Indicators:
├── Elasticsearch cluster red/yellow
├── Search queries failing
├── Autocomplete not working
├── Faceted search broken
├── Indexing failures
└── Application errors

Impact:
├── Product search broken
├── Category pages empty
├── Poor user experience
├── Lost sales
└── SEO impact

Detection

# Check cluster health
curl -XGET 'localhost:9200/_cluster/health?pretty'

# Check node status
curl -XGET 'localhost:9200/_cat/nodes?v'

# Check index status
curl -XGET 'localhost:9200/_cat/indices?v'

# Check pending tasks
curl -XGET 'localhost:9200/_cluster/pending_tasks?pretty'

Common Causes

1. Cluster health red (unassigned shards)
2. Disk space full
3. JVM heap overflow
4. Network partition
5. Node failure
6. Index corruption
7. Too many shards
8. Slow queries

Fallback to Database

// Search with fallback
class SearchService
{
    public function search($query)
    {
        try {
            // Try Elasticsearch first
            return $this->elasticsearch->search($query);
            
        } catch (\Exception $e) {
            $this->logger->warning('Elasticsearch failed', [
                'error' => $e->getMessage()
            ]);
            
            // Fallback to database
            return $this->databaseFallback($query);
        }
    }
    
    private function databaseFallback($query)
    {
        // Use MySQL full-text search
        $results = $this->db->fetchAll(
            "SELECT * FROM catalog_product_entity 
             WHERE MATCH(name, description) AGAINST(? IN BOOLEAN MODE)
             LIMIT 20",
            [$query]
        );
        
        return [
            'hits' => $results,
            'total' => count($results),
            'fallback' => true
        ];
    }
}

Cached Search Results

// Use cached results during outage
public function searchWithCache($query)
{
    $cacheKey = 'search_' . md5($query);
    
    try {
        $results = $this->elasticsearch->search($query);
        
        // Cache successful results
        $this->cache->save(serialize($results), $cacheKey, [], 3600);
        
        return $results;
        
    } catch (\Exception $e) {
        // Try cache
        $cached = $this->cache->load($cacheKey);
        if ($cached) {
            $results = unserialize($cached);
            $results['cached'] = true;
            return $results;
        }
        
        // Fallback to database
        return $this->databaseFallback($query);
    }
}

Circuit Breaker

// Circuit breaker for search
class SearchCircuitBreaker
{
    private $failureCount = 0;
    private $lastFailureTime = 0;
    private $state = 'closed'; // closed, open, half-open
    
    public function call($callback)
    {
        if ($this->state === 'open') {
            if (time() - $this->lastFailureTime > 60) {
                $this->state = 'half-open';
            } else {
                throw new \Exception('Circuit breaker open');
            }
        }
        
        try {
            $result = $callback();
            $this->reset();
            return $result;
            
        } catch (\Exception $e) {
            $this->failureCount++;
            $this->lastFailureTime = time();
            
            if ($this->failureCount >= 5) {
                $this->state = 'open';
            }
            
            throw $e;
        }
    }
    
    private function reset()
    {
        $this->failureCount = 0;
        $this->state = 'closed';
    }
}

Graceful Degradation

// Degraded search mode
public function searchDegraded($query)
{
    // Disable advanced features
    $features = [
        'autocomplete' => false,
        'facets' => false,
        'personalization' => false,
        'spell_check' => false
    ];
    
    // Use basic search only
    $results = $this->basicSearch($query);
    
    // Show degraded notice
    $results['notice'] = 'Search is operating in reduced mode';
    
    return $results;
}

// Basic search using database
private function basicSearch($query)
{
    return $this->db->fetchAll(
        "SELECT entity_id, sku, name, price 
         FROM catalog_product_entity 
         JOIN catalog_product_entity_varchar ON ... 
         WHERE name LIKE ? 
         LIMIT 20",
        ['%' . $query . '%']
    );
}

Feature Flags

// Control features during outage
public function getFeatures()
{
    $searchAvailable = $this->checkSearchHealth();
    
    return [
        'search' => $searchAvailable,
        'autocomplete' => $searchAvailable,
        'facets' => $searchAvailable,
        'recommendations' => $searchAvailable,
        'basic_search' => true // Always available
    ];
}

// Use in frontend
if ($features['autocomplete']) {
    // Show autocomplete
} else {
    // Hide autocomplete
    // Show basic search input only
}

Cluster Recovery

# Check unassigned shards
curl -XGET 'localhost:9200/_cat/shards?v&h=index,shard,prirep,state,unassigned.reason'

# Reroute unassigned shards
curl -XPOST 'localhost:9200/_cluster/reroute' -d '{
  "commands": [{
    "allocate_stale_primary": {
      "index": "catalog_product",
      "shard": 0,
      "node": "node1",
      "accept_data_loss": true
    }
  }]
}'

# Increase disk watermarks
curl -XPUT 'localhost:9200/_cluster/settings' -d '{
  "transient": {
    "cluster.routing.allocation.disk.watermark.low": "90%",
    "cluster.routing.allocation.disk.watermark.high": "95%"
  }
}'

# Restart failed nodes
sudo systemctl start elasticsearch

Index Recovery

# Rebuild corrupted index
curl -XDELETE 'localhost:9200/catalog_product'
# Then reindex from database
bin/magento indexer:reindex catalogsearch_fulltext

# Optimize index
curl -XPOST 'localhost:9200/catalog_product/_optimize'

# Check index health
curl -XGET 'localhost:9200/_cat/indices?v'

Verification

// Test search after recovery
public function testSearch()
{
    $queries = ['wireless', 'headphones', 'blue', 'sale'];
    
    foreach ($queries as $query) {
        $results = $this->searchService->search($query);
        
        if (empty($results['hits'])) {
            throw new \Exception('Search failed for: ' . $query);
        }
    }
    
    // Check cluster health
    $health = $this->elasticsearch->cluster()->health();
    if ($health['status'] !== 'green') {
        throw new \Exception('Cluster not healthy: ' . $health['status']);
    }
}