Changeset 3783147

Timestamp:

11/04/2026 06:02:59 AM (4 days ago)

Author:

funnelwon

Message:

Release 3.1.4: search uses less WordPress server resources (search boxes go straight to the results page, fewer WordPress database queries); crawlers, monitors and scripts get a short search results page; Smart 404 skips scanner-word addresses and more scanner and spam patterns are ignored.

Location:

searchus/trunk

Files:

• core/api-client.php (modified) (12 diffs)
• core/query-cache.php (modified) (1 diff)
• core/query-guard.php (modified) (3 diffs)
• core/tier.php (modified) (3 diffs)
• fw-search-indexer.php (modified) (4 diffs)
• languages/searchus.pot (modified) (40 diffs)
• readme.txt (modified) (3 diffs)
• serp/page-template-404-recovery.php (modified) (1 diff)
• serp/search-request-guard.php (added)
• serp/search-results.php (modified) (4 diffs)
• serp/serp-content-block.php (modified) (1 diff)
• serp/smart-404.php (modified) (5 diffs)

searchus/trunk/core/api-client.php

@@ -1861 +1861 @@
 }
 
+if ( ! defined( 'FW_SI_OPTION_API_BACKOFF' ) ) {
+  /** Option: visitor searches skip the API until this time (after HTTP 429 Retry-After, or the API not answering). */
+  define( 'FW_SI_OPTION_API_BACKOFF', 'fw_si_api_backoff' );
+}
+
+/**
+ * Active API pause for visitor searches: ['until' => unix time, 'code' => 429 or 0], or [] when none.
+ * Stored autoloaded, so checking it on every search costs no database query.
+ *
+ * @return array{until?:int,code?:int}
+ */
+function fw_si_api_backoff(): array {
+  $v = get_option( FW_SI_OPTION_API_BACKOFF, null );
+  if ( $v === null ) {
+    add_option( FW_SI_OPTION_API_BACKOFF, [], '', true );
+    return [];
+  }
+  return ( is_array( $v ) && (int) ( $v['until'] ?? 0 ) > time() ) ? $v : [];
+}
+
+/**
+ * Pause visitor searches for $seconds (1–120). A longer pause already running is kept.
+ *
+ * @param int $seconds Pause length.
+ * @param int $code    429 (service asked to slow down) or 0 (service did not answer).
+ */
+function fw_si_api_backoff_start( int $seconds, int $code ): void {
+  $until = time() + max( 1, min( 120, $seconds ) );
+  $cur   = fw_si_api_backoff();
+  if ( (int) ( $cur['until'] ?? 0 ) >= $until ) {
+    return;
+  }
+  update_option( FW_SI_OPTION_API_BACKOFF, [ 'until' => $until, 'code' => $code ], true );
+}
+
+/**
+ * Start a pause when the service answered 429 (uses its Retry-After) or could not be reached in time.
+ * Without it, every search during the burst or outage holds a PHP worker for the full HTTP timeout.
+ *
+ * @param array|WP_Error $response wp_remote_post() result.
+ */
+function fw_si_api_backoff_note_response( $response ): void {
+  if ( is_wp_error( $response ) ) {
+    /**
+     * Seconds visitor searches skip the API after it could not be reached in time.
+     *
+     * @param int $seconds Default 10.
+     */
+    fw_si_api_backoff_start( (int) apply_filters( 'fw_si_api_unreachable_pause_seconds', 10 ), 0 );
+    return;
+  }
+  if ( (int) wp_remote_retrieve_response_code( $response ) === 429 ) {
+    $retry = (int) wp_remote_retrieve_header( $response, 'retry-after' );
+    fw_si_api_backoff_start( $retry > 0 ? $retry : 5, 429 );
+  }
+}
+
 /**
  * Tell the service to count a visitor search that was answered from the local cache.
@@ -1867 +1924 @@
  * @param string $query        Search text the visitor submitted.
  * @param string $live_session Optional live search session id; the service counts one search per session.
+ * @param bool   $is_preview   True for a Predictive Search preview.
+ * @param string $source       'smart404' for a Smart 404 recovery search; empty otherwise.
+ * @param string $crawler      Search engine or AI crawler short name; empty for visitors.
+ * @param string $crawler_ip   The crawler request's address, verified by the service.
  * @return bool True when the service counted the search.
  */
-function fw_si_report_cached_search_usage( string $query, string $live_session = '', bool $is_preview = false ): bool {
+function fw_si_report_cached_search_usage( string $query, string $live_session = '', bool $is_preview = false, string $source = '', string $crawler = '', string $crawler_ip = '' ): bool {
   $site_id = function_exists( 'fw_si_get_site_id' ) ? fw_si_get_site_id() : '';
   if ( $site_id === '' || $query === '' ) {
@@ -1875 +1936 @@
   }
   if ( function_exists( 'fw_si_is_junk_query' ) && fw_si_is_junk_query( $query ) ) {
+    return false;
+  }
+  // The cached results are still shown; the search is just not counted while the service asked for a pause.
+  if ( fw_si_api_backoff() ) {
     return false;
   }
@@ -1882 +1947 @@
   if ( function_exists( 'fw_si_tier' ) ) {
     $body['tier'] = fw_si_tier();
+  }
+  if ( $source === 'smart404' ) {
+    $body['source'] = 'smart404';
+  }
+  if ( $crawler !== '' && ! $is_preview ) {
+    $body['crawler']    = $crawler;
+    $body['crawler_ip'] = $crawler_ip;
   }
   if ( $live_session !== '' ) {
@@ -1901 +1973 @@
     $url,
     [
-      'timeout' => 8,
+      /**
+       * Seconds a visitor's page waits for the service to count a cached search.
+       *
+       * @param int $timeout Default 3.
+       */
+      'timeout' => (int) apply_filters( 'fw_si_query_count_timeout', 3 ),
       'headers' => $headers,
       'body'    => $encoded,
     ]
   );
+  fw_si_api_backoff_note_response( $response );
   if ( is_wp_error( $response ) ) {
     return false;
@@ -1933 +2011 @@
  *                        ['live_session' => string] = live search session id (one counted search per session).
  *                        ['live_preview' => int] = live search preview: completes a partial last word, top N results.
+ *                        ['source' => 'smart404'] = Smart 404 recovery search (the service drops scanner path probes).
  * @return array|null Payload (all, content, media, ...) or debug payload (debug_results); null on failure.
  */
@@ -1972 +2051 @@
     : '';
   $live_preview   = isset( $options['live_preview'] ) ? max( 0, (int) $options['live_preview'] ) : 0;
+  $search_source  = ( isset( $options['source'] ) && $options['source'] === 'smart404' ) ? 'smart404' : '';
+  $crawler        = ( $live_preview === 0 && isset( $options['crawler'] ) && is_string( $options['crawler'] ) ) ? sanitize_key( $options['crawler'] ) : '';
+  $crawler_ip     = ( $crawler !== '' && isset( $options['crawler_ip'] ) && is_string( $options['crawler_ip'] ) ) ? $options['crawler_ip'] : '';
   // Transient cache applies to all tiers so repeat queries avoid POST /v1/query. Return true from
   // fw_si_query_api_skip_transient_cache to bypass (e.g. if service-supplied attribution HTML must stay in lockstep with usage before TTL).
@@ -1987 +2069 @@
           fw_si_query_cache_extend_ttl( $cache_key );
         }
-        if ( empty( $options['skip_log'] ) && function_exists( 'fw_si_report_cached_search_usage' ) && fw_si_report_cached_search_usage( $query, $live_session, $live_preview > 0 ) ) {
+        if ( empty( $options['skip_log'] ) && function_exists( 'fw_si_report_cached_search_usage' ) && fw_si_report_cached_search_usage( $query, $live_session, $live_preview > 0, $search_source, $crawler, $crawler_ip ) ) {
           $cached['_fw_usage_counted'] = true;
         }
@@ -2010 +2092 @@
     }
   }
+  $backoff = $is_debug ? [] : fw_si_api_backoff();
+  if ( $backoff ) {
+    $wait = max( 1, (int) $backoff['until'] - time() );
+    return [
+      '_fw_error'        => true,
+      '_fw_error_code'   => (int) ( $backoff['code'] ?? 0 ),
+      /* translators: %d: seconds until searches are sent to the service again */
+      '_fw_error_detail' => sprintf( __( 'Searches paused for %d seconds after the service was busy or unreachable.', 'searchus' ), $wait ),
+    ];
+  }
   $base = fw_si_get_api_base_url();
   $url  = $is_debug ? ( $base . '/v1/debug/query' ) : ( $base . '/v1/query' );
@@ -2029 +2121 @@
     $body['preview_limit'] = $live_preview;
   }
+  if ( $search_source !== '' ) {
+    $body['source'] = $search_source;
+  }
+  if ( $crawler !== '' ) {
+    $body['crawler']    = $crawler;
+    $body['crawler_ip'] = $crawler_ip;
+  }
   if ( ! $is_debug && function_exists( 'fw_si_query_cache_take_refresh' ) && fw_si_query_cache_take_refresh( $cache_key ) ) {
     $body['cache_refresh'] = true;
@@ -2047 +2146 @@
 
   $qt_http_start = $qt_api_debug ? microtime( true ) : 0;
+  // A visitor's page holds a PHP worker while it waits; debug and pre-cache runs are not on a visitor's page.
+  $visitor_wait = $is_debug || ! empty( $options['skip_log'] ) ? 15 : (int) apply_filters( 'fw_si_query_api_timeout', 6, $live_preview > 0 );
   $response = wp_remote_post(
     $url,
     [
-      'timeout' => 15,
+      'timeout' => $visitor_wait,
       'headers' => $headers,
       'body'    => $query_encoded,
     ]
   );
+  if ( ! $is_debug ) {
+    fw_si_api_backoff_note_response( $response );
+  }
   if ( $qt_api_debug ) { fw_si_plugin_diagnostic_log( '[FW_SI_QT] api_source=http query=' . substr( $query, 0, 40 ) . ' http_ms=' . round( ( microtime( true ) - $qt_http_start ) * 1000 ) ); }
   if ( is_wp_error( $response ) ) {
@@ -2169 +2273 @@
       fw_si_query_cache_register_doc_ids( $cache_key, $decoded );
     }
-    if ( function_exists( 'fw_si_query_cache_trim_to_top' ) ) {
-      fw_si_query_cache_trim_to_top();
+    if ( function_exists( 'fw_si_query_cache_schedule_trim' ) ) {
+      fw_si_query_cache_schedule_trim();
     }
     if ( ! $is_debug && empty( $options['skip_log'] ) && function_exists( 'fw_si_log_query_for_precache' ) ) {

searchus/trunk/core/query-cache.php

@@ -357 +357 @@
         }
 }
+
+/**
+ * Trim the cache in a cron run once it holds more than FW_SI_QUERY_CACHE_MAX_ENTRIES searches.
+ * Trimming asks the service for its top tokens, which a visitor's search should not wait for.
+ */
+function fw_si_query_cache_schedule_trim() {
+        $map = get_option( FW_SI_OPTION_QUERY_CACHE_MAP, [] );
+        if ( ! is_array( $map ) || count( $map ) <= (int) FW_SI_QUERY_CACHE_MAX_ENTRIES ) {
+                return;
+        }
+        if ( ! wp_next_scheduled( 'fw_si_query_cache_trim' ) ) {
+                wp_schedule_single_event( time(), 'fw_si_query_cache_trim' );
+        }
+}
+add_action( 'fw_si_query_cache_trim', 'fw_si_query_cache_trim_to_top' );
 
 /**

searchus/trunk/core/query-guard.php

@@ -122 +122 @@
         }
 
+        // Scanner signatures that reach search as words once "/" and "-" become spaces (Smart 404),
+        // and torrent listings pasted as searches. Same rules as the SearchUs API.
+        if ( preg_match( '/\bproc\s+self\b|(?:^|\s)@fs(?:\s|$)|\btfvars\b|\btfstate\b|\bapi_server\b|\bbafk[a-z2-7]{20,}|\bz9x8c7v6b5\b/i', $lower ) ) {
+                return true;
+        }
+        if ( preg_match( '/\b[0-9a-f]{32,40}\b|&tr=|magnet:|urn:btih/i', $lower ) ) {
+                return true;
+        }
+        if ( preg_match( '/[\'"]\s*>|>\s*[\'"]/', $q ) ) {
+                return true;
+        }
+        // Only symbols ("**", "'>\""). A lone "*" is the browse-all query.
+        if ( $q !== '*' && ! preg_match( '/[\p{L}\p{N}]/u', $q ) ) {
+                return true;
+        }
+
         return false;
+}
+
+/**
+ * True when a Smart 404 search (words from a missing URL) is only scanner path words and numbers,
+ * e.g. /admin/login/ or /api/v1/config/. Not used for typed searches: a visitor may search "login" on purpose.
+ *
+ * @param string $query Query derived from the missing URL.
+ * @return bool
+ */
+function fw_si_is_smart404_probe_query( $query ): bool {
+        static $probe = null;
+        if ( $probe === null ) {
+                $probe = array_flip(
+                        preg_split(
+                                '/\s+/',
+                                'admin administrator login logout signin signup register auth oauth sso user users account accounts profile '
+                                . 'api apis rest graphql v1 v2 v3 v4 config configs configuration settings env environment environ secret secrets '
+                                . 'token tokens key keys credential credentials private public backup backups bak old new test tests tmp temp dev '
+                                . 'staging stage prod production debug console server status health healthz metrics actuator ui uploads upload '
+                                . 'wp wordpress includes content ajax json xmlrpc cgi bin vendor plugin plugins theme themes assets core js css '
+                                . 'jquery min map maint maintenance install setup shell cmd db database sql dump git svn docs doc cron log logs '
+                                . 'phpmyadmin adminer dashboard panel cpanel manager portal internal system root home swagger openapi schema'
+                        )
+                );
+        }
+        $words = preg_split( '/[^a-z0-9]+/', strtolower( (string) $query ), -1, PREG_SPLIT_NO_EMPTY );
+        if ( ! is_array( $words ) || ! $words ) {
+                return false;
+        }
+        $has_probe_word = false;
+        foreach ( $words as $w ) {
+                if ( isset( $probe[ $w ] ) ) {
+                        $has_probe_word = true;
+                } elseif ( ! ctype_digit( $w ) ) {
+                        return false;
+                }
+        }
+        return $has_probe_word;
 }
 
@@ -141 +195 @@
                 return $is_bot;
         }
+        $is_bot = fw_si_is_bot_user_agent_request();
+        return $is_bot;
+}
+
+/**
+ * Same test as fw_si_is_bot_request() for use before `init`: a login cookie stands in for is_user_logged_in(),
+ * because asking for the current user that early can run before plugins add their login handlers.
+ *
+ * @return bool
+ */
+function fw_si_is_bot_request_early(): bool {
+        if ( ( defined( 'WP_CLI' ) && WP_CLI ) || wp_doing_cron() ) {
+                return false;
+        }
+        if ( defined( 'LOGGED_IN_COOKIE' ) && isset( $_COOKIE[ LOGGED_IN_COOKIE ] ) ) {
+                return false;
+        }
+        return fw_si_is_bot_user_agent_request();
+}
+
+/**
+ * Search engine or AI crawler named by the user agent, as the short name the SearchUs service verifies
+ * ('google', 'bing', 'openai', …), or ''. Anyone can send these user agents: the service checks the
+ * request's address, and shows results only to the real crawler. Crawler searches are never counted.
+ *
+ * @return string
+ */
+function fw_si_search_crawler_name(): string {
+        static $name = null;
+        if ( $name !== null ) {
+                return $name;
+        }
+        $ua    = isset( $_SERVER['HTTP_USER_AGENT'] ) ? strtolower( sanitize_text_field( wp_unslash( (string) $_SERVER['HTTP_USER_AGENT'] ) ) ) : '';
+        $found = '';
+        if ( $ua !== '' ) {
+                $crawlers = array(
+                        'google'     => 'googlebot|google-inspectiontool',
+                        'bing'       => 'bingbot|adidxbot',
+                        'duckduckgo' => 'duckduckbot|duckassistbot',
+                        'yandex'     => 'yandexbot',
+                        'baidu'      => 'baiduspider',
+                        'apple'      => 'applebot',
+                        'yahoo'      => 'slurp',
+                        'naver'      => 'yeti\/',
+                        'seznam'     => 'seznambot',
+                        'qwant'      => 'qwantbot|qwantify',
+                        'openai'     => 'gptbot|oai-searchbot|chatgpt-user',
+                        'anthropic'  => 'claudebot|claude-user|claude-searchbot',
+                        'perplexity' => 'perplexitybot|perplexity-user',
+                );
+                foreach ( $crawlers as $short => $pattern ) {
+                        if ( preg_match( '/' . $pattern . '/', $ua ) ) {
+                                $found = $short;
+                                break;
+                        }
+                }
+        }
+        /**
+         * Search engine or AI crawler for the current request ('' for none). Smart 404 pages, and results pages
+         * on the Free plan, show such crawlers real results and the attribution; other bots get the theme's 404
+         * or the short results page.
+         *
+         * @param string $found Short crawler name, or ''.
+         * @param string $ua    Lowercased user agent.
+         */
+        $name = (string) apply_filters( 'fw_si_search_crawler', $found, $ua );
+        return $name;
+}
+
+/**
+ * True on the Free plan, where results pages carry the SearchUs attribution and search engines are
+ * welcome to crawl them. On paid plans results pages are kept away from crawlers.
+ *
+ * @return bool
+ */
+function fw_si_crawlers_read_results_pages(): bool {
+        $free = function_exists( 'fw_si_tier' ) && fw_si_tier() === 'free';
+        /**
+         * Whether search engine and AI crawlers get full results pages (and robots.txt does not block them).
+         *
+         * @param bool $allow Default: true on the Free plan, false on paid plans.
+         */
+        return (bool) apply_filters( 'fw_si_crawlers_read_results_pages', $free );
+}
+
+/**
+ * User agent half of bot detection (no login check).
+ *
+ * @return bool
+ */
+function fw_si_is_bot_user_agent_request(): bool {
         $ua = isset( $_SERVER['HTTP_USER_AGENT'] ) ? strtolower( sanitize_text_field( wp_unslash( (string) $_SERVER['HTTP_USER_AGENT'] ) ) ) : '';
         // Every browser sends a user agent; an empty one is a script.
@@ -155 +300 @@
          * @param string $ua     Lowercased user agent.
          */
-        $is_bot = (bool) apply_filters( 'fw_si_is_bot_request', $is_bot, $ua );
-        return $is_bot;
+        return (bool) apply_filters( 'fw_si_is_bot_request', $is_bot, $ua );
 }
 

searchus/trunk/core/tier.php

@@ -7 +7 @@
 
 /**
- * Plan/tier detection. Single source of truth: plan name (Free, Growth, Pro, Business, Enterprise, Custom).
+ * Plan/tier detection. Single source of truth: plan name (Free, Starter, Growth, Scale, Pro, Enterprise, Custom).
  * API and plugin use the same concept; binary "free vs paid" is derived from plan.
  *
@@ -14 +14 @@
 
 /** Canonical plan names (API and plugin). */
-define( 'FW_SI_PLANS', 'Free,Growth,Pro,Business,Enterprise,Custom' );
+define( 'FW_SI_PLANS', 'Free,Starter,Growth,Scale,Pro,Enterprise,Custom' );
 
 /**
@@ -90 +90 @@
 
 /**
- * Current plan name: 'Free' | 'Growth' | 'Pro' | 'Business' | 'Enterprise' | 'Custom'.
+ * Current plan name: 'Free' | 'Starter' | 'Growth' | 'Scale' | 'Pro' | 'Enterprise' | 'Custom'.
  * From activation response (stored in fw_si_license_plan); default 'Free' if no license or not set.
  */
 function fw_si_plan(): string {
-  $plans = array( 'Free', 'Growth', 'Pro', 'Business', 'Enterprise', 'Custom' );
+  $plans = array( 'Free', 'Starter', 'Growth', 'Scale', 'Pro', 'Enterprise', 'Custom' );
 
   // From activation response (stored in fw_si_license_plan)

searchus/trunk/fw-search-indexer.php

@@ -5 +5 @@
  * Plugin Name: SearchUs – Site Search for WordPress and WooCommerce
  * Description: Intelligent indexing, intent scoring, and SERP control for WordPress.
- * Version: 3.1.3
+ * Version: 3.1.4
  * Author: FunnelWon
  * Author URI: https://searchus.io
@@ -129 +129 @@
 ========================================================= */
 
-define( 'FW_SI_VERSION', '3.1.3' );
+define( 'FW_SI_VERSION', '3.1.4' );
 define( 'FW_SI_PATCH', 13 );
 define( 'FW_SI_PLUGIN_DIR', __DIR__ );
@@ -340 +340 @@
   $fw_si_lock_bypass[] = 'fw_si_search_fallback';
   $fw_si_lock_bypass[] = 'fw_si_attribution_check';
+  $fw_si_lock_bypass[] = 'fw_si_api_backoff';
   if ( in_array( $option, $fw_si_lock_bypass, true ) ) {
     return $value;
@@ -889 +890 @@
 }
 require_once FW_SI_PLUGIN_DIR . '/serp/search-integration.php';
+require_once FW_SI_PLUGIN_DIR . '/serp/search-request-guard.php';
 require_once FW_SI_PLUGIN_DIR . '/serp/smart-404.php';
 require_once FW_SI_PLUGIN_DIR . '/admin/result-control-admin.php';

searchus/trunk/languages/searchus.pot

@@ -3 +3 @@
 msgid ""
 msgstr ""
-"Project-Id-Version: SearchUs – Site Search for WordPress and WooCommerce 3.1.3\n"
+"Project-Id-Version: SearchUs – Site Search for WordPress and WooCommerce 3.1.4\n"
 "Report-Msgid-Bugs-To: https://wordpress.org/support/plugin/searchus\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
@@ -10 +10 @@
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"POT-Creation-Date: 2026-10-28T03:06:56+00:00\n"
+"POT-Creation-Date: 2026-11-04T05:37:56+00:00\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "X-Generator: WP-CLI 2.12.0\n"
@@ -942 +942 @@
 #. translators: %d: Minimum excerpt length in characters (UTF-8).
 #: admin/result-control-admin.php:470
-#: fw-search-indexer.php:517
+#: fw-search-indexer.php:518
 #, php-format
 msgid "Required for this type: at least %d characters (UTF-8). For products this is the short description under the price."
@@ -1185 +1185 @@
 #: admin/search-embeds-admin.php:265
 #: admin/settings-page.php:4776
-#: fw-search-indexer.php:429
+#: fw-search-indexer.php:430
 msgid "Delete"
 msgstr ""
@@ -1251 +1251 @@
 #: admin/settings-page.php:956
 #: admin/settings-page.php:982
-#: fw-search-indexer.php:478
+#: fw-search-indexer.php:479
 msgid "Another user"
 msgstr ""
 
 #: admin/settings-page.php:242
-#: fw-search-indexer.php:479
+#: fw-search-indexer.php:480
 msgid "Editing in progress."
 msgstr ""
@@ -1385 +1385 @@
 #: admin/settings-page.php:1256
 #: admin/settings-page.php:3968
-#: serp/serp-content-block.php:2284
-#: serp/serp-content-block.php:2709
+#: serp/serp-content-block.php:2287
+#: serp/serp-content-block.php:2712
 msgid "Read more"
 msgstr ""
@@ -1487 +1487 @@
 #: admin/settings-page.php:1480
 #: analytics/analytics-ui.php:763
-#: serp/serp-content-block.php:1852
-#: serp/serp-content-block.php:2739
+#: serp/serp-content-block.php:1855
+#: serp/serp-content-block.php:2742
 msgid "Previous"
 msgstr ""
@@ -1494 +1494 @@
 #: admin/settings-page.php:1494
 #: analytics/analytics-ui.php:763
-#: serp/serp-content-block.php:1867
-#: serp/serp-content-block.php:2754
+#: serp/serp-content-block.php:1870
+#: serp/serp-content-block.php:2757
 msgid "Next"
 msgstr ""
@@ -1565 +1565 @@
 #: admin/settings-register.php:1020
 #: core/hmac-auth.php:213
-#: fw-search-indexer.php:734
-#: fw-search-indexer.php:796
+#: fw-search-indexer.php:735
+#: fw-search-indexer.php:797
 #: serp/search-page.php:695
 msgid "Unauthorized"
@@ -1793 +1793 @@
 
 #: admin/settings-page.php:3198
-#: fw-search-indexer.php:438
+#: fw-search-indexer.php:439
 msgid "Disable Crawling"
 msgstr ""
 
 #: admin/settings-page.php:3198
-#: fw-search-indexer.php:439
+#: fw-search-indexer.php:440
 msgid "Enable Crawling"
 msgstr ""
@@ -2012 +2012 @@
 
 #: admin/settings-page.php:3760
-#: fw-search-indexer.php:436
+#: fw-search-indexer.php:437
 msgid "Reindex in progress."
 msgstr ""
@@ -2132 +2132 @@
 #: admin/settings-page.php:4046
 #: admin/setup-wizard.php:78
-#: fw-search-indexer.php:484
+#: fw-search-indexer.php:485
 msgid "Unlock Settings"
 msgstr ""
 
 #: admin/settings-page.php:4046
-#: fw-search-indexer.php:485
+#: fw-search-indexer.php:486
 msgid "Lock Settings"
 msgstr ""
@@ -2167 +2167 @@
 #: admin/settings-page.php:5104
 #: admin/settings-page.php:5223
-#: fw-search-indexer.php:428
+#: fw-search-indexer.php:429
 msgid "Expand"
 msgstr ""
@@ -2866 +2866 @@
 
 #: admin/settings-page.php:5350
-#: fw-search-indexer.php:435
+#: fw-search-indexer.php:436
 msgid "Minimize"
 msgstr ""
@@ -3813 +3813 @@
 #. translators: 1: Current page number, 2: Total number of pages.
 #: analytics/analytics-ui.php:761
-#: serp/serp-content-block.php:1859
-#: serp/serp-content-block.php:2746
+#: serp/serp-content-block.php:1862
+#: serp/serp-content-block.php:2749
 #, php-format
 msgid "Page %1$d of %2$d"
@@ -3828 +3828 @@
 
 #: analytics/analytics-ui.php:763
-#: serp/serp-content-block.php:1905
+#: serp/serp-content-block.php:1908
 msgid "Results Per Page:"
 msgstr ""
@@ -4160 +4160 @@
 msgctxt "HTML link in API notice"
 msgid "Manage your plan or limits: %s"
+msgstr ""
+
+#. translators: %d: seconds until searches are sent to the service again
+#: core/api-client.php:2101
+#, php-format
+msgid "Searches paused for %d seconds after the service was busy or unreachable."
 msgstr ""
 
@@ -4496 +4502 @@
 
 #: crawler/crawl-confirm.php:512
-#: fw-search-indexer.php:446
+#: fw-search-indexer.php:447
 msgid "Auto-Crawl Disabled"
 msgstr ""
@@ -4502 +4508 @@
 #. translators: %d: Number of pages sent to the service and waiting for it to confirm they are indexed.
 #: crawler/crawl-confirm.php:516
-#: fw-search-indexer.php:449
+#: fw-search-indexer.php:450
 #, php-format
 msgid "Waiting for confirmation (%d)"
@@ -4508 +4514 @@
 
 #: crawler/crawl-confirm.php:519
-#: fw-search-indexer.php:447
+#: fw-search-indexer.php:448
 msgid "Active — Processing queue"
 msgstr ""
 
 #: crawler/crawl-confirm.php:521
-#: fw-search-indexer.php:450
+#: fw-search-indexer.php:451
 msgid "Idle — No queued items"
 msgstr ""
@@ -4545 +4551 @@
 msgstr ""
 
-#: fw-search-indexer.php:413
+#: fw-search-indexer.php:414
 msgid "Result"
 msgstr ""
 
-#: fw-search-indexer.php:414
+#: fw-search-indexer.php:415
 msgid "Injected"
 msgstr ""
 
 #. translators: %s: Previous title or label for a reordered search result.
-#: fw-search-indexer.php:416
+#: fw-search-indexer.php:417
 #, php-format
 msgid "(Previously %s)"
 msgstr ""
 
-#: fw-search-indexer.php:425
+#: fw-search-indexer.php:426
 msgid "Error"
 msgstr ""
 
-#: fw-search-indexer.php:426
+#: fw-search-indexer.php:427
 msgid "Reindex failed on server."
 msgstr ""
 
-#: fw-search-indexer.php:427
+#: fw-search-indexer.php:428
 msgid "Collapse"
 msgstr ""
 
-#: fw-search-indexer.php:430
+#: fw-search-indexer.php:431
 msgid "Deleting..."
 msgstr ""
 
 #. translators: %s = comma-separated list of query phrases where the URL is a duplicate
-#: fw-search-indexer.php:432
+#: fw-search-indexer.php:433
 #, php-format
 msgid "This URL already appears in organic search results for: %s. Duplicate injections are automatically skipped on the SERP, so this injection will not display. Remove the URL from the injection or from the index first."
 msgstr ""
 
-#: fw-search-indexer.php:433
+#: fw-search-indexer.php:434
 msgid "Save is taking longer than expected. The page did not finish loading — check whether your changes were saved, then try again."
 msgstr ""
 
-#: fw-search-indexer.php:434
+#: fw-search-indexer.php:435
 msgid "Close"
 msgstr ""
 
-#: fw-search-indexer.php:437
+#: fw-search-indexer.php:438
 msgid "Search index is rebuilding. Controls are temporarily locked."
 msgstr ""
 
-#: fw-search-indexer.php:440
+#: fw-search-indexer.php:441
 msgid "Active — Reindexing"
 msgstr ""
 
-#: fw-search-indexer.php:441
+#: fw-search-indexer.php:442
 msgid "Active"
 msgstr ""
 
-#: fw-search-indexer.php:442
+#: fw-search-indexer.php:443
 msgid "Paused"
 msgstr ""
 
-#: fw-search-indexer.php:443
+#: fw-search-indexer.php:444
 msgid "Indexing is paused."
 msgstr ""
 
-#: fw-search-indexer.php:444
+#: fw-search-indexer.php:445
 msgid "1 or more items are queued for crawling. Once crawling is enabled, these will be processed automatically."
 msgstr ""
 
-#: fw-search-indexer.php:445
+#: fw-search-indexer.php:446
 msgid "Unknown"
 msgstr ""
 
-#: fw-search-indexer.php:451
+#: fw-search-indexer.php:452
 msgid "Reindexing…"
 msgstr ""
 
-#: fw-search-indexer.php:452
+#: fw-search-indexer.php:453
 msgid "Settings are Locked - Click Unlock Settings to make edits"
 msgstr ""
 
-#: fw-search-indexer.php:453
+#: fw-search-indexer.php:454
 msgid "Settings are locked. Unlock settings to clear cache."
 msgstr ""
 
-#: fw-search-indexer.php:454
+#: fw-search-indexer.php:455
 msgid "Failed to clear cache."
 msgstr ""
 
-#: fw-search-indexer.php:455
+#: fw-search-indexer.php:456
 msgid "Settings are locked. Unlock settings to run pre-cache."
 msgstr ""
 
-#: fw-search-indexer.php:456
+#: fw-search-indexer.php:457
 msgid "Pre-cache failed."
 msgstr ""
 
-#: fw-search-indexer.php:457
+#: fw-search-indexer.php:458
 msgid "Reset endpoint is not available. Please refresh and try again."
 msgstr ""
 
-#: fw-search-indexer.php:458
+#: fw-search-indexer.php:459
 msgid "Settings are locked. Unlock settings to reset."
 msgstr ""
 
-#: fw-search-indexer.php:459
+#: fw-search-indexer.php:460
 msgid "Reset all settings to defaults? This cannot be undone."
 msgstr ""
 
-#: fw-search-indexer.php:460
+#: fw-search-indexer.php:461
 msgid "Final confirmation: this will overwrite your current settings. Continue?"
 msgstr ""
 
-#: fw-search-indexer.php:461
+#: fw-search-indexer.php:462
 msgid "Failed to reset settings."
 msgstr ""
 
-#: fw-search-indexer.php:462
+#: fw-search-indexer.php:463
 msgid "Replace all existing synonyms?"
 msgstr ""
 
-#: fw-search-indexer.php:463
+#: fw-search-indexer.php:464
 msgid "Replace all existing variant option aliases?"
 msgstr ""
 
-#: fw-search-indexer.php:464
+#: fw-search-indexer.php:465
 msgid "Replace all existing crawl exclusion rules?"
 msgstr ""
 
-#: fw-search-indexer.php:465
+#: fw-search-indexer.php:466
 msgid "Suggesting…"
 msgstr ""
 
-#: fw-search-indexer.php:466
+#: fw-search-indexer.php:467
 msgid "Suggest failed. Try again."
 msgstr ""
 
-#: fw-search-indexer.php:467
+#: fw-search-indexer.php:468
 msgid "Unlock settings to use Suggest."
 msgstr ""
 
-#: fw-search-indexer.php:468
+#: fw-search-indexer.php:469
 msgid ""
 "Rebuild the search index now?\n"
@@ -4692 +4698 @@
 msgstr ""
 
-#: fw-search-indexer.php:469
+#: fw-search-indexer.php:470
 msgid "Settings are locked. Unlock settings to change crawling."
 msgstr ""
 
-#: fw-search-indexer.php:470
+#: fw-search-indexer.php:471
 msgid "Enabling..."
 msgstr ""
 
-#: fw-search-indexer.php:471
+#: fw-search-indexer.php:472
 msgid "Disabling..."
 msgstr ""
 
-#: fw-search-indexer.php:472
+#: fw-search-indexer.php:473
 msgid "Settings are locked. Unlock settings to reindex."
 msgstr ""
 
-#: fw-search-indexer.php:473
+#: fw-search-indexer.php:474
 msgid "Analyzing..."
 msgstr ""
 
-#: fw-search-indexer.php:474
+#: fw-search-indexer.php:475
 msgid "Run Full Reindex"
 msgstr ""
 
-#: fw-search-indexer.php:475
+#: fw-search-indexer.php:476
 msgid "Reindex failed. Check your license and try again."
 msgstr ""
 
-#: fw-search-indexer.php:476
+#: fw-search-indexer.php:477
 msgid "OK"
 msgstr ""
 
-#: fw-search-indexer.php:477
+#: fw-search-indexer.php:478
 msgid "Reindex request failed. This may be a timeout on large sites. Refresh the page to check if the reindex completed in the background."
 msgstr ""
 
 #. translators: %s = editor display name
-#: fw-search-indexer.php:481
+#: fw-search-indexer.php:482
 #, php-format
 msgid "%s is currently editing FunnelWon Search Engine settings. You can view configuration, but editing is temporarily disabled."
@@ -4735 +4741 @@
 
 #. translators: %s: Remaining time until editing may be available (human-readable, e.g. countdown).
-#: fw-search-indexer.php:483
+#: fw-search-indexer.php:484
 #, php-format
 msgid "Editing access may become available in %s."
 msgstr ""
 
-#: fw-search-indexer.php:486
+#: fw-search-indexer.php:487
 msgid "- Settings are read-only to prevent accidental changes."
 msgstr ""
 
-#: fw-search-indexer.php:487
+#: fw-search-indexer.php:488
 msgid "- Changes are enabled. Remember to lock settings when finished."
 msgstr ""
 
-#: fw-search-indexer.php:488
+#: fw-search-indexer.php:489
 msgid "Unlocking..."
 msgstr ""
 
-#: fw-search-indexer.php:489
+#: fw-search-indexer.php:490
 msgid "Locking..."
 msgstr ""
 
-#: fw-search-indexer.php:490
+#: fw-search-indexer.php:491
 msgid "Dismiss this notice."
 msgstr ""
 
-#: fw-search-indexer.php:491
+#: fw-search-indexer.php:492
 msgid ""
 "Disable FunnelWon Search?\n"
@@ -4770 +4776 @@
 
 #. translators: %s = user display name
-#: fw-search-indexer.php:493
+#: fw-search-indexer.php:494
 #, php-format
 msgid "%s has taken over. Refreshing…"
 msgstr ""
 
-#: fw-search-indexer.php:494
+#: fw-search-indexer.php:495
 msgid "Settings are currently locked."
 msgstr ""
 
 #. translators: %s = countdown timer string like "2:30"
-#: fw-search-indexer.php:496
+#: fw-search-indexer.php:497
 #, php-format
 msgid " Try again in %s."
@@ -4786 +4792 @@
 
 #. translators: %s = editor display name
-#: fw-search-indexer.php:498
+#: fw-search-indexer.php:499
 #, php-format
 msgid "%s is currently editing these settings."
 msgstr ""
 
-#: fw-search-indexer.php:499
+#: fw-search-indexer.php:500
 msgid ""
 "\n"
@@ -4798 +4804 @@
 msgstr ""
 
-#: fw-search-indexer.php:500
+#: fw-search-indexer.php:501
 msgid "Could not verify edit session. Please refresh the page and try again."
 msgstr ""
 
-#: fw-search-indexer.php:501
+#: fw-search-indexer.php:502
 msgid "Military Time:"
 msgstr ""
 
-#: fw-search-indexer.php:502
-#: serp/serp-content-block.php:1849
-#: serp/serp-content-block.php:2736
+#: fw-search-indexer.php:503
+#: serp/serp-content-block.php:1852
+#: serp/serp-content-block.php:2739
 msgid "Pagination"
 msgstr ""
 
-#: fw-search-indexer.php:503
+#: fw-search-indexer.php:504
 msgid "Enter a query above and click \"Show results\" to load the preview list."
 msgstr ""
 
-#: fw-search-indexer.php:504
+#: fw-search-indexer.php:505
 msgid "No results for this query."
 msgstr ""
 
-#: fw-search-indexer.php:505
+#: fw-search-indexer.php:506
 msgid "(Promoted)"
 msgstr ""
 
-#: fw-search-indexer.php:506
+#: fw-search-indexer.php:507
 msgid "Could not load results."
 msgstr ""
 
-#: fw-search-indexer.php:507
+#: fw-search-indexer.php:508
 msgid "Removing reorder rule…"
 msgstr ""
 
-#: fw-search-indexer.php:508
+#: fw-search-indexer.php:509
 msgid "Could not remove reorder rule."
 msgstr ""
 
-#: fw-search-indexer.php:509
+#: fw-search-indexer.php:510
 msgid "Saving reorder rule…"
 msgstr ""
 
-#: fw-search-indexer.php:510
+#: fw-search-indexer.php:511
 msgid "Reorder rule save failed."
 msgstr ""
 
-#: fw-search-indexer.php:511
+#: fw-search-indexer.php:512
 msgid "Removing…"
 msgstr ""
 
-#: fw-search-indexer.php:512
+#: fw-search-indexer.php:513
 msgid "Remove failed"
 msgstr ""
 
-#: fw-search-indexer.php:513
+#: fw-search-indexer.php:514
 msgid "Paste a single embed URL (https://www.youtube.com/embed/… or player.vimeo.com/…) or the full iframe block from the host's share dialog (we read src). Watch links are converted when possible. Must end up as https on an allowed host. After the first save, this cannot be changed."
 msgstr ""
 
-#: fw-search-indexer.php:514
+#: fw-search-indexer.php:515
 msgid "Not used for video embed injections. The SERP shows an \"Embed\" line; the modal uses an iframe player."
 msgstr ""
 
-#: fw-search-indexer.php:515
+#: fw-search-indexer.php:516
 msgid "Not used for this media type. The SERP shows size and format from the file URL when available (same as organic media results)."
 msgstr ""
 
-#: fw-search-indexer.php:518
+#: fw-search-indexer.php:519
 msgid "Please select a content type before saving an injection."
 msgstr ""
 
 #. translators: %1$d = min chars, %2$d = entered chars
-#: fw-search-indexer.php:520
+#: fw-search-indexer.php:521
 #, php-format
 msgid "Excerpt must be at least %1$d characters (%2$d entered). Use Clear form if you only want to save other settings."
@@ -4875 +4881 @@
 
 #. translators: 1: result type (e.g. Pages), 2: new position, 3: number of rows. Read out after reordering the All tab.
-#: fw-search-indexer.php:552
+#: fw-search-indexer.php:553
 #, php-format
 msgid "%1$s: position %2$d of %3$d"
@@ -5122 +5128 @@
 
 #. translators: %d: number of image results
-#: serp/search-results.php:353
+#: serp/search-results.php:379
 #, php-format
 msgid "See %d image"
@@ -5130 +5136 @@
 
 #. translators: %d: number of video results
-#: serp/search-results.php:357
+#: serp/search-results.php:383
 #, php-format
 msgid "See %d video"
@@ -5138 +5144 @@
 
 #. translators: %d: number of document results
-#: serp/search-results.php:361
+#: serp/search-results.php:387
 #, php-format
 msgid "See %d document"
@@ -5146 +5152 @@
 
 #. translators: %d: number of audio results
-#: serp/search-results.php:365
+#: serp/search-results.php:391
 #, php-format
 msgid "See %d audio file"
@@ -5154 +5160 @@
 
 #. translators: %d: number of product results
-#: serp/search-results.php:369
+#: serp/search-results.php:395
 #, php-format
 msgid "See %d product"
@@ -5161 +5167 @@
 msgstr[1] ""
 
-#: serp/search-results.php:375
+#: serp/search-results.php:401
 msgid "No pages found."
 msgstr ""
@@ -5209 +5215 @@
 
 #. translators: %s: suggested search term (link to corrected search)
-#: serp/serp-content-block.php:1740
+#: serp/serp-content-block.php:1743
 #, php-format
 msgid "Did you mean %s?"
@@ -5215 +5221 @@
 
 #. translators: %s: suggested search term (link to corrected search)
-#: serp/serp-content-block.php:1748
+#: serp/serp-content-block.php:1751
 #, php-format
 msgid "Search for %s"
 msgstr ""
 
-#: serp/serp-content-block.php:1767
+#: serp/serp-content-block.php:1770
 msgid "Search result types"
 msgstr ""
 
-#: serp/serp-content-block.php:1795
+#: serp/serp-content-block.php:1798
 msgid "Page not found; here are similar results."
 msgstr ""
 
 #. translators: %s: search query used to find similar results
-#: serp/serp-content-block.php:1799
+#: serp/serp-content-block.php:1802
 #, php-format
 msgid "Page not found; here are similar results for %s"
@@ -5235 +5241 @@
 
 #. translators: %s: search query
-#: serp/serp-content-block.php:1806
+#: serp/serp-content-block.php:1809
 #, php-format
 msgid "Search Results for: %s"
@@ -5241 +5247 @@
 
 #. translators: 1: matched query term, 2: canonical synonym
-#: serp/serp-content-block.php:1814
+#: serp/serp-content-block.php:1817
 #, php-format
 msgid " (%1$s is a synonym of %2$s)"
@@ -5247 +5253 @@
 
 #. translators: %d: number of results
-#: serp/serp-content-block.php:1842
-#: serp/serp-content-block.php:2730
+#: serp/serp-content-block.php:1845
+#: serp/serp-content-block.php:2733
 #, php-format
 msgid "%d Results"
 msgstr ""
 
-#: serp/serp-content-block.php:2766
+#: serp/serp-content-block.php:2769
 msgid "Continue to search results"
 msgstr ""
 
-#: serp/serp-content-block.php:2773
+#: serp/serp-content-block.php:2776
 msgid "Authentication failed"
 msgstr ""
 
-#: serp/serp-content-block.php:2775
+#: serp/serp-content-block.php:2778
 msgid "Rate limited"
 msgstr ""
 
-#: serp/serp-content-block.php:2777
+#: serp/serp-content-block.php:2780
 msgid "Could not reach search API"
 msgstr ""
 
-#: serp/serp-content-block.php:2779
+#: serp/serp-content-block.php:2782
 msgid "Search API error"
 msgstr ""
 
-#: serp/serp-content-block.php:2782
+#: serp/serp-content-block.php:2785
 msgid "Search is temporarily unavailable. Please try again later."
 msgstr ""
 
-#: serp/serp-content-block.php:2785
+#: serp/serp-content-block.php:2788
 msgid "Try a more specific search. Common words like “and” or “the” are skipped."
 msgstr ""
 
-#: serp/serp-content-block.php:2790
+#: serp/serp-content-block.php:2793
 msgid "Page not found. We couldn't find similar pages. Try searching above."
 msgstr ""
 
 #. translators: %s: the user's search query
-#: serp/serp-content-block.php:2794
+#: serp/serp-content-block.php:2797
 #, php-format
 msgid "No products matched “%s”."
 msgstr ""
 
-#: serp/serp-content-block.php:2797
+#: serp/serp-content-block.php:2800
 msgid "You’re searching products only. Try product names, styles, sizes, colors, or a budget — for example “red shoes size 7” or “red shoes under $100”."
 msgstr ""
@@ -5297 +5303 @@
 #. translators: %s: URL to sitewide search without product scope
 #. translators: %s: URL to sitewide search without scope
-#: serp/serp-content-block.php:2807
-#: serp/serp-content-block.php:2829
+#: serp/serp-content-block.php:2810
+#: serp/serp-content-block.php:2832
 #, php-format
 msgid "<a href=\"%s\">Search everything on this site</a>"
 msgstr ""
 
-#: serp/serp-content-block.php:2813
+#: serp/serp-content-block.php:2816
 msgid "this scope"
 msgstr ""
 
 #. translators: 1: search query, 2: scope phrase e.g. Products
-#: serp/serp-content-block.php:2816
+#: serp/serp-content-block.php:2819
 #, php-format
 msgid "No results for “%1$s” within %2$s."
 msgstr ""
 
-#: serp/serp-content-block.php:2833
+#: serp/serp-content-block.php:2836
 msgid "No results found."
 msgstr ""
 
-#: serp/serp-content-block.php:2863
+#: serp/serp-content-block.php:2866
 msgid "Media Preview"
 msgstr ""
 
-#: serp/serp-content-block.php:2865
+#: serp/serp-content-block.php:2868
 msgid "Expand to full screen"
 msgstr ""
 
-#: serp/serp-content-block.php:2868
+#: serp/serp-content-block.php:2871
 msgid "Close preview"
 msgstr ""
 
-#: serp/serp-content-block.php:2875
+#: serp/serp-content-block.php:2878
 msgid "Go to page"
 msgstr ""
 
-#: serp/serp-content-block.php:2876
+#: serp/serp-content-block.php:2879
 msgid "Open in new tab"
 msgstr ""
@@ -5380 +5386 @@
 msgid "Sorry, you are not allowed to do that."
 msgstr ""
+
+#: serp/smart-404.php:611
+msgid "Page not found"
+msgstr ""

searchus/trunk/readme.txt

@@ -4 +4 @@
 Requires at least: 5.9
 Tested up to: 7.1
-Stable tag: 3.1.3
+Stable tag: 3.1.4
 Requires PHP: 7.4
 License: GPLv2 or later
@@ -239 +239 @@
 
 == Changelog ==
+
+= 3.1.4 =
+* Improved: Search uses less of your server. Search boxes send visitors straight to the results page instead of through a redirect, and the results page loads its results with fewer WordPress database queries.
+* Improved: Crawlers, monitors and scripts that request a search results page get a short page right away instead of a full results page. Their searches were already not counted.
+* Improved: Smart 404 no longer searches for missing addresses made only of scanner words (such as /admin/login or /api/config), and more scanner and spam patterns in searches are ignored, so they don't count toward your plan.
 
 = 3.1.3 =
@@ -393 +398 @@
 == Upgrade Notice ==
 
+= 3.1.4 =
+Search uses less of the WordPress server resources and improved blocking of bot crawlers. No reindex required.
+
 = 3.1.3 =
 Theme compatibility improvements and a WordPress search fallback.

searchus/trunk/serp/page-template-404-recovery.php

@@ -78 +78 @@
         $recovery_query_text = '';
         $hide_recovery_detail = false;
-        if ( function_exists( 'fw_si_get_search_results' ) ) {
-                $payload = fw_si_get_search_results(
-                        $fw_si_recovery_query,
-                        [
-                                'apply_result_control' => true,
-                                'context'              => 'smart404',
-                        ]
-                );
+        // Same search the status code was decided from (smart-404.php, template_include).
+        if ( function_exists( 'fw_si_smart404_recovery' ) ) {
+                $fw_si_recovery       = fw_si_smart404_recovery();
+                $fw_si_recovery_query = $fw_si_recovery['query'];
+                $payload              = $fw_si_recovery['payload'];
+                $recovery_h1          = $fw_si_recovery['h1'];
         }
         // Analytics: do not call fw_si_analytics_record_search here — fw_si_render_serp_content_block()
         // records smart404 searches once when it renders (avoids double count with preloaded payload).
-        if ( is_array( $payload ) && function_exists( 'fw_si_build_smart404_h1' ) ) {
-                $recovery_h1 = fw_si_build_smart404_h1( $fw_si_recovery_query, $payload );
-        }
         if ( $recovery_h1 !== '' ) {
                 $recovery_query_text = $recovery_h1;

searchus/trunk/serp/search-results.php

@@ -206 +206 @@
   }
 
-  // Crawlers, monitors and scripts: no API call, so nothing is billed or recorded.
-  $fw_si_bot = function_exists( 'fw_si_is_bot_request' ) && fw_si_is_bot_request();
+  $fw_si_is_smart404 = ( $options['context'] ?? '' ) === 'smart404';
+  // Crawlers, monitors and scripts: no API call, so nothing is billed or recorded. Search engine and AI
+  // crawlers on a Smart 404 page, or on a results page on the Free plan, are the exception: the service
+  // never counts their searches and returns results when the request's address is the crawler's.
+  $fw_si_bot     = function_exists( 'fw_si_is_bot_request' ) && fw_si_is_bot_request();
+  $fw_si_crawler = ( $fw_si_bot && empty( $options['live_preview'] ) && function_exists( 'fw_si_search_crawler_name' )
+    && ( $fw_si_is_smart404 || ( function_exists( 'fw_si_crawlers_read_results_pages' ) && fw_si_crawlers_read_results_pages() ) ) )
+    ? fw_si_search_crawler_name() : '';
+  if ( $fw_si_crawler !== '' ) {
+    $fw_si_bot = false;
+  }
   // Only stop words: the API would drop every word and still bill the search, so skip it.
   // Live previews are exempt: their last word is a prefix ("and" may complete to "android").
   $fw_si_stop_only = empty( $options['live_preview'] ) && function_exists( 'fw_si_is_stop_words_only_query' ) && fw_si_is_stop_words_only_query( $api_query );
-  if ( $fw_si_bot || $fw_si_stop_only ) {
+  // A missing URL made only of scanner path words (/admin/login, /api/config) is a probe, not a visitor.
+  $fw_si_probe = $fw_si_is_smart404 && function_exists( 'fw_si_is_smart404_probe_query' ) && fw_si_is_smart404_probe_query( $api_query );
+  if ( $fw_si_bot || $fw_si_stop_only || $fw_si_probe ) {
     if ( $fw_si_bot ) {
       // A page cache must not keep this empty page and serve it to visitors.
@@ -254 +265 @@
           $api_options[ $live_key ] = $options[ $live_key ];
         }
+      }
+      if ( $fw_si_is_smart404 ) {
+        $api_options['source'] = 'smart404';
+      }
+      if ( $fw_si_crawler !== '' ) {
+        $api_options['crawler'] = $fw_si_crawler;
+        // REMOTE_ADDR only: forwarded-for headers are set by whoever sends the request.
+        $remote_addr = isset( $_SERVER['REMOTE_ADDR'] ) ? sanitize_text_field( wp_unslash( (string) $_SERVER['REMOTE_ADDR'] ) ) : '';
+        /**
+         * Address sent to the service to verify a crawler's search (default REMOTE_ADDR).
+         * Behind a proxy that does not restore the visitor's address, return the trusted client address here.
+         *
+         * @param string $remote_addr Request address.
+         */
+        $api_options['crawler_ip'] = (string) apply_filters( 'fw_si_search_crawler_ip', $remote_addr );
       }
       // Results page reached from live search (Enter / See all): same session, so it is not counted twice.
@@ -548 +574 @@
 
 /**
+ * Load the posts for a page of results, with their meta, terms and featured images, in a few queries
+ * instead of several per result.
+ *
+ * @param array $ids Result IDs (int or "media:attachment:123"); others are skipped.
+ */
+function fw_si_prime_result_post_caches( array $ids ): void {
+  if ( ! function_exists( '_prime_post_caches' ) || ! function_exists( 'fw_si_resolve_post_id' ) ) {
+    return;
+  }
+  $post_ids = [];
+  foreach ( $ids as $rid ) {
+    $pid = ( is_int( $rid ) || is_string( $rid ) ) ? fw_si_resolve_post_id( $rid ) : null;
+    if ( $pid !== null && $pid > 0 ) {
+      $post_ids[ $pid ] = $pid;
+    }
+  }
+  if ( ! $post_ids ) {
+    return;
+  }
+  _prime_post_caches( array_values( $post_ids ), true, true );
+  $thumbs = [];
+  foreach ( $post_ids as $pid ) {
+    $thumb = (int) get_post_meta( $pid, '_thumbnail_id', true );
+    if ( $thumb > 0 ) {
+      $thumbs[ $thumb ] = $thumb;
+    }
+  }
+  if ( $thumbs ) {
+    _prime_post_caches( array_values( $thumbs ), false, true );
+  }
+}
+
+/**
  * Convert payload IDs to WP_Post array for the_posts filter.
  *
@@ -554 +613 @@
  */
 function fw_si_payload_ids_to_posts( array $ids ): array {
+  fw_si_prime_result_post_caches( $ids );
   $posts = [];
   foreach ( $ids as $pid ) {

searchus/trunk/serp/serp-content-block.php

@@ -1646 +1646 @@
         }
         $qt_build_start = $qt_serp_debug ? microtime( true ) : 0;
+        if ( function_exists( 'fw_si_prime_result_post_caches' ) ) {
+          fw_si_prime_result_post_caches( $is_404_recovery ? array_slice( $slice, 0, 12 ) : $slice );
+        }
         foreach ( $slice as $rid ) {
           if ( $is_404_recovery && count( $serp_display ) >= 6 ) {

searchus/trunk/serp/smart-404.php

@@ -74 +74 @@
 
 /**
+ * True when the missing URL ends in an image, style, script, font or media file extension.
+ *
+ * @return bool
+ */
+function fw_si_smart404_path_is_static_file(): bool {
+        $uri  = isset( $_SERVER['REQUEST_URI'] ) ? sanitize_text_field( wp_unslash( (string) $_SERVER['REQUEST_URI'] ) ) : '';
+        $path = (string) wp_parse_url( $uri, PHP_URL_PATH );
+        return (bool) preg_match( '/\.(?:jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|css|js|mjs|map|json|xml|txt|woff2?|ttf|otf|eot|mp4|webm|mov|mp3|wav|ogg|zip|gz)$/i', $path );
+}
+
+/**
+ * True when the missing URL is being opened as a page by a visitor's browser or a search engine or AI crawler.
+ * Those crawlers see the recovery page with similar results and the attribution. Other crawlers and scripts,
+ * and missing images or scripts, get the theme's 404.
+ *
+ * @return bool
+ */
+function fw_si_smart404_request_wants_page(): bool {
+        if ( function_exists( 'fw_si_is_bot_request' ) && fw_si_is_bot_request() ) {
+                // Crawlers often send Accept: */*, so file type comes from the URL.
+                return function_exists( 'fw_si_search_crawler_name' ) && fw_si_search_crawler_name() !== '' && ! fw_si_smart404_path_is_static_file();
+        }
+        $dest = isset( $_SERVER['HTTP_SEC_FETCH_DEST'] ) ? strtolower( sanitize_text_field( wp_unslash( (string) $_SERVER['HTTP_SEC_FETCH_DEST'] ) ) ) : '';
+        if ( $dest !== '' ) {
+                return in_array( $dest, array( 'document', 'iframe', 'frame' ), true );
+        }
+        // Browsers without Sec-Fetch-Dest still list text/html when opening a page.
+        $accept = isset( $_SERVER['HTTP_ACCEPT'] ) ? strtolower( sanitize_text_field( wp_unslash( (string) $_SERVER['HTTP_ACCEPT'] ) ) ) : '';
+        return strpos( $accept, 'text/html' ) !== false;
+}
+
+/**
  * Derive a search query from the 404 URL.
  *
@@ -301 +333 @@
         }
         $serving_recovery = ! empty( $GLOBALS['fw_si_serving_404_recovery'] );
-        $is_404_recovery = $serving_recovery || ( is_404() && (bool) get_option( 'fw_si_smart_404_enabled', true ) && fw_si_derive_query_from_404_url() !== '' );
+        $is_404_recovery = $serving_recovery || ( is_404() && (bool) get_option( 'fw_si_smart_404_enabled', true ) && fw_si_smart404_request_wants_page() && fw_si_derive_query_from_404_url() !== '' );
         if ( ! $is_404_recovery ) {
                 return;
@@ -380 +412 @@
                 return;
         }
+        if ( ! fw_si_smart404_request_wants_page() ) {
+                if ( defined( 'WP_DEBUG' ) && WP_DEBUG ) {
+                        fw_si_smart404_debug_bail( 'not_a_visitor_page_request' );
+                }
+                return;
+        }
         $query = fw_si_derive_query_from_404_url();
         if ( $query === '' ) {
@@ -422 +460 @@
 
 /**
+ * Search, payload and heading for this request's Smart 404 page. The search runs once, before the page
+ * header is sent, so the HTTP status can say whether similar results were found.
+ *
+ * @return array{query:string,payload:array,h1:string,found:bool}
+ */
+function fw_si_smart404_recovery(): array {
+        static $recovery = null;
+        if ( $recovery !== null ) {
+                return $recovery;
+        }
+        $query   = fw_si_derive_query_from_404_url();
+        $payload = ( $query !== '' && function_exists( 'fw_si_get_search_results' ) )
+                ? fw_si_get_search_results(
+                        $query,
+                        [
+                                'apply_result_control' => true,
+                                'context'              => 'smart404',
+                        ]
+                )
+                : [];
+        $payload  = is_array( $payload ) ? $payload : [];
+        $recovery = [
+                'query'   => $query,
+                'payload' => $payload,
+                'h1'      => $payload ? fw_si_build_smart404_h1( $query, $payload ) : '',
+                // Similar means a word from the missing URL is in the shown results' titles, not just any fuzzy match.
+                'found'   => fw_si_smart404_url_words_in_titles( $query, $payload ) && fw_si_smart404_payload_has_results( $payload ),
+        ];
+        return $recovery;
+}
+
+/**
+ * True when a word from the missing URL (plural or not) appears in a title of the first 12 results.
+ *
+ * @param string $query   Query derived from the URL.
+ * @param array  $payload Search payload.
+ * @return bool
+ */
+function fw_si_smart404_url_words_in_titles( string $query, array $payload ): bool {
+        // Singular forms a word may stand for: "hoodies" → hoodie, "boxes" → box, "berries" → berry.
+        $forms = static function ( string $word ): array {
+                $out = [ $word ];
+                $len = strlen( $word );
+                if ( $len > 3 && substr( $word, -1 ) === 's' ) {
+                        $out[] = substr( $word, 0, -1 );
+                }
+                if ( $len > 4 && substr( $word, -2 ) === 'es' ) {
+                        $out[] = substr( $word, 0, -2 );
+                }
+                if ( $len > 4 && substr( $word, -3 ) === 'ies' ) {
+                        $out[] = substr( $word, 0, -3 ) . 'y';
+                }
+                return $out;
+        };
+        $filler    = array_flip( array( '404', 'www', 'http', 'https', 'html', 'htm', 'php', 'com', 'net', 'org', 'bad', 'url', 'page', 'post', 'search', 'old', 'new', 'the', 'and', 'for', 'with' ) );
+        $url_words = [];
+        foreach ( preg_split( '/[^a-z0-9]+/', strtolower( $query ), -1, PREG_SPLIT_NO_EMPTY ) ?: [] as $word ) {
+                if ( strlen( $word ) >= 2 && ! isset( $filler[ $word ] ) ) {
+                        foreach ( $forms( $word ) as $form ) {
+                                $url_words[ $form ] = true;
+                        }
+                }
+        }
+        if ( ! $url_words || empty( $payload['all'] ) || ! is_array( $payload['all'] ) ) {
+                return false;
+        }
+        $titles = isset( $payload['titles'] ) && is_array( $payload['titles'] ) ? $payload['titles'] : [];
+        foreach ( array_slice( $payload['all'], 0, 12 ) as $rid ) {
+                $title = isset( $titles[ $rid ] ) ? (string) $titles[ $rid ] : '';
+                if ( $title === '' && function_exists( 'fw_si_resolve_post_id' ) && ( is_int( $rid ) || is_string( $rid ) ) ) {
+                        $post_id = fw_si_resolve_post_id( $rid );
+                        $title   = $post_id ? (string) get_the_title( $post_id ) : '';
+                }
+                foreach ( preg_split( '/[^a-z0-9]+/', strtolower( $title ), -1, PREG_SPLIT_NO_EMPTY ) ?: [] as $word ) {
+                        foreach ( $forms( $word ) as $form ) {
+                                if ( isset( $url_words[ $form ] ) ) {
+                                        return true;
+                                }
+                        }
+                }
+        }
+        return false;
+}
+
+/**
+ * True when the payload has at least one result the Smart 404 page shows.
+ *
+ * @param array $payload Search payload.
+ * @return bool
+ */
+function fw_si_smart404_payload_has_results( array $payload ): bool {
+        if ( ! empty( $payload['_fw_error'] ) || empty( $payload['all'] ) || ! is_array( $payload['all'] ) ) {
+                return false;
+        }
+        foreach ( array_slice( $payload['all'], 0, 50 ) as $rid ) {
+                if ( function_exists( 'fw_si_result_control_is_synthetic_result_id' ) && fw_si_result_control_is_synthetic_result_id( $rid ) ) {
+                        return true;
+                }
+                $post_id = ( function_exists( 'fw_si_resolve_post_id' ) && ( is_int( $rid ) || is_string( $rid ) ) ) ? fw_si_resolve_post_id( $rid ) : null;
+                $post    = $post_id ? get_post( $post_id ) : null;
+                if ( $post instanceof WP_Post && ( ! function_exists( 'fw_si_serp_post_allowed_for_404_recovery' ) || fw_si_serp_post_allowed_for_404_recovery( $post ) ) ) {
+                        return true;
+                }
+        }
+        return false;
+}
+
+/**
+ * This request's URL (no query string), the Smart 404 page's own address.
+ *
+ * @return string
+ */
+function fw_si_smart404_page_url(): string {
+        $uri  = isset( $_SERVER['REQUEST_URI'] ) ? esc_url_raw( wp_unslash( (string) $_SERVER['REQUEST_URI'] ) ) : '/';
+        $path = (string) wp_parse_url( $uri, PHP_URL_PATH );
+        $home = wp_parse_url( home_url( '/' ) );
+        if ( ! is_array( $home ) || empty( $home['host'] ) ) {
+                return '';
+        }
+        $port = isset( $home['port'] ) ? ':' . (int) $home['port'] : '';
+        return esc_url_raw( ( $home['scheme'] ?? 'https' ) . '://' . $home['host'] . $port . ( $path !== '' ? $path : '/' ) );
+}
+
+/**
+ * Head tags for the Smart 404 page. The page is built on the results page with the home page's header, so
+ * WordPress and SEO plugins would otherwise name the home page as canonical and use the site name as title.
+ * With results: canonical is the page's own URL, so search engines can index it. Without: no canonical.
+ *
+ * @param array $recovery fw_si_smart404_recovery().
+ */
+function fw_si_smart404_page_head( array $recovery ): void {
+        remove_action( 'wp_head', 'rel_canonical' );
+        remove_action( 'wp_head', 'wp_shortlink_wp_head', 10 );
+        $canonical = ! empty( $recovery['found'] ) ? fw_si_smart404_page_url() : '';
+        $canonical_filter = static function () use ( $canonical ) {
+                return $canonical !== '' ? $canonical : false;
+        };
+        add_filter( 'wpseo_canonical', $canonical_filter, 999 );
+        add_filter( 'rank_math/frontend/canonical', $canonical_filter, 999 );
+        if ( $canonical !== '' && ! defined( 'WPSEO_VERSION' ) && ! defined( 'RANK_MATH_VERSION' ) ) {
+                add_action(
+                        'wp_head',
+                        static function () use ( $canonical ) {
+                                echo '<link rel="canonical" href="' . esc_url( $canonical ) . '" />' . "\n";
+                        },
+                        10
+                );
+        }
+        $heading = ! empty( $recovery['found'] ) && isset( $recovery['h1'] ) ? trim( (string) $recovery['h1'] ) : '';
+        $title   = ( $heading !== '' ? $heading : __( 'Page not found', 'searchus' ) ) . ' | ' . get_bloginfo( 'name', 'display' );
+        $title_filter = static function () use ( $title ) {
+                return esc_html( $title );
+        };
+        add_filter( 'pre_get_document_title', $title_filter, 999 );
+        add_filter( 'wpseo_title', $title_filter, 999 );
+        add_filter( 'rank_math/frontend/title', $title_filter, 999 );
+}
+
+/**
  * template_include: Serve recovery template when we replaced 404 with Search page (theme header renders).
  * Runs at 5 so we override before search-page.php (25) when serving Smart 404.
@@ -441 +638 @@
         $recovery_template = FW_SI_PLUGIN_DIR . '/serp/page-template-404-recovery.php';
         if ( file_exists( $recovery_template ) ) {
-                status_header( 404 );
+                // A page of similar results is content search engines may index; a page with none is still a 404.
+                $recovery = fw_si_smart404_recovery();
+                status_header( $recovery['found'] ? 200 : 404 );
                 nocache_headers();
+                fw_si_smart404_page_head( $recovery );
                 return $recovery_template;
         }