From e85869f3a4f1b2f03da84f9eb5a0449bbe1a91cd Mon Sep 17 00:00:00 2001 From: Vedanshu Jain Date: Tue, 10 Sep 2024 04:50:25 +0530 Subject: [PATCH] Add phone to searchable FTS fields for order addresses. (#51065) This PR adds the phone number to the FTS index on the order address table, and adds the migration routines so that we can recreate the index for shops that already have this enabled. Additionally, it also removes support for relevancy operators to make search performant. Fixes #51213 --- .../woocommerce/changelog/enhance-fts-index | 4 ++ .../woocommerce/changelog/enhance-fts-index-2 | 4 ++ .../woocommerce/includes/class-wc-install.php | 3 ++ ...rest-system-status-tools-v2-controller.php | 15 +++++++ .../includes/wc-update-functions.php | 20 +++++++++ .../Orders/CustomOrdersTableController.php | 41 +++++++++++++++-- .../Orders/OrdersTableSearchQuery.php | 17 +++++-- .../src/Internal/Utilities/DatabaseUtil.php | 44 ++++++++++++++++++- .../Internal/Utilities/DatabaseUtilTest.php | 24 ++++++++++ 9 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 plugins/woocommerce/changelog/enhance-fts-index create mode 100644 plugins/woocommerce/changelog/enhance-fts-index-2 diff --git a/plugins/woocommerce/changelog/enhance-fts-index b/plugins/woocommerce/changelog/enhance-fts-index new file mode 100644 index 00000000000..279ab53ddaa --- /dev/null +++ b/plugins/woocommerce/changelog/enhance-fts-index @@ -0,0 +1,4 @@ +Significance: patch +Type: enhancement + +Add phone number to FTS index to improve order searchability. diff --git a/plugins/woocommerce/changelog/enhance-fts-index-2 b/plugins/woocommerce/changelog/enhance-fts-index-2 new file mode 100644 index 00000000000..91e6392ec1d --- /dev/null +++ b/plugins/woocommerce/changelog/enhance-fts-index-2 @@ -0,0 +1,4 @@ +Significance: patch +Type: performance + +Remove relevancy ranking from FTS queries to improve performance. diff --git a/plugins/woocommerce/includes/class-wc-install.php b/plugins/woocommerce/includes/class-wc-install.php index e6d591d8d11..002844d49c4 100644 --- a/plugins/woocommerce/includes/class-wc-install.php +++ b/plugins/woocommerce/includes/class-wc-install.php @@ -266,6 +266,9 @@ class WC_Install { 'wc_update_930_add_woocommerce_coming_soon_option', 'wc_update_930_migrate_user_meta_for_launch_your_store_tour', ), + '9.4.0' => array( + 'wc_update_940_add_phone_to_order_address_fts_index', + ), ); /** diff --git a/plugins/woocommerce/includes/rest-api/Controllers/Version2/class-wc-rest-system-status-tools-v2-controller.php b/plugins/woocommerce/includes/rest-api/Controllers/Version2/class-wc-rest-system-status-tools-v2-controller.php index 822b2a651f1..fd50fc10a88 100644 --- a/plugins/woocommerce/includes/rest-api/Controllers/Version2/class-wc-rest-system-status-tools-v2-controller.php +++ b/plugins/woocommerce/includes/rest-api/Controllers/Version2/class-wc-rest-system-status-tools-v2-controller.php @@ -8,6 +8,9 @@ * @since 3.0.0 */ +use Automattic\WooCommerce\Internal\DataStores\Orders\CustomOrdersTableController; +use Automattic\WooCommerce\Internal\Utilities\DatabaseUtil; + defined( 'ABSPATH' ) || exit; /** @@ -217,6 +220,11 @@ class WC_REST_System_Status_Tools_V2_Controller extends WC_REST_Controller { __( 'This tool will update your WooCommerce database to the latest version. Please ensure you make sufficient backups before proceeding.', 'woocommerce' ) ), ), + 'recreate_order_address_fts_index' => array( + 'name' => __( 'Re-create Order Address FTS index', 'woocommerce' ), + 'button' => __( 'Recreate index', 'woocommerce' ), + 'desc' => __( 'This tool will recreate the full text search index for order addresses. If the index does not exist, it will try to create it.', 'woocommerce' ), + ), ); if ( method_exists( 'WC_Install', 'verify_base_tables' ) ) { $tools['verify_db_tables'] = array( @@ -598,6 +606,13 @@ class WC_REST_System_Status_Tools_V2_Controller extends WC_REST_Controller { } break; + case 'recreate_order_address_fts_index': + $hpos_controller = wc_get_container()->get( CustomOrdersTableController::class ); + $results = $hpos_controller->recreate_order_address_fts_index(); + $ran = $results['status']; + $message = $results['message']; + break; + default: $tools = $this->get_tools(); if ( isset( $tools[ $tool ]['callback'] ) ) { diff --git a/plugins/woocommerce/includes/wc-update-functions.php b/plugins/woocommerce/includes/wc-update-functions.php index 34b8efdbd26..e4f880a2a76 100644 --- a/plugins/woocommerce/includes/wc-update-functions.php +++ b/plugins/woocommerce/includes/wc-update-functions.php @@ -24,12 +24,14 @@ use Automattic\WooCommerce\Database\Migrations\MigrationHelper; use Automattic\WooCommerce\Internal\Admin\Marketing\MarketingSpecs; use Automattic\WooCommerce\Internal\Admin\Notes\WooSubscriptionsNotes; use Automattic\WooCommerce\Internal\AssignDefaultCategory; +use Automattic\WooCommerce\Internal\DataStores\Orders\CustomOrdersTableController; use Automattic\WooCommerce\Internal\DataStores\Orders\DataSynchronizer; use Automattic\WooCommerce\Internal\DataStores\Orders\OrdersTableDataStore; use Automattic\WooCommerce\Internal\ProductAttributesLookup\DataRegenerator; use Automattic\WooCommerce\Internal\ProductAttributesLookup\LookupDataStore; use Automattic\WooCommerce\Internal\ProductDownloads\ApprovedDirectories\Register as Download_Directories; use Automattic\WooCommerce\Internal\ProductDownloads\ApprovedDirectories\Synchronize as Download_Directories_Sync; +use Automattic\WooCommerce\Internal\Utilities\DatabaseUtil; use Automattic\WooCommerce\Utilities\StringUtil; /** @@ -2852,3 +2854,21 @@ function wc_update_930_migrate_user_meta_for_launch_your_store_tour() { ) ); } + +/** + * Recreate FTS index if it already exists, so that phone number can be added to the index. + */ +function wc_update_940_add_phone_to_order_address_fts_index(): void { + $fts_already_exists = get_option( CustomOrdersTableController::HPOS_FTS_ADDRESS_INDEX_CREATED_OPTION ) === 'yes'; + if ( ! $fts_already_exists ) { + return; + } + + $hpos_controller = wc_get_container()->get( CustomOrdersTableController::class ); + $result = $hpos_controller->recreate_order_address_fts_index(); + if ( ! $result['status'] ) { + if ( class_exists( 'WC_Admin_Settings ' ) ) { + WC_Admin_Settings::add_error( $result['message'] ); + } + } +} diff --git a/plugins/woocommerce/src/Internal/DataStores/Orders/CustomOrdersTableController.php b/plugins/woocommerce/src/Internal/DataStores/Orders/CustomOrdersTableController.php index 3345f6a61e5..cd646c03269 100644 --- a/plugins/woocommerce/src/Internal/DataStores/Orders/CustomOrdersTableController.php +++ b/plugins/woocommerce/src/Internal/DataStores/Orders/CustomOrdersTableController.php @@ -345,9 +345,9 @@ class CustomOrdersTableController { // Check again to see if index was actually created. if ( $this->db_util->fts_index_on_order_address_table_exists() ) { - update_option( self::HPOS_FTS_ADDRESS_INDEX_CREATED_OPTION, 'yes', true ); + update_option( self::HPOS_FTS_ADDRESS_INDEX_CREATED_OPTION, 'yes', false ); } else { - update_option( self::HPOS_FTS_ADDRESS_INDEX_CREATED_OPTION, 'no', true ); + update_option( self::HPOS_FTS_ADDRESS_INDEX_CREATED_OPTION, 'no', false ); if ( class_exists( 'WC_Admin_Settings ' ) ) { WC_Admin_Settings::add_error( __( 'Failed to create FTS index on address table', 'woocommerce' ) ); } @@ -359,15 +359,48 @@ class CustomOrdersTableController { // Check again to see if index was actually created. if ( $this->db_util->fts_index_on_order_item_table_exists() ) { - update_option( self::HPOS_FTS_ORDER_ITEM_INDEX_CREATED_OPTION, 'yes', true ); + update_option( self::HPOS_FTS_ORDER_ITEM_INDEX_CREATED_OPTION, 'yes', false ); } else { - update_option( self::HPOS_FTS_ORDER_ITEM_INDEX_CREATED_OPTION, 'no', true ); + update_option( self::HPOS_FTS_ORDER_ITEM_INDEX_CREATED_OPTION, 'no', false ); if ( class_exists( 'WC_Admin_Settings ' ) ) { WC_Admin_Settings::add_error( __( 'Failed to create FTS index on order item table', 'woocommerce' ) ); } } } + /** + * Recreate order addresses FTS index. Useful when updating to 9.4 when phone number was added to index, or when other recreating index is needed. + * + * @since 9.4.0. + * + * @return array Array with keys status (bool) and message (string). + */ + public function recreate_order_address_fts_index(): array { + $this->db_util->drop_fts_index_order_address_table(); + if ( $this->db_util->fts_index_on_order_address_table_exists() ) { + return array( + 'status' => false, + 'message' => __( 'Failed to modify existing FTS index. Please go to WooCommerce > Status > Tools and run the "Re-create Order Address FTS index" tool.', 'woocommerce' ), + ); + } else { + update_option( self::HPOS_FTS_ADDRESS_INDEX_CREATED_OPTION, 'no', false ); + } + + $this->db_util->create_fts_index_order_address_table(); + if ( ! $this->db_util->fts_index_on_order_address_table_exists() ) { + return array( + 'status' => false, + 'message' => __( 'Failed to create FTS index on order address table. Please go to WooCommerce > Status > Tools and run the "Re-create Order Address FTS index" tool.', 'woocommerce' ), + ); + } else { + update_option( self::HPOS_FTS_ADDRESS_INDEX_CREATED_OPTION, 'yes', false ); + return array( + 'status' => true, + 'message' => __( 'FTS index recreated.', 'woocommerce' ), + ); + } + } + /** * Handler for the setting pre-update hook. * We use it to verify that authoritative orders table switch doesn't happen while sync is pending. diff --git a/plugins/woocommerce/src/Internal/DataStores/Orders/OrdersTableSearchQuery.php b/plugins/woocommerce/src/Internal/DataStores/Orders/OrdersTableSearchQuery.php index 73b6c9e31aa..08b3b112767 100644 --- a/plugins/woocommerce/src/Internal/DataStores/Orders/OrdersTableSearchQuery.php +++ b/plugins/woocommerce/src/Internal/DataStores/Orders/OrdersTableSearchQuery.php @@ -2,6 +2,7 @@ namespace Automattic\WooCommerce\Internal\DataStores\Orders; +use Automattic\WooCommerce\Internal\Utilities\DatabaseUtil; use Exception; /** @@ -238,6 +239,7 @@ class OrdersTableSearchQuery { */ private function get_where_for_products() { global $wpdb; + $db_util = wc_get_container()->get( DatabaseUtil::class ); $items_table = $this->query->get_table_name( 'items' ); $orders_table = $this->query->get_table_name( 'orders' ); $fts_enabled = get_option( CustomOrdersTableController::HPOS_FTS_INDEX_OPTION ) === 'yes' && get_option( CustomOrdersTableController::HPOS_FTS_ORDER_ITEM_INDEX_CREATED_OPTION ) === 'yes'; @@ -251,7 +253,7 @@ $orders_table.id in ( MATCH ( search_query_items.order_item_name ) AGAINST ( %s IN BOOLEAN MODE ) ) ", - '*' . $wpdb->esc_like( $this->search_term ) . '*' + $wpdb->esc_like( $db_util->sanitise_boolean_fts_search_term( $this->search_term ) ), ); // phpcs:enable } @@ -279,18 +281,27 @@ $orders_table.id in ( $order_table = $this->query->get_table_name( 'orders' ); $address_table = $this->query->get_table_name( 'addresses' ); + $db_util = wc_get_container()->get( DatabaseUtil::class ); + $fts_enabled = get_option( CustomOrdersTableController::HPOS_FTS_INDEX_OPTION ) === 'yes' && get_option( CustomOrdersTableController::HPOS_FTS_ADDRESS_INDEX_CREATED_OPTION ) === 'yes'; if ( $fts_enabled ) { + $matchers = "$address_table.first_name, $address_table.last_name, $address_table.company, $address_table.address_1, $address_table.address_2, $address_table.city, $address_table.state, $address_table.postcode, $address_table.country, $address_table.email"; + + // Support for phone was added in 9.4. + if ( version_compare( get_option( 'woocommerce_db_version' ), '9.4.0', '>=' ) ) { + $matchers .= ", $address_table.phone"; + } + // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- $order_table and $address_table are hardcoded. return $wpdb->prepare( " $order_table.id IN ( SELECT order_id FROM $address_table WHERE - MATCH( $address_table.first_name, $address_table.last_name, $address_table.company, $address_table.address_1, $address_table.address_2, $address_table.city, $address_table.state, $address_table.postcode, $address_table.country, $address_table.email ) AGAINST ( %s IN BOOLEAN MODE ) + MATCH( $matchers ) AGAINST ( %s IN BOOLEAN MODE ) ) ", - '*' . $wpdb->esc_like( $this->search_term ) . '*' + $wpdb->esc_like( $db_util->sanitise_boolean_fts_search_term( $this->search_term ) ) ); // phpcs:enable } diff --git a/plugins/woocommerce/src/Internal/Utilities/DatabaseUtil.php b/plugins/woocommerce/src/Internal/Utilities/DatabaseUtil.php index ab9dd8d26f5..86b7bf8140c 100644 --- a/plugins/woocommerce/src/Internal/Utilities/DatabaseUtil.php +++ b/plugins/woocommerce/src/Internal/Utilities/DatabaseUtil.php @@ -357,7 +357,49 @@ $on_duplicate_clause global $wpdb; $address_table = $wpdb->prefix . 'wc_order_addresses'; // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- $address_table is hardcoded. - $wpdb->query( "CREATE FULLTEXT INDEX order_addresses_fts ON $address_table (first_name, last_name, company, address_1, address_2, city, state, postcode, country, email)" ); + $wpdb->query( "CREATE FULLTEXT INDEX order_addresses_fts ON $address_table (first_name, last_name, company, address_1, address_2, city, state, postcode, country, email, phone)" ); + } + + /** + * Helper method to drop the fulltext index on order address table. + * + * @since 9.4.0 + * + * @return void + */ + public function drop_fts_index_order_address_table(): void { + global $wpdb; + $address_table = $wpdb->prefix . 'wc_order_addresses'; + // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- $address_table is hardcoded. + $wpdb->query( "ALTER TABLE $address_table DROP INDEX order_addresses_fts;" ); + } + + /** + * Sanitize FTS Search params to remove relevancy operators for performance, and add partial matches. Useful when the sorting is already happening based on some other conditions, so relevancy calculation is not needed. + * + * @since 9.4.0 + * + * @param string $param Search term. + * + * @return string Sanitized search term. + */ + public function sanitise_boolean_fts_search_term( string $param ): string { + // Remove any operator to prevent incorrect query and fatals, such as search starting with `++`. We can allow this in the future if we have proper validation for FTS search operators. + // Space is allowed to provide multiple words. + $sanitized_param = preg_replace( '/[^\p{L}\p{N}_]+/u', ' ', $param ); + if ( $sanitized_param !== $param ) { + $param = str_replace( '"', '', $param ); + return '"' . $param . '"'; + } + // Split the search phrase into words so that we can add operators when needed. + $words = explode( ' ', $param ); + $sanitized_words = array(); + foreach ( $words as $word ) { + // Add `*` as suffix to every term so that partial matches happens. + $word = $word . '*'; + $sanitized_words[] = $word; + } + return implode( ' ', $sanitized_words ); } /** diff --git a/plugins/woocommerce/tests/php/src/Internal/Utilities/DatabaseUtilTest.php b/plugins/woocommerce/tests/php/src/Internal/Utilities/DatabaseUtilTest.php index 6f98bac9163..d83d37b3830 100644 --- a/plugins/woocommerce/tests/php/src/Internal/Utilities/DatabaseUtilTest.php +++ b/plugins/woocommerce/tests/php/src/Internal/Utilities/DatabaseUtilTest.php @@ -131,4 +131,28 @@ class DatabaseUtilTest extends \WC_Unit_Test_Case { // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared -- Hardcoded query. $this->assertEquals( 'Updated Content', $wpdb->get_var( $content_query ) ); } + + /** + * @testDox Test that sanitise_boolean_fts_search_term() works as expected. + */ + public function test_sanitise_boolean_fts_search_term(): void { + $terms_sanitized_mapping = array( + // Normal terms are suffixed with wildcard. + 'abc' => 'abc*', + 'abc def' => 'abc* def*', + // Terms containing operators are quoted. + '+abc -def' => '"+abc -def"', + '++abc-def' => '"++abc-def"', + 'abc (>def '"abc (>def '"abc def"', + 'abc*' => '"abc*"', + // Some edge cases. + '' => '*', + '"' => '""', + ); + + foreach ( $terms_sanitized_mapping as $term => $expected ) { + $this->assertEquals( $expected, $this->sut->sanitise_boolean_fts_search_term( $term ) ); + } + } }