Fix utf8mb4_0900_ai_ci collation in tables created by WordPress

.github/workflows/phpunit-tests.yml

@@ -170,16 +170,24 @@ jobs:
         run: cargo build --release
         working-directory: packages/php-ext-wp-mysql-parser
 
+      # Load the extension via an INI file, so that it also applies to child
+      # processes, such as PHPUnit tests that run in a separate process.
+      - name: Enable parser extension
+        if: matrix.extension
+        run: |
+          echo "extension=$GITHUB_WORKSPACE/packages/php-ext-wp-mysql-parser/target/release/libwp_mysql_parser.so" \
+            | sudo tee "$(php -r 'echo PHP_CONFIG_FILE_SCAN_DIR;')/wp-mysql-parser.ini"
+
       - name: Verify native parser extension
         if: matrix.extension
-        run: php -d extension="$GITHUB_WORKSPACE/packages/php-ext-wp-mysql-parser/target/release/libwp_mysql_parser.so" tests/tools/verify-native-parser-extension.php
+        run: php tests/tools/verify-native-parser-extension.php
         working-directory: packages/mysql-on-sqlite
 
       - name: Run PHPUnit suite with parser extension
         if: matrix.extension
         env:
           WP_SQLITE_REQUIRE_NATIVE_PARSER_EXTENSION: '1'
-        run: php -d extension="$GITHUB_WORKSPACE/packages/php-ext-wp-mysql-parser/target/release/libwp_mysql_parser.so" ./vendor/bin/phpunit -c ./phpunit.xml.dist
+        run: php ./vendor/bin/phpunit -c ./phpunit.xml.dist
         working-directory: packages/mysql-on-sqlite
 
       - name: Run PHPUnit suite

packages/mysql-on-sqlite/src/sqlite/class-wp-sqlite-information-schema-builder.php

@@ -450,6 +450,7 @@ public function get_computed_information_schema_table_definition( string $table_
 					('utf8_unicode_ci', 'utf8', 192, '', 'Yes', 8, 'PAD SPACE'),
 					('utf8mb4_bin', 'utf8mb4', 46, '', 'Yes', 1, 'PAD SPACE'),
 					('utf8mb4_unicode_ci', 'utf8mb4', 224, '', 'Yes', 8, 'PAD SPACE'),
+					('utf8mb4_unicode_520_ci', 'utf8mb4', 246, '', 'Yes', 8, 'PAD SPACE'),
 					('utf8mb4_0900_ai_ci', 'utf8mb4', 255, 'Yes', 'Yes', 0, 'NO PAD')
 				)";
 			default:

packages/mysql-on-sqlite/tests/WP_MySQL_On_SQLite_Metadata_Tests.php

@@ -186,6 +186,15 @@ public function testInfromationSchemaCollations(): void {
 					'SORTLEN'            => '1',
 					'PAD_ATTRIBUTE'      => 'PAD SPACE',
 				),
+				(object) array(
+					'COLLATION_NAME'     => 'utf8mb4_unicode_520_ci',
+					'CHARACTER_SET_NAME' => 'utf8mb4',
+					'ID'                 => '246',
+					'IS_DEFAULT'         => '',
+					'IS_COMPILED'        => 'Yes',
+					'SORTLEN'            => '8',
+					'PAD_ATTRIBUTE'      => 'PAD SPACE',
+				),
 				(object) array(
 					'COLLATION_NAME'     => 'utf8mb4_unicode_ci',
 					'CHARACTER_SET_NAME' => 'utf8mb4',
@@ -516,19 +525,20 @@ public function testShowCollation(): void {
 		// Simple.
 		$this->assertQuery( 'SHOW COLLATION' );
 		$actual = $this->last_result;
-		$this->assertCount( 7, $actual );
+		$this->assertCount( 8, $actual );
 		$this->assertEquals( 'binary', $actual[0]->Collation );
 		$this->assertEquals( 'utf8_bin', $actual[1]->Collation );
 		$this->assertEquals( 'utf8_general_ci', $actual[2]->Collation );
 		$this->assertEquals( 'utf8_unicode_ci', $actual[3]->Collation );
 		$this->assertEquals( 'utf8mb4_bin', $actual[4]->Collation );
 		$this->assertEquals( 'utf8mb4_unicode_ci', $actual[5]->Collation );
-		$this->assertEquals( 'utf8mb4_0900_ai_ci', $actual[6]->Collation );
+		$this->assertEquals( 'utf8mb4_unicode_520_ci', $actual[6]->Collation );
+		$this->assertEquals( 'utf8mb4_0900_ai_ci', $actual[7]->Collation );
 
 		// With LIKE clause.
 		$this->assertQuery( "SHOW COLLATION LIKE 'utf8%'" );
 		$actual = $this->last_result;
-		$this->assertCount( 6, $actual );
+		$this->assertCount( 7, $actual );
 		$this->assertEquals( 'utf8_bin', $actual[0]->Collation );
 		$this->assertEquals( 'utf8_general_ci', $actual[1]->Collation );
 		$this->assertEquals( 'utf8_unicode_ci', $actual[2]->Collation );

packages/mysql-on-sqlite/tests/WP_MySQL_On_SQLite_Tests.php

@@ -4083,7 +4083,7 @@ public function testFoundRowsWithoutSqlCalcFoundRows(): void {
 		// SHOW COLLATION
 		$this->assertQuery( 'SHOW COLLATION' );
 		$result = $this->assertQuery( 'SELECT FOUND_ROWS()' );
-		$this->assertSame( '7', $result[0]->{'FOUND_ROWS()'} );
+		$this->assertSame( '8', $result[0]->{'FOUND_ROWS()'} );
 
 		// SHOW DATABASES
 		$this->assertQuery( 'SHOW DATABASES' );

packages/mysql-on-sqlite/tests/WP_SQLite_DB_Tests.php

@@ -162,4 +162,62 @@ public function __construct( ?WP_MySQL_On_SQLite $driver ) {
 		$this->expectExceptionMessage( 'Cannot escape data without an active database connection.' );
 		$wpdb->_real_escape( 'value' );
 	}
+
+	/**
+	 * Each data set runs in a separate process to define its own constants.
+	 *
+	 * @dataProvider dataCharsetConstants
+	 * @runInSeparateProcess
+	 * @preserveGlobalState disabled
+	 */
+	public function testInitCharset( array $constants, string $expected_collate ): void {
+		foreach ( $constants as $name => $value ) {
+			define( $name, $value );
+		}
+
+		$wpdb = new class() extends WP_SQLite_DB {
+			public $charset;
+			public $collate;
+
+			public function __construct() {}
+		};
+		$wpdb->init_charset();
+
+		$this->assertSame( 'utf8mb4', $wpdb->charset );
+		$this->assertSame( $expected_collate, $wpdb->collate );
+	}
+
+	public static function dataCharsetConstants(): array {
+		return array(
+			'no constants'              => array( array(), 'utf8mb4_unicode_520_ci' ),
+			'empty collation'           => array( array( 'DB_COLLATE' => '' ), 'utf8mb4_unicode_520_ci' ),
+			'null collation'            => array( array( 'DB_COLLATE' => null ), 'utf8mb4_unicode_520_ci' ),
+			'utf8 charset'              => array(
+				array(
+					'DB_CHARSET' => 'utf8',
+					'DB_COLLATE' => '',
+				),
+				'utf8mb4_unicode_520_ci',
+			),
+			'non-UTF-8 charset'         => array(
+				array(
+					'DB_CHARSET' => 'latin1',
+					'DB_COLLATE' => 'latin1_swedish_ci',
+				),
+				'utf8mb4_unicode_520_ci',
+			),
+			'non-UTF-8 collation'       => array( array( 'DB_COLLATE' => 'latin1_swedish_ci' ), 'utf8mb4_unicode_520_ci' ),
+			'charset as collation'      => array( array( 'DB_COLLATE' => 'utf8mb4' ), 'utf8mb4_unicode_520_ci' ),
+			'utf8_general_ci'           => array( array( 'DB_COLLATE' => 'utf8_general_ci' ), 'utf8mb4_unicode_520_ci' ),
+			'utf8_unicode_ci'           => array( array( 'DB_COLLATE' => 'utf8_unicode_ci' ), 'utf8mb4_unicode_520_ci' ),
+			'utf8_bin'                  => array( array( 'DB_COLLATE' => 'utf8_bin' ), 'utf8mb4_bin' ),
+			'utf8_swedish_ci'           => array( array( 'DB_COLLATE' => 'utf8_swedish_ci' ), 'utf8mb4_swedish_ci' ),
+			'utf8mb3_general_ci'        => array( array( 'DB_COLLATE' => 'utf8mb3_general_ci' ), 'utf8mb4_unicode_520_ci' ),
+			'utf8mb3_bin'               => array( array( 'DB_COLLATE' => 'utf8mb3_bin' ), 'utf8mb4_bin' ),
+			'utf8mb4_unicode_ci'        => array( array( 'DB_COLLATE' => 'utf8mb4_unicode_ci' ), 'utf8mb4_unicode_520_ci' ),
+			'utf8mb4_general_ci'        => array( array( 'DB_COLLATE' => 'utf8mb4_general_ci' ), 'utf8mb4_general_ci' ),
+			'utf8mb4_0900_ai_ci'        => array( array( 'DB_COLLATE' => 'utf8mb4_0900_ai_ci' ), 'utf8mb4_0900_ai_ci' ),
+			'uppercase UTF-8 collation' => array( array( 'DB_COLLATE' => 'UTF8_BIN' ), 'utf8mb4_bin' ),
+		);
+	}
 }

packages/mysql-on-sqlite/tests/tools/verify-native-parser-extension.php

@@ -30,7 +30,9 @@ function wp_sqlite_assert_native_parser_delegate( WP_MySQL_Parser $parser, strin
 	}
 
 	$native_property = $reflection->getProperty( 'native' );
-	$native_property->setAccessible( true );
+	if ( PHP_VERSION_ID < 80100 ) {
+		$native_property->setAccessible( true );
+	}
 	if ( ! ( $native_property->getValue( $parser ) instanceof WP_MySQL_Native_Parser ) ) {
 		wp_sqlite_native_parser_verification_fail( $context );
 	}

packages/plugin-sqlite-database-integration/wp-includes/sqlite/class-wp-sqlite-db.php

@@ -49,8 +49,10 @@ public function __construct( $dbname ) {
 		 */
 		$GLOBALS['wpdb'] = $this;
 
+		// The SQLite driver may use the charset and collation while connecting.
+		$this->init_charset();
+
 		parent::__construct( '', '', $dbname, '' );
-		$this->charset = 'utf8mb4';
 	}
 
 	/**
@@ -232,6 +234,36 @@ public function close() {
 		return true;
 	}
 
+	/**
+	 * Sets $this->charset and $this->collate.
+	 *
+	 * This overrides wpdb::init_charset(). SQLite stores all text as UTF-8, and
+	 * the emulated MySQL connection always uses utf8mb4 (see set_charset()), so
+	 * the charset is always utf8mb4, and only a UTF-8 DB_COLLATE applies.
+	 *
+	 * Unlike in wpdb, the result doesn't depend on the database connection.
+	 * This is important, as the SQLite driver may use the charset and collation
+	 * while connecting (see the constructor).
+	 *
+	 * @see wpdb::init_charset()
+	 */
+	public function init_charset() {
+		$collate = defined( 'DB_COLLATE' ) ? strtolower( (string) DB_COLLATE ) : '';
+
+		// MySQL 8.0.30 and newer name the utf8 collations "utf8mb3_*".
+		$collate = preg_replace( '/^utf8mb3_/', 'utf8_', $collate );
+
+		// Collations of other charsets don't apply to the utf8mb4 connection.
+		if ( ! preg_match( '/^utf8(mb4)?_/', $collate ) ) {
+			$collate = '';
+		}
+
+		$charset_collate = $this->resolve_charset( 'utf8mb4', $collate );
+
+		$this->charset = $charset_collate['charset'];
+		$this->collate = $charset_collate['collate'];
+	}
+
 	/**
 	 * Determines the best charset and collation to use given a charset and collation.
 	 *
@@ -254,25 +286,7 @@ public function determine_charset( $charset, $collate ) {
 			return compact( 'charset', 'collate' );
 		}
 
-		if ( 'utf8' === $charset ) {
-			$charset = 'utf8mb4';
-		}
-
-		if ( 'utf8mb4' === $charset ) {
-			// _general_ is outdated, so we can upgrade it to _unicode_, instead.
-			if ( ! $collate || 'utf8_general_ci' === $collate ) {
-				$collate = 'utf8mb4_unicode_ci';
-			} else {
-				$collate = str_replace( 'utf8_', 'utf8mb4_', $collate );
-			}
-		}
-
-		// _unicode_520_ is a better collation, we should use that when it's available.
-		if ( $this->has_cap( 'utf8mb4_520' ) && 'utf8mb4_unicode_ci' === $collate ) {
-			$collate = 'utf8mb4_unicode_520_ci';
-		}
-
-		return compact( 'charset', 'collate' );
+		return $this->resolve_charset( $charset, $collate );
 	}
 
 	/**
@@ -428,10 +442,6 @@ public function db_connect( $allow_bail = true ) {
 			);
 		}
 
-		if ( ! isset( $this->charset ) ) {
-			$this->init_charset();
-		}
-
 		if ( null === $this->dbname || '' === $this->dbname ) {
 			$this->bail(
 				'The database name was not set. The SQLite driver requires a database name to be set to emulate MySQL information schema tables.',
@@ -764,6 +774,45 @@ public function db_server_info() {
 		return $this->dbh->getAttribute( PDO::ATTR_SERVER_VERSION ); // phpcs:ignore WordPress.DB.RestrictedClasses.mysql__PDO
 	}
 
+	/**
+	 * Resolves the charset and collation as wpdb::determine_charset() does.
+	 *
+	 * Unlike in wpdb, this doesn't need a database connection. The emulated
+	 * MySQL server (5.7 or newer) always supports utf8mb4_unicode_520_ci.
+	 *
+	 * @see wpdb::determine_charset()
+	 *
+	 * @param string $charset The character set to check.
+	 * @param string $collate The collation to check.
+	 * @return array {
+	 *     The most appropriate character set and collation to use.
+	 *
+	 *     @type string $charset Character set.
+	 *     @type string $collate Collation.
+	 * }
+	 */
+	private function resolve_charset( $charset, $collate ) {
+		if ( 'utf8' === $charset ) {
+			$charset = 'utf8mb4';
+		}
+
+		if ( 'utf8mb4' === $charset ) {
+			// _general_ is outdated, so we can upgrade it to _unicode_, instead.
+			if ( ! $collate || 'utf8_general_ci' === $collate ) {
+				$collate = 'utf8mb4_unicode_ci';
+			} else {
+				$collate = str_replace( 'utf8_', 'utf8mb4_', $collate );
+			}
+		}
+
+		// _unicode_520_ is a better collation, we should use that when it's available.
+		if ( 'utf8mb4_unicode_ci' === $collate ) {
+			$collate = 'utf8mb4_unicode_520_ci';
+		}
+
+		return compact( 'charset', 'collate' );
+	}
+
 	/**
 	 * Format MySQL-on-SQLite driver error message.
 	 *

tests/phpunit/WP_SQLite_Database_Integration_Charset_Test.php

@@ -0,0 +1,105 @@
+<?php
+
+class WP_SQLite_Database_Integration_Charset_Test extends PHPUnit_Adapter_TestCase {
+
+	public function test_charset_and_collation() {
+		global $wpdb;
+
+		$this->assertSame( 'utf8mb4', $wpdb->charset );
+		$this->assertSame( 'utf8mb4_unicode_520_ci', $wpdb->collate );
+		$this->assertSame(
+			'DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci',
+			$wpdb->get_charset_collate()
+		);
+	}
+
+	public function test_charset_and_collation_without_connection() {
+		global $wpdb;
+
+		// The SQLite driver uses the charset while connecting (e.g., to reconstruct tables).
+		$this->assertTrue( $wpdb->close() );
+		try {
+			$wpdb->charset = null;
+			$wpdb->collate = null;
+			$wpdb->init_charset();
+			$this->assertSame( 'utf8mb4', $wpdb->charset );
+			$this->assertSame( 'utf8mb4_unicode_520_ci', $wpdb->collate );
+		} finally {
+			$this->assertTrue( $wpdb->check_connection() );
+		}
+	}
+
+	public function test_wordpress_table_reconstructed_while_connecting() {
+		global $wpdb;
+
+		$table      = $wpdb->options;
+		$driver     = $wpdb->get_driver();
+		$connection = $driver->get_connection();
+		$this->assertFalse( $driver->inTransaction() );
+
+		// Remove the table from the information schema, and make the next connection reconstruct it.
+		$connection->query(
+			sprintf(
+				'DELETE FROM %s WHERE table_name = ?',
+				$connection->quote_identifier( WP_MySQL_On_SQLite::RESERVED_PREFIX . 'mysql_information_schema_tables' )
+			),
+			array( $table )
+		);
+		$connection->query(
+			sprintf(
+				'DELETE FROM %s WHERE name = ?',
+				$connection->quote_identifier( WP_MySQL_On_SQLite::GLOBAL_VARIABLES_TABLE_NAME )
+			),
+			array( WP_MySQL_On_SQLite::DRIVER_VERSION_VARIABLE_NAME )
+		);
+
+		$original_wpdb = $wpdb;
+		$this->assertTrue( $wpdb->close() );
+		$db = null;
+		try {
+			// As in the SQLite drop-in, the constructor connects and reconstructs the table.
+			$db        = new WP_SQLite_DB( DB_NAME );
+			$collation = $db->get_var(
+				$db->prepare(
+					'SELECT table_collation FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = %s',
+					$table
+				)
+			);
+		} finally {
+			if ( $db ) {
+				$db->close();
+			}
+			$GLOBALS['wpdb'] = $original_wpdb;
+			$this->assertTrue( $original_wpdb->check_connection() );
+		}
+
+		$this->assertSame( 'utf8mb4_unicode_520_ci', $collation );
+	}
+
+	public function test_table_created_with_charset_collate() {
+		global $wpdb;
+
+		$table = $wpdb->prefix . 'sqlite_charset_test';
+		$wpdb->query( $wpdb->prepare( 'DROP TABLE IF EXISTS %i', $table ) );
+
+		try {
+			$wpdb->query( $wpdb->prepare( 'CREATE TABLE %i (id INT, name VARCHAR(20))', $table ) . ' ' . $wpdb->get_charset_collate() );
+			$this->assertSame( '', $wpdb->last_error );
+
+			$this->assertSame(
+				'utf8mb4_unicode_520_ci',
+				$wpdb->get_var(
+					$wpdb->prepare(
+						'SELECT table_collation FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = %s',
+						$table
+					)
+				)
+			);
+
+			$create_table = $wpdb->get_row( $wpdb->prepare( 'SHOW CREATE TABLE %i', $table ), ARRAY_N )[1];
+			$this->assertStringEndsWith( 'DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci', $create_table );
+		} finally {
+			$wpdb->query( $wpdb->prepare( 'DROP TABLE IF EXISTS %i', $table ) );
+		}
+	}
+}