Merge remote-tracking branch 'maplibre/main' into quickbounds

Author: sharkAndshark

Cargo.lock

@@ -2,9 +2,16 @@
 
 ## `mbtiles diff`
 
-Copy command can also be used to compare two mbtiles files and generate a delta (diff) file. The diff file can be [applied](#mbtiles-apply-patch) to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset.  The delta file will contain all tiles that are different between the two files (modifications, insertions, and deletions as `NULL` values), for both the tile and metadata tables.
+Copy command can also be used to compare two mbtiles files and generate a delta (diff) file. The diff file can
+be [applied](#mbtiles-apply-patch) to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire
+modified dataset. The delta file will contain all tiles that are different between the two files (modifications,
+insertions, and deletions as `NULL` values), for both the tile and metadata tables.
 
-There is one exception: `agg_tiles_hash` metadata value will be renamed to `agg_tiles_hash_in_diff`, and a new `agg_tiles_hash` will be generated for the diff file itself. This is done to avoid confusion when applying the diff file to the original file, as the `agg_tiles_hash` value will be different after the diff is applied. The `apply-diff` command will automatically rename the `agg_tiles_hash_in_diff` value back to `agg_tiles_hash` when applying the diff.
+There is one exception: `agg_tiles_hash` metadata value will be renamed to `agg_tiles_hash_after_apply`, and a
+new `agg_tiles_hash` will be generated for the diff file itself. This is done to avoid confusion when applying the diff
+file to the original file, as the `agg_tiles_hash` value will be different after the diff is applied. The `apply-diff`
+command will automatically rename the `agg_tiles_hash_after_apply` value back to `agg_tiles_hash` when applying the
+diff.
 
 ```shell
 # This command will compare `file1.mbtiles` and `file2.mbtiles`, and generate a new diff file `diff.mbtiles`.
@@ -24,17 +31,27 @@ mbtiles validate file2a.mbtiles
 
 ## `mbtiles apply-patch`
 
-Apply the diff file generated with the `mbtiles diff` command above to an MBTiles file. The diff file can be applied to the `src_file.mbtiles` that has been previously downloaded to avoid copying/transmitting the entire modified dataset again. The `src_file.mbtiles` will modified in-place. It is also possible to apply the diff file while copying the source file to a new destination file, by using the [`mbtiles copy --apply-patch`](mbtiles-copy.md#mbtiles-copy---apply-patch) command.
+Apply the diff file generated with the `mbtiles diff` command above to an MBTiles file. The diff file can be applied to
+the `src_file.mbtiles` that has been previously downloaded to avoid copying/transmitting the entire modified dataset
+again. The `src_file.mbtiles` will modified in-place. It is also possible to apply the diff file while copying the
+source file to a new destination file, by using
+the [`mbtiles copy --apply-patch`](mbtiles-copy.md#mbtiles-copy---apply-patch) command.
 
-Note that the `agg_tiles_hash_in_diff` metadata value will be renamed to `agg_tiles_hash` when applying the diff. This is done to avoid confusion when applying the diff file to the original file, as the `agg_tiles_hash` value will be different after the diff is applied.
+Note that the `agg_tiles_hash_after_apply` metadata value will be renamed to `agg_tiles_hash` when applying the diff.
+This is done to avoid confusion when applying the diff file to the original file, as the `agg_tiles_hash` value will be
+different after the diff is applied.
 
 ```shell
 mbtiles apply-patch src_file.mbtiles diff_file.mbtiles
 ```
 
 #### Applying diff with SQLite
 
-Another way to apply the diff is to use the `sqlite3` command line tool directly. This SQL will delete all tiles from `src_file.mbtiles` that are set to `NULL` in `diff_file.mbtiles`, and then insert or update all new tiles from `diff_file.mbtiles` into `src_file.mbtiles`, where both files are of `flat` type. The name of the diff file is passed as a query parameter to the sqlite3 command line tool, and then used in the SQL statements.  Note that this does not update the `agg_tiles_hash` metadata value, so it will be incorrect after the diff is applied.
+Another way to apply the diff is to use the `sqlite3` command line tool directly. This SQL will delete all tiles
+from `src_file.mbtiles` that are set to `NULL` in `diff_file.mbtiles`, and then insert or update all new tiles
+from `diff_file.mbtiles` into `src_file.mbtiles`, where both files are of `flat` type. The name of the diff file is
+passed as a query parameter to the sqlite3 command line tool, and then used in the SQL statements. Note that this does
+not update the `agg_tiles_hash` metadata value, so it will be incorrect after the diff is applied.
 
 ```shell
 sqlite3 src_file.mbtiles \

docs/src/mbtiles-diff.md

@@ -9,6 +9,7 @@ use deadpool_postgres::tokio_postgres::Config;
 use log::{info, warn};
 use regex::Regex;
 use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
+use rustls::crypto::ring::default_provider;
 use rustls::crypto::{verify_tls12_signature, verify_tls13_signature};
 use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
 use rustls::{DigitallySignedStruct, Error, SignatureScheme};
@@ -80,7 +81,7 @@ impl ServerCertVerifier for NoCertificateVerification {
             message,
             cert,
             dss,
-            &rustls::crypto::ring::default_provider().signature_verification_algorithms,
+            &default_provider().signature_verification_algorithms,
         )
     }
 
@@ -94,12 +95,12 @@ impl ServerCertVerifier for NoCertificateVerification {
             message,
             cert,
             dss,
-            &rustls::crypto::ring::default_provider().signature_verification_algorithms,
+            &default_provider().signature_verification_algorithms,
         )
     }
 
     fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
-        rustls::crypto::ring::default_provider()
+        default_provider()
             .signature_verification_algorithms
             .supported_schemes()
     }

martin/src/pg/tls.rs

@@ -282,10 +282,14 @@ mod tests {
             let expected = std::fs::read(path.with_extension("png"))
                 .expect("Unable to open expected PNG file, make sure to bless tests with\n  cargo test --features bless-tests\n");
 
-            assert_eq!(
-                png, expected,
-                "Make sure to run bless if needed:\n  cargo test --features bless-tests\n\n{json}",
-            );
+            // The PNG output is too flaky to be reliably used in a test
+            if png != expected {
+                warn!("Generated PNG does not match expected PNG, make sure to bless tests with\n  cargo test --features bless-tests\n");
+            }
+            // assert_eq!(
+            //     png, expected,
+            //     "Make sure to run bless if needed:\n  cargo test --features bless-tests\n\n{json}",
+            // );
         }
     }
 }

martin/src/sprites/mod.rs

@@ -17,7 +17,7 @@ use crate::queries::{
 use crate::MbtType::{Flat, FlatWithHash, Normalized};
 use crate::{
     invert_y_value, reset_db_settings, CopyType, MbtError, MbtType, MbtTypeCli, Mbtiles,
-    AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF,
+    AGG_TILES_HASH, AGG_TILES_HASH_AFTER_APPLY,
 };
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)]
@@ -233,27 +233,29 @@ impl MbtileCopierInt {
         if dif.is_some() {
             // Insert all rows from diffDb.metadata if they do not exist or are different in sourceDb.metadata.
             // Also insert all names from sourceDb.metadata that do not exist in diffDb.metadata, with their value set to NULL.
-            // Rename agg_tiles_hash to agg_tiles_hash_in_diff because agg_tiles_hash will be auto-added later
+            // Rename agg_tiles_hash to agg_tiles_hash_after_apply because agg_tiles_hash will be auto-added later
             if self.options.diff_with_file.is_some() {
+                // Include agg_tiles_hash value even if it is the same because we will still need it when applying the diff
                 sql = format!(
                     "
     INSERT {on_dupl} INTO metadata (name, value)
-        SELECT IIF(name = '{AGG_TILES_HASH}','{AGG_TILES_HASH_IN_DIFF}', name) as name
+        SELECT IIF(name = '{AGG_TILES_HASH}','{AGG_TILES_HASH_AFTER_APPLY}', name) as name
              , value
         FROM (
             SELECT COALESCE(difMD.name, srcMD.name) as name
                  , difMD.value as value
             FROM sourceDb.metadata AS srcMD FULL JOIN diffDb.metadata AS difMD
                  ON srcMD.name = difMD.name
-            WHERE srcMD.value != difMD.value OR srcMD.value ISNULL OR difMD.value ISNULL
+            WHERE srcMD.value != difMD.value OR srcMD.value ISNULL OR difMD.value ISNULL OR srcMD.name = '{AGG_TILES_HASH}'
         ) joinedMD
-        WHERE name != '{AGG_TILES_HASH_IN_DIFF}'"
+        WHERE name != '{AGG_TILES_HASH_AFTER_APPLY}'"
                 );
+                debug!("Copying metadata, taking into account diff file with {sql}");
             } else {
                 sql = format!(
                     "
     INSERT {on_dupl} INTO metadata (name, value)
-        SELECT IIF(name = '{AGG_TILES_HASH_IN_DIFF}','{AGG_TILES_HASH}', name) as name
+        SELECT IIF(name = '{AGG_TILES_HASH_AFTER_APPLY}','{AGG_TILES_HASH}', name) as name
              , value
         FROM (
             SELECT COALESCE(srcMD.name, difMD.name) as name
@@ -264,10 +266,6 @@ impl MbtileCopierInt {
         ) joinedMD
         WHERE name != '{AGG_TILES_HASH}'"
                 );
-            }
-            if self.options.diff_with_file.is_some() {
-                debug!("Copying metadata, taking into account diff file with {sql}");
-            } else {
                 debug!("Copying metadata, and applying the diff file with {sql}");
             }
         } else {

mbtiles/src/copier.rs

@@ -32,7 +32,7 @@ pub use update::UpdateZoomType;
 mod validation;
 pub use validation::{
     calc_agg_tiles_hash, AggHashType, IntegrityCheckType, MbtType, AGG_TILES_HASH,
-    AGG_TILES_HASH_IN_DIFF,
+    AGG_TILES_HASH_AFTER_APPLY,
 };
 
 /// `MBTiles` uses a TMS (Tile Map Service) scheme for its tile coordinates (inverted along the Y axis).

mbtiles/src/lib.rs

@@ -5,7 +5,7 @@ use sqlx::query;
 
 use crate::queries::detach_db;
 use crate::MbtType::{Flat, FlatWithHash, Normalized};
-use crate::{MbtResult, MbtType, Mbtiles, AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF};
+use crate::{MbtResult, MbtType, Mbtiles, AGG_TILES_HASH, AGG_TILES_HASH_AFTER_APPLY};
 
 pub async fn apply_patch(base_file: PathBuf, patch_file: PathBuf) -> MbtResult<()> {
     let base_mbt = Mbtiles::new(base_file)?;
@@ -53,7 +53,7 @@ pub async fn apply_patch(base_file: PathBuf, patch_file: PathBuf) -> MbtResult<(
     query(&format!(
         "
     INSERT OR REPLACE INTO metadata (name, value)
-    SELECT IIF(name = '{AGG_TILES_HASH_IN_DIFF}', '{AGG_TILES_HASH}', name) as name,
+    SELECT IIF(name = '{AGG_TILES_HASH_AFTER_APPLY}', '{AGG_TILES_HASH}', name) as name,
            value
     FROM patchDb.metadata
     WHERE name NOTNULL AND name != '{AGG_TILES_HASH}';"

mbtiles/src/patcher.rs

@@ -26,7 +26,7 @@ pub const AGG_TILES_HASH: &str = "agg_tiles_hash";
 
 /// Metadata key for a diff file,
 /// describing the eventual [`AGG_TILES_HASH`] value once the diff is applied
-pub const AGG_TILES_HASH_IN_DIFF: &str = "agg_tiles_hash_after_apply";
+pub const AGG_TILES_HASH_AFTER_APPLY: &str = "agg_tiles_hash_after_apply";
 
 #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, EnumDisplay, Serialize)]
 #[enum_display(case = "Kebab")]