corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1913183 - Vendor in app-services, r=nanj,supply-chain-reviewers

Browse Source 
Removed the `IsValid` check when converting a `RustBuffer` to an
`OwnedRustBuffer`.  The vendored changes revealed some issues with this
logic.  See https://bugzilla.mozilla.org/show_bug.cgi?id=1913183#c4 for
details.

Differential Revision: https://phabricator.services.mozilla.com/D219189
This commit is contained in:
Ben Dean-Kawamura
2024-08-15 19:35:32 +00:00
parent 9f0dbd355d
commit d0725f213e
118 changed files with 13894 additions and 1095 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.cargo/config.toml.in
View File

@@ -55,9 +55,9 @@ git = "https://github.com/mozilla-spidermonkey/jsparagus"
rev = "61f399c53a641ebd3077c1f39f054f6d396a633c"
replace-with = "vendored-sources"
[source."git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72"]
[source."git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9"]
git = "https://github.com/mozilla/application-services"
rev = "8fd08c6f2f8acd38579bd3142fecda9272957b72"
rev = "c3774b262f27fabdd8ae7d064db5745029b347b9"
replace-with = "vendored-sources"
[source."git+https://github.com/mozilla/audioipc?rev=e6f44a2bd1e57d11dfc737632a9e849077632330"]

54
Cargo.lock generated
View File

@@ -1718,7 +1718,7 @@ dependencies = [
[[package]]
name = "error-support"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"error-support-macros",
"lazy_static",
@@ -1730,7 +1730,7 @@ dependencies = [
[[package]]
name = "error-support-macros"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"proc-macro2",
"quote",
@@ -3046,7 +3046,7 @@ dependencies = [
[[package]]
name = "interrupt-support"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"lazy_static",
"parking_lot",
@@ -4253,7 +4253,7 @@ dependencies = [
[[package]]
name = "nss_build_common"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
[[package]]
name = "nsstring"
@@ -4466,7 +4466,7 @@ checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
[[package]]
name = "payload-support"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"serde",
"serde_derive",
@@ -4938,7 +4938,7 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
[[package]]
name = "relevancy"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"anyhow",
"base64 0.21.3",
@@ -4961,8 +4961,9 @@ dependencies = [
[[package]]
name = "remote_settings"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"log",
"parking_lot",
"serde",
"serde_json",
@@ -5019,6 +5020,28 @@ dependencies = [
"uuid",
]
[[package]]
name = "rmp"
version = "0.8.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4"
dependencies = [
"byteorder",
"num-traits",
"paste",
]
[[package]]
name = "rmp-serde"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db"
dependencies = [
"byteorder",
"rmp",
"serde",
]
[[package]]
name = "ron"
version = "0.8.1"
@@ -5496,7 +5519,7 @@ dependencies = [
[[package]]
name = "sql-support"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"ffi-support",
"interrupt-support",
@@ -5678,7 +5701,7 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
[[package]]
name = "suggest"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"anyhow",
"chrono",
@@ -5689,6 +5712,7 @@ dependencies = [
"once_cell",
"parking_lot",
"remote_settings",
"rmp-serde",
"rusqlite",
"serde",
"serde_json",
@@ -5729,7 +5753,7 @@ dependencies = [
[[package]]
name = "sync-guid"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"base64 0.21.3",
"rand",
@@ -5740,7 +5764,7 @@ dependencies = [
[[package]]
name = "sync15"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"anyhow",
"error-support",
@@ -5772,7 +5796,7 @@ dependencies = [
[[package]]
name = "tabs"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"anyhow",
"error-support",
@@ -6097,7 +6121,7 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
[[package]]
name = "types"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"rusqlite",
"serde",
@@ -6464,7 +6488,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "viaduct"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"ffi-support",
"log",
@@ -6612,7 +6636,7 @@ dependencies = [
[[package]]
name = "webext-storage"
version = "0.1.0"
source = "git+https://github.com/mozilla/application-services?rev=8fd08c6f2f8acd38579bd3142fecda9272957b72#8fd08c6f2f8acd38579bd3142fecda9272957b72"
source = "git+https://github.com/mozilla/application-services?rev=c3774b262f27fabdd8ae7d064db5745029b347b9#c3774b262f27fabdd8ae7d064db5745029b347b9"
dependencies = [
"anyhow",
"error-support",

16
Cargo.toml
View File

@@ -210,14 +210,14 @@ midir = { git = "https://github.com/mozilla/midir.git", rev = "85156e360a37d8517
malloc_size_of_derive = { path = "xpcom/rust/malloc_size_of_derive" }
# application-services overrides to make updating them all simpler.
interrupt-support = { git = "https://github.com/mozilla/application-services", rev = "8fd08c6f2f8acd38579bd3142fecda9272957b72" }
relevancy = { git = "https://github.com/mozilla/application-services", rev = "8fd08c6f2f8acd38579bd3142fecda9272957b72" }
sql-support = { git = "https://github.com/mozilla/application-services", rev = "8fd08c6f2f8acd38579bd3142fecda9272957b72" }
suggest = { git = "https://github.com/mozilla/application-services", rev = "8fd08c6f2f8acd38579bd3142fecda9272957b72" }
sync15 = { git = "https://github.com/mozilla/application-services", rev = "8fd08c6f2f8acd38579bd3142fecda9272957b72" }
tabs = { git = "https://github.com/mozilla/application-services", rev = "8fd08c6f2f8acd38579bd3142fecda9272957b72" }
viaduct = { git = "https://github.com/mozilla/application-services", rev = "8fd08c6f2f8acd38579bd3142fecda9272957b72" }
webext-storage = { git = "https://github.com/mozilla/application-services", rev = "8fd08c6f2f8acd38579bd3142fecda9272957b72" }
interrupt-support = { git = "https://github.com/mozilla/application-services", rev = "c3774b262f27fabdd8ae7d064db5745029b347b9" }
relevancy = { git = "https://github.com/mozilla/application-services", rev = "c3774b262f27fabdd8ae7d064db5745029b347b9" }
sql-support = { git = "https://github.com/mozilla/application-services", rev = "c3774b262f27fabdd8ae7d064db5745029b347b9" }
suggest = { git = "https://github.com/mozilla/application-services", rev = "c3774b262f27fabdd8ae7d064db5745029b347b9" }
sync15 = { git = "https://github.com/mozilla/application-services", rev = "c3774b262f27fabdd8ae7d064db5745029b347b9" }
tabs = { git = "https://github.com/mozilla/application-services", rev = "c3774b262f27fabdd8ae7d064db5745029b347b9" }
viaduct = { git = "https://github.com/mozilla/application-services", rev = "c3774b262f27fabdd8ae7d064db5745029b347b9" }
webext-storage = { git = "https://github.com/mozilla/application-services", rev = "c3774b262f27fabdd8ae7d064db5745029b347b9" }
# Patch `gpu-descriptor` 0.3.0 to remove unnecessary `allocator-api2` dep.:
# Still waiting for the now-merged <https://github.com/zakarumych/gpu-descriptor/pull/40> to be released.

15
supply-chain/audits.toml
View File

@@ -3776,6 +3776,21 @@ who = "Chris H-C <chutten@mozilla.com>"
criteria = "safe-to-deploy"
version = "0.18.4"
[[audits.rmp]]
who = "Ben Dean-Kawamura <bdk@mozilla.com>"
criteria = "safe-to-deploy"
version = "0.8.14"
notes = """
Very popular crate. 1 instance of unsafe code, which is used to adjust a slice to work around
lifetime issues. No network or file access.
"""
[[audits.rmp-serde]]
who = "Ben Dean-Kawamura <bdk@mozilla.com>"
criteria = "safe-to-deploy"
version = "1.3.0"
notes = "Very popular crate. No unsafe code, network or file access."
[[audits.ron]]
who = "Mike Hommey <mh+mozilla@glandium.org>"
criteria = "safe-to-deploy"

2
third_party/rust/nss_build_common/.cargo-checksum.json vendored
View File

@@ -1 +1 @@
{"files":{"Cargo.toml":"6597b6d5217376ab747534364a58958ddbb23ffda52045e68b610cd8f2dcdfd1","src/lib.rs":"9f7379d76c3c663fba7aceb0cc273ab459e1dd9a25664354402c3895bbd25240"},"package":null}
{"files":{"Cargo.toml":"6597b6d5217376ab747534364a58958ddbb23ffda52045e68b610cd8f2dcdfd1","src/lib.rs":"cb149b57b95a240499decf1afbfa4e7854645f8729079f936a5e00dcc94edf24"},"package":null}

6
third_party/rust/nss_build_common/src/lib.rs vendored
View File

@@ -92,13 +92,13 @@ fn link_nss_libs(kind: LinkingKind) {
let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
if target_arch == "x86_64" && target_os == "android" {
let android_home = env::var("ANDROID_HOME").expect("ANDROID_HOME not set");
const ANDROID_NDK_VERSION: &str = "26.2.11394342";
const ANDROID_NDK_VERSION: &str = "27.0.12077973";
// One of these will exist, depending on the host platform.
const DARWIN_X86_64_LIB_DIR: &str =
"/toolchains/llvm/prebuilt/darwin-x86_64/lib/clang/17/lib/linux/";
"/toolchains/llvm/prebuilt/darwin-x86_64/lib/clang/18/lib/linux/";
println!("cargo:rustc-link-search={android_home}/ndk/{ANDROID_NDK_VERSION}/{DARWIN_X86_64_LIB_DIR}");
const LINUX_X86_64_LIB_DIR: &str =
"/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/17/lib/linux/";
"/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/18/lib/linux/";
println!("cargo:rustc-link-search={android_home}/ndk/{ANDROID_NDK_VERSION}/{LINUX_X86_64_LIB_DIR}");
println!("cargo:rustc-link-lib=static=clang_rt.builtins-x86_64-android");
}

2
third_party/rust/remote_settings/.cargo-checksum.json vendored
View File

@@ -1 +1 @@
{"files":{"Cargo.toml":"b7ce43b44cab449291d711602e00307ef113e85d220990123756fdf46d345bbf","build.rs":"4326f03729cf8f1673e4228e6dc111de1ea4d8bcc06351f7ae563efb2613f866","src/client.rs":"289bd72c901d99eebbf9d6f559456ca05a4f23b1570ccba2d206284a9dec7d65","src/config.rs":"52a209256acd8b1fada2b91e9d9f669df0ee6e9609baad7ec34a2111ed2a6541","src/error.rs":"4bb15cd7f6ebc438119f36291ab0eb951fe2fb05e166445817cb05aa89397000","src/lib.rs":"655559b1b0f09ad221ceba462ace73d9216a6551d70062126ffc8a085d8b89bb","src/remote_settings.udl":"1ffeb10385e4db63606cda79bb59e77170af1d2ca0028da8ab2c4d7622969734","uniffi.toml":"f8ec8dc593e0d501c2e9e40368ec93ec33b1edd8608e29495e0a54b63144e880"},"package":null}
{"files":{"Cargo.toml":"15a6e0298dd7d708f4aa8514e218925c1f013c66cb117b12e19148d051fd5de5","build.rs":"4326f03729cf8f1673e4228e6dc111de1ea4d8bcc06351f7ae563efb2613f866","src/cache.rs":"9c16f17353730103d57132a0f055863b085d2d3eb3a78f6a6f8d095f897d0c0d","src/client.rs":"1c589317907d21745e945dfd992c11535aaafcaca9a6ebe33a5a7c2e2cb93997","src/config.rs":"52a209256acd8b1fada2b91e9d9f669df0ee6e9609baad7ec34a2111ed2a6541","src/error.rs":"4bb15cd7f6ebc438119f36291ab0eb951fe2fb05e166445817cb05aa89397000","src/lib.rs":"ff1b9f66ae6069005d36c84b296f22424b4305a5873353e351fbe057f126e140","src/remote_settings.udl":"1ffeb10385e4db63606cda79bb59e77170af1d2ca0028da8ab2c4d7622969734","uniffi.toml":"f8ec8dc593e0d501c2e9e40368ec93ec33b1edd8608e29495e0a54b63144e880"},"package":null}

1
third_party/rust/remote_settings/Cargo.toml vendored
View File

@@ -25,6 +25,7 @@ description = "A Remote Settings client intended for application layer platforms
license = "MPL-2.0"
[dependencies]
log = "0.4"
parking_lot = "0.12"
serde_json = "1"
thiserror = "1.0"

152
third_party/rust/remote_settings/src/cache.rs vendored Normal file
View File

@@ -0,0 +1,152 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use crate::RemoteSettingsResponse;
use std::collections::HashSet;
/// Merge a cached RemoteSettingsResponse and a newly downloaded one to get a merged response
///
/// cached is a previously downloaded remote settings response (possibly run through merge_cache_and_response).
/// new is a newly downloaded remote settings response (with `_expected` set to the last_modified
/// time of the cached response).
///
/// This will merge the records from both responses, handle deletions/tombstones, and return a
/// response that has:
/// - The newest `last_modified_date`
/// - A record list containing the newest version of all live records. Deleted records will not
/// be present in this list.
///
/// If everything is working properly, the returned value will exactly match what the server would
/// have returned if there was no `_expected` param.
pub fn merge_cache_and_response(
cached: RemoteSettingsResponse,
new: RemoteSettingsResponse,
) -> RemoteSettingsResponse {
let new_record_ids = new
.records
.iter()
.map(|r| r.id.as_str())
.collect::<HashSet<&str>>();
// Start with any cached records that don't appear in new.
let mut records = cached
.records
.into_iter()
.filter(|r| !new_record_ids.contains(r.id.as_str()))
// deleted should always be false, check it just in case
.filter(|r| !r.deleted)
.collect::<Vec<_>>();
// Add all (non-deleted) records from new
records.extend(new.records.into_iter().filter(|r| !r.deleted));
RemoteSettingsResponse {
last_modified: new.last_modified,
records,
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::{RemoteSettingsRecord, RsJsonObject};
// Quick way to generate the fields data for our mock records
fn fields(data: &str) -> RsJsonObject {
let mut map = serde_json::Map::new();
map.insert("data".into(), data.into());
map
}
#[test]
fn test_combine_cache_and_response() {
let cached_response = RemoteSettingsResponse {
last_modified: 1000,
records: vec![
RemoteSettingsRecord {
id: "a".into(),
last_modified: 100,
deleted: false,
attachment: None,
fields: fields("a"),
},
RemoteSettingsRecord {
id: "b".into(),
last_modified: 200,
deleted: false,
attachment: None,
fields: fields("b"),
},
RemoteSettingsRecord {
id: "c".into(),
last_modified: 300,
deleted: false,
attachment: None,
fields: fields("c"),
},
],
};
let new_response = RemoteSettingsResponse {
last_modified: 2000,
records: vec![
// d is new
RemoteSettingsRecord {
id: "d".into(),
last_modified: 1300,
deleted: false,
attachment: None,
fields: fields("d"),
},
// b was deleted
RemoteSettingsRecord {
id: "b".into(),
last_modified: 1200,
deleted: true,
attachment: None,
fields: RsJsonObject::new(),
},
// a was updated
RemoteSettingsRecord {
id: "a".into(),
last_modified: 1100,
deleted: false,
attachment: None,
fields: fields("a-with-new-data"),
},
// c was not modified, so it's not present in the new response
],
};
let mut merged = merge_cache_and_response(cached_response, new_response);
// Sort the records to make the assertion easier
merged.records.sort_by_key(|r| r.id.clone());
assert_eq!(
merged,
RemoteSettingsResponse {
last_modified: 2000,
records: vec![
// a was updated
RemoteSettingsRecord {
id: "a".into(),
last_modified: 1100,
deleted: false,
attachment: None,
fields: fields("a-with-new-data"),
},
RemoteSettingsRecord {
id: "c".into(),
last_modified: 300,
deleted: false,
attachment: None,
fields: fields("c"),
},
RemoteSettingsRecord {
id: "d".into(),
last_modified: 1300,
deleted: false,
attachment: None,
fields: fields("d")
},
],
}
);
}
}

10
third_party/rust/remote_settings/src/client.rs vendored
View File

@@ -6,7 +6,7 @@ use crate::config::RemoteSettingsConfig;
use crate::error::{RemoteSettingsError, Result};
use crate::{RemoteSettingsServer, UniffiCustomTypeConverter};
use parking_lot::Mutex;
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use std::{
borrow::Cow,
time::{Duration, Instant},
@@ -215,20 +215,20 @@ impl Client {
/// Data structure representing the top-level response from the Remote Settings.
/// [last_modified] will be extracted from the etag header of the response.
#[derive(Clone, Debug, Eq, PartialEq)]
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
pub struct RemoteSettingsResponse {
pub records: Vec<RemoteSettingsRecord>,
pub last_modified: u64,
}
#[derive(Deserialize)]
#[derive(Deserialize, Serialize)]
struct RecordsResponse {
data: Vec<RemoteSettingsRecord>,
}
/// A parsed Remote Settings record. Records can contain arbitrary fields, so clients
/// are required to further extract expected values from the [fields] member.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
pub struct RemoteSettingsRecord {
pub id: String,
pub last_modified: u64,
@@ -241,7 +241,7 @@ pub struct RemoteSettingsRecord {
/// Attachment metadata that can be optionally attached to a [Record]. The [location] should
/// included in calls to [Client::get_attachment].
#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
pub struct Attachment {
pub filename: String,
pub mimetype: String,

1
third_party/rust/remote_settings/src/lib.rs vendored
View File

@@ -2,6 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
pub mod cache;
pub mod error;
pub use error::{RemoteSettingsError, Result};
use std::{fs::File, io::prelude::Write};

1
third_party/rust/rmp-serde/.cargo-checksum.json vendored Normal file
View File

@@ -0,0 +1 @@
{"files":{"CHANGELOG.md":"69c06b50f745330356f3f95e8a7cf7af9a6188dfb9f0d0b7711df60dacfe9f35","Cargo.toml":"c4ebc9b96a7d2e55e39e3915c0ca89e4a4cab513b417aa4551c3171ee3660eee","LICENSE":"979d35e1d157289100ba2a7eb13fffe4e7a1b93506c755c4c66890918e467c4e","README.md":"d4e8e5f69e3793c16efac367726a45b2a13cf4ec3a7f7f36444d45c6d9e52b12","benches/buf.rs":"821d3e79a6a960504755776feacd921bd37bcee58743d2376a4643e5b42a8491","src/bytes.rs":"e1325a2f50e5d867c07b97f1c2aa8982d743b54ccb7f5ddb1294e368e800928e","src/config.rs":"6405d368b5957c5b62636969ddc17c57236c416b6c941a581890deff3acb0d54","src/decode.rs":"e09eff4487825feacd3668b227d62ad21bf9047de6d2522b24e404bcc2cd6e68","src/encode.rs":"ff514bfd211e4cf9c7341fbface3735ceca4b7e46de9ff6df7ff92064d8eef83","src/lib.rs":"6d5674cf75cf13efe0a5871879a26b81acfd9fb447d73e26a7ac234e132f17fe","tests/decode.rs":"defac0fd802827a8df95bdf42f51093b9c0887bd1cd1016586960d49c23a70c2","tests/decode_derive.rs":"da20ab40a2add8df91e840b4c37126b331b6eccecd5da8bdfa711bcbd2262ac9","tests/encode.rs":"4308e85db7d1d065f5e8eb8405c00fe442f3a2bc37f8bc5355446dcf0615b6c2","tests/encode_derive.rs":"f4e7edb3a57da9417c3dcffdf801b0e9381fa0e0e5f0b844e50a341f330e4c3e","tests/round.rs":"d7de2294b00e32ef7ef8daa120a1f50bec7bc3ef3dc576da451f0e2c06ccdb03"},"package":"52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db"}

164
third_party/rust/rmp-serde/CHANGELOG.md vendored Normal file
View File

@@ -0,0 +1,164 @@
# Change Log
All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](http://semver.org/).
## [Unreleased][unreleased]
### Added:
- Generic `decode::from_read_ref` function that allows to deserialize a borrowed byte-array into the specified type.
- Add `Ext` trait for `Serializer` that allows to wrap a serializer with another one, that overrides exactly one serialization policy. For example using `with_struct_map` method it is possible to serialize structs as a MessagePack map with field names, overriding default serialization policy, which emits structs as a tuple.
- Add `UnderlyingWrite` trait for `Serializer` and its wrappers to be able to obtain the underlying writer.
- Add missing `Debug` implementations.
- More `decode::Error` conversions.
- Support for serializing and deserializing 128-bit values in serde.
- Support for serializing sequences and maps with unknown length, that enables the use of `#[serde(flatten)]` attribute (#196).
- Depth limit is now enforced for `Deserializer`.
### Changed:
- (Breaking) Serialize newtype structs by serializing its inner type without wrapping into a tuple.
- (Breaking) Enums are now encoded as a map `{tag: data}` rather than as a list `[tag, data]`. (#149)
- Function `encode::to_vec_named` now accepts unsized values.
- Renamed `decode::Read` trait to `decode::ReadSlice` to avoid clashing with `std::io::Read` and to specify more precisely what it does.
- Support reading encoded integers as floats when safe (#204)
### Removed:
- Type parameter `VariantWriter` is no longer a type member of `Serializer`. Instead a `Serializer` can be wrapped by another serializer using `with_struct_map`, `with_struct_tuple` etc. methods.
### Fixed:
- Fix error decoding `Some(enum)` (#185)
- Fix error decoding unit structs which were encoded as `[]` (#181)
- Fix `Display` implementations for errors not including all relevant information (#199)
- Fix deserialization of nested `Option`s (#245)
## 0.13.7 - 2017-09-13
### Changed:
- `Raw` and `RawRef` are now serializable.
- Allow to construct `Raw` and `RawRef` from string or from a byte array.
## 0.13.6 - 2017-08-04
### Added:
- Serialize struct as a map (#140).
## 0.13.5 - 2017-07-21
### Changed
- Switch to using `char::encode_utf8`.
In Rust 1.15, the function `char::encode_utf8` was stabilized. Assuming that `rmp` follows the `serde` standard of supporting the last 3 stable releases, this function is now safe to use. I believe this removes the last allocation required on the serialization path.
## 0.13.4 - 2017-07-11
### Fixed
- Fixed build on nightly rustc (#135).
## 0.13.3 - 2017-05-27
### Fixed
- Fixed build on nightly rustc (#131).
## 0.13.2 - 2017-04-30
### Changed
- Fixed `rmp_serde::decode::from_read` signature by marking that it can only deserialize into `DeserializeOwned`. The previous signature let try to deserialize, for example `&str` and other borrow types and it failed at runtime instead of catching it at compile time.
## 0.13.1 - 2017-04-25
### Added
- Add helper `RawRef` struct that allows to deserialize borrowed strings even if they contain invalid UTF-8. This can be when deserializing frames from older MessagePack spec.
## 0.13.0 - 2017-04-24
### Added
- Zero-copy deserialization from `&[u8]`.
### Changed
- Adapt with serde 1.0.
## 0.12.4 - 2017-03-26
### Fixed
- Fix compilation on rustc 1.13.
## 0.12.3 - 2017-03-26
### Added
- Add helper `Raw` struct that allows to deserialize strings even if they contain invalid UTF-8. This can be when deserializing frames from older MessagePack spec.
- Serializer can now return back its underlying writer by reference, mutable reference and by value.
## 0.12.2 - 2017-02-17
### Added
- Added `write`, `to_vec` and `from_read` functions to reduce boilerplate for serializing and deserializing custom types that implement `Serialize` or `Deserialize`.
## 0.12.1 - 2017-02-11
### Added
- Allow `Deserializer` to return number of bytes read in case of using Cursor as an underlying reader.
## 0.12.0 - 2017-02-08
### Changed
- Adapt with serde 0.9.
## 0.11.0 - 2017-01-05
### Changed
- Adapt with RMP core 0.8.
- The `Serializer` now encodes integers using the most effective representation.
- The `Deserializer` now properly decodes integer values that fit in the expected type.
- Default stack protector depth is now 1024 instead of 1000.
- Internal buffer in the `Deserializer` now have some capacity preallocated.
## 0.10.0 - 2016-10-06
### Changed
- Update serde dependency to 0.8.
## 0.9.6 - 2016-08-05
### Fixed
- Switch unit structs to using the same serialization mechanism as other structs (#76).
## 0.9.5 - 2016-07-28
### Added
- Added a wrapper over `rmp::Value` to be able to serialize it.
## 0.9.4 - 2016-07-11
### Fixed
- Reading binary should no longer trigger unexpected EOF error on valid read.
## 0.9.3 - 2016-07-11
### Changed
- Reuse deserializer buffer on every read for string and binary deserialization without unnecessary intermediate buffer creation.
This change increases the string and binary deserialization performance (many thanks to Fedor Gogolev <knsd@knsd.net>).
## 0.9.2 - 2016-07-03
### Added
- Implement `size_hint()` function for `SeqVisitor` and `MapVisitor`, so it can be possible to preallocate things, increasing the performance greatly.
## 0.9.1 - 2016-06-24
### Fixed
- Serializer should no longer panic with unimplemented error on struct variant serialization ([#64]).
## 0.9.0 - 2016-03-28
### Changed
- Adapt code to be compilable with Serde v0.7.
## 0.8.2 - 2015-11-10
### Changed
- Fixed stack overflow when unpacking recursive data structures.
## 0.8.1 - 2015-10-03
### Changed
- Upper limit for serde version.
### Fixed
- Use the most effective int encoding
Even if the value is explicitly marked as i64 it must be encoded using
the most effective bytes representation despite of signed it or
unsigned.
## 0.8.0 - 2015-09-11
### Changed
- Serializer can now be extended with custom struct encoding policy.
- Improved error types and its messages for serialization part.
- New error type introduced - UnknownLength. Returned on attempt to serialize struct, map or serquence with unknown
length (Serde allows this).
- The new type is returned if necessary.
### Fixed
- Deserializer now properly works with enums.
- Options with default values (that can be initialized using unit marker) deserialization.
This fix also forbids the following Option deserialization cases:
- Option<()>.
- Option<Option<...>>.
It's impossible to properly deserialize the listed cases without explicit option marker in protocol.
- Serializer now properly serializes unit structs.
Previously it was serialized as a unit (nil), now there is just an empty array ([]).
[#64]: (https://github.com/3Hren/msgpack-rust/pull/64)
[#76]: (https://github.com/3Hren/msgpack-rust/pull/76)

50
third_party/rust/rmp-serde/Cargo.toml vendored Normal file
View File

@@ -0,0 +1,50 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2021"
name = "rmp-serde"
version = "1.3.0"
authors = ["Evgeny Safronov <division494@gmail.com>"]
description = "Serde bindings for RMP"
documentation = "https://docs.rs/rmp-serde"
readme = "README.md"
keywords = [
"msgpack",
"MessagePack",
"serde",
"serialization",
]
categories = ["encoding"]
license = "MIT"
repository = "https://github.com/3Hren/msgpack-rust"
[package.metadata.release]
tag-prefix = "{{crate_name}}/"
[dependencies.byteorder]
version = "1.4.3"
[dependencies.rmp]
version = "0.8.14"
[dependencies.serde]
version = "1.0.197"
[dev-dependencies.serde]
version = "1.0.197"
features = ["derive"]
[dev-dependencies.serde_bytes]
version = "0.11.5"
[badges.maintenance]
status = "looking-for-maintainer"

21
third_party/rust/rmp-serde/LICENSE vendored Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2017 Evgeny Safronov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

60
third_party/rust/rmp-serde/README.md vendored Normal file
View File

@@ -0,0 +1,60 @@
# MessagePack + Serde
This crate connects Rust MessagePack library with [`serde`][serde] providing an ability to
easily serialize and deserialize both Rust built-in types, the standard library and custom data
structures.
## Motivating example
```rust
let buf = rmp_serde::to_vec(&(42, "the Answer")).unwrap();
assert_eq!(
vec![0x92, 0x2a, 0xaa, 0x74, 0x68, 0x65, 0x20, 0x41, 0x6e, 0x73, 0x77, 0x65, 0x72],
buf
);
assert_eq!((42, "the Answer"), rmp_serde::from_slice(&buf).unwrap());
```
## Type-based Serialization and Deserialization
Serde provides a mechanism for low boilerplate serialization & deserialization of values to and
from MessagePack via the serialization API.
To be able to serialize a piece of data, it must implement the `serde::Serialize` trait. To be
able to deserialize a piece of data, it must implement the `serde::Deserialize` trait. Serde
provides an annotation to automatically generate the code for these
traits: `#[derive(Serialize, Deserialize)]`.
## Examples
```rust
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use rmp_serde::{Deserializer, Serializer};
#[derive(Debug, PartialEq, Deserialize, Serialize)]
struct Human {
age: u32,
name: String,
}
fn main() {
let mut buf = Vec::new();
let val = Human {
age: 42,
name: "John".into(),
};
val.serialize(&mut Serializer::new(&mut buf)).unwrap();
}
```
## Efficient storage of `&[u8]` types
MessagePack can efficiently store binary data. However, Serde's standard derived implementations *do not* use binary representations by default. Serde prefers to represent types like `&[u8; N]` or `Vec<u8>` as arrays of objects of arbitrary/unknown type, and not as slices of bytes. This creates about a 50% overhead in storage size.
Wrap your data in [`serde_bytes`](https://lib.rs/crates/serde_bytes) to store blobs quickly and efficiently. Alternatively, [configure an override in `rmp_serde` to force use of byte slices](https://docs.rs/rmp-serde/latest/rmp_serde/encode/struct.Serializer.html#method.with_bytes).
[serde]: https://serde.rs/

35
third_party/rust/rmp-serde/benches/buf.rs vendored Normal file
View File

@@ -0,0 +1,35 @@
#![feature(test)]
extern crate test;
use serde::{Deserialize, Serialize};
use test::Bencher;
#[bench]
fn bench_strings_1000(bencher: &mut Bencher) {
bench_strings(bencher, 1000);
}
#[bench]
fn bench_strings_5000(bencher: &mut Bencher) {
bench_strings(bencher, 5000);
}
#[bench]
fn bench_strings_10000(bencher: &mut Bencher) {
bench_strings(bencher, 10000);
}
fn bench_strings(bencher: &mut Bencher, size: usize) {
let vec: Vec<String> = ::std::iter::repeat("abcdefghijklmnopqrstuvwxyz".into())
.take(size)
.collect();
let mut buf = Vec::new();
vec.serialize(&mut rmp_serde::Serializer::new(&mut buf)).unwrap();
bencher.iter(|| {
<Vec<String>>::deserialize(&mut rmp_serde::Deserializer::new(&buf[..])).unwrap();
});
}

171
third_party/rust/rmp-serde/src/bytes.rs vendored Normal file
View File

@@ -0,0 +1,171 @@
/// Hacky serializer that only allows `u8`
use std::fmt;
use serde::ser::Impossible;
use serde::Serialize;
pub(crate) struct OnlyBytes;
pub(crate) struct Nope;
impl std::error::Error for Nope {
}
impl std::fmt::Display for Nope {
fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result {
Ok(())
}
}
impl std::fmt::Debug for Nope {
fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result {
Ok(())
}
}
impl serde::ser::Error for Nope {
fn custom<T: fmt::Display>(_: T) -> Self {
Self
}
}
impl serde::de::Error for Nope {
fn custom<T: fmt::Display>(_: T) -> Self {
Self
}
}
impl serde::Serializer for OnlyBytes {
type Ok = u8;
type Error = Nope;
type SerializeSeq = Impossible<u8, Nope>;
type SerializeTuple = Impossible<u8, Nope>;
type SerializeTupleStruct = Impossible<u8, Nope>;
type SerializeTupleVariant = Impossible<u8, Nope>;
type SerializeMap = Impossible<u8, Nope>;
type SerializeStruct = Impossible<u8, Nope>;
type SerializeStructVariant = Impossible<u8, Nope>;
fn serialize_u8(self, val: u8) -> Result<u8, Nope> {
Ok(val)
}
fn serialize_bool(self, _: bool) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_i8(self, _: i8) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_i16(self, _: i16) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_i32(self, _: i32) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_i64(self, _: i64) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_u16(self, _: u16) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_u32(self, _: u32) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_u64(self, _: u64) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_f32(self, _: f32) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_f64(self, _: f64) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_char(self, _: char) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_str(self, _: &str) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_bytes(self, _: &[u8]) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_none(self) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_some<T: ?Sized>(self, _: &T) -> Result<u8, Nope> where T: Serialize {
Err(Nope)
}
fn serialize_unit(self) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_unit_struct(self, _: &'static str) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_unit_variant(self, _: &'static str, _: u32, _: &'static str) -> Result<u8, Nope> {
Err(Nope)
}
fn serialize_newtype_struct<T: ?Sized>(self, _: &'static str, _: &T) -> Result<u8, Nope> where T: Serialize {
Err(Nope)
}
fn serialize_newtype_variant<T: ?Sized>(self, _: &'static str, _: u32, _: &'static str, _: &T) -> Result<u8, Nope> where T: Serialize {
Err(Nope)
}
fn serialize_seq(self, _: Option<usize>) -> Result<Self::SerializeSeq, Nope> {
Err(Nope)
}
fn serialize_tuple(self, _: usize) -> Result<Self::SerializeTuple, Nope> {
Err(Nope)
}
fn serialize_tuple_struct(self, _: &'static str, _: usize) -> Result<Self::SerializeTupleStruct, Nope> {
Err(Nope)
}
fn serialize_tuple_variant(self, _: &'static str, _: u32, _: &'static str, _: usize) -> Result<Self::SerializeTupleVariant, Nope> {
Err(Nope)
}
fn serialize_map(self, _: Option<usize>) -> Result<Self::SerializeMap, Nope> {
Err(Nope)
}
fn serialize_struct(self, _: &'static str, _: usize) -> Result<Self::SerializeStruct, Nope> {
Err(Nope)
}
fn serialize_struct_variant(self, _: &'static str, _: u32, _: &'static str, _: usize) -> Result<Self::SerializeStructVariant, Nope> {
Err(Nope)
}
fn collect_seq<I>(self, _: I) -> Result<u8, Nope> where I: IntoIterator, <I as IntoIterator>::Item: Serialize {
Err(Nope)
}
fn collect_map<K, V, I>(self, _: I) -> Result<u8, Nope> where K: Serialize, V: Serialize, I: IntoIterator<Item = (K, V)> {
Err(Nope)
}
fn collect_str<T: ?Sized>(self, _: &T) -> Result<u8, Nope> where T: fmt::Display {
Err(Nope)
}
}

245
third_party/rust/rmp-serde/src/config.rs vendored Normal file
View File

@@ -0,0 +1,245 @@
//! Change MessagePack behavior with configuration wrappers.
/// Represents configuration that dicatates what the serializer does.
///
/// Implemented as an empty trait depending on a hidden trait in order to allow changing the
/// methods of this trait without breaking backwards compatibility.
pub trait SerializerConfig: sealed::SerializerConfig {}
impl<T: sealed::SerializerConfig> SerializerConfig for T {}
pub(crate) mod sealed {
use crate::config::BytesMode;
/// This is the inner trait - the real `SerializerConfig`.
///
/// This hack disallows external implementations and usage of `SerializerConfig` and thus
/// allows us to change `SerializerConfig` methods freely without breaking backwards compatibility.
pub trait SerializerConfig: Copy {
/// Determines the value of `Serializer::is_human_readable` and
/// `Deserializer::is_human_readable`.
fn is_human_readable(&self) -> bool;
/// String struct fields
fn is_named(&self) -> bool;
fn bytes(&self) -> BytesMode;
}
}
#[derive(Copy, Clone, Debug)]
pub(crate) struct RuntimeConfig {
pub(crate) is_human_readable: bool,
pub(crate) is_named: bool,
pub(crate) bytes: BytesMode,
}
/// When to encode `[u8]` as `bytes` rather than a sequence
/// of integers. Serde without `serde_bytes` has trouble
/// using `bytes`, and this is hack to force it. It may
/// break some data types.
#[non_exhaustive]
#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)]
pub enum BytesMode {
/// Use bytes only when Serde requires it
/// (typically only when `serde_bytes` is used)
#[default]
Normal,
/// Use bytes for slices, `Vec`, and a few other types that
/// use `Iterator` in Serde.
///
/// This may break some implementations of `Deserialize`.
///
/// This does not include fixed-length arrays.
ForceIterables,
/// Use bytes for everything that looks like a container of `u8`.
/// This breaks some implementations of `Deserialize`.
ForceAll,
}
impl RuntimeConfig {
pub(crate) fn new(other: impl sealed::SerializerConfig) -> Self {
Self {
is_human_readable: other.is_human_readable(),
is_named: other.is_named(),
bytes: other.bytes(),
}
}
}
impl sealed::SerializerConfig for RuntimeConfig {
#[inline]
fn is_human_readable(&self) -> bool {
self.is_human_readable
}
#[inline]
fn is_named(&self) -> bool {
self.is_named
}
#[inline]
fn bytes(&self) -> BytesMode {
self.bytes
}
}
/// The default serializer/deserializer configuration.
///
/// This configuration:
/// - Writes structs as a tuple, without field names
/// - Writes enum variants as integers
/// - Writes and reads types as binary, not human-readable
//
/// This is the most compact representation.
#[derive(Copy, Clone, Debug)]
pub struct DefaultConfig;
impl sealed::SerializerConfig for DefaultConfig {
#[inline(always)]
fn is_named(&self) -> bool {
false
}
#[inline(always)]
fn is_human_readable(&self) -> bool {
false
}
#[inline(always)]
fn bytes(&self) -> BytesMode {
BytesMode::default()
}
}
/// Config wrapper, that overrides struct serialization by packing as a map with field names.
///
/// MessagePack specification does not tell how to serialize structs. This trait allows you to
/// extend serialization to match your app's requirements.
///
/// Default `Serializer` implementation writes structs as a tuple, i.e. only its length is encoded,
/// because it is the most compact representation.
#[derive(Copy, Clone, Debug)]
pub struct StructMapConfig<C>(C);
impl<C> StructMapConfig<C> {
/// Creates a `StructMapConfig` inheriting unchanged configuration options from the given configuration.
#[inline]
pub fn new(inner: C) -> Self {
StructMapConfig(inner)
}
}
impl<C> sealed::SerializerConfig for StructMapConfig<C>
where
C: sealed::SerializerConfig,
{
#[inline(always)]
fn is_named(&self) -> bool {
true
}
#[inline(always)]
fn is_human_readable(&self) -> bool {
self.0.is_human_readable()
}
fn bytes(&self) -> BytesMode {
self.0.bytes()
}
}
/// Config wrapper that overrides struct serlization by packing as a tuple without field
/// names.
#[derive(Copy, Clone, Debug)]
pub struct StructTupleConfig<C>(C);
impl<C> StructTupleConfig<C> {
/// Creates a `StructTupleConfig` inheriting unchanged configuration options from the given configuration.
#[inline]
pub fn new(inner: C) -> Self {
StructTupleConfig(inner)
}
}
impl<C> sealed::SerializerConfig for StructTupleConfig<C>
where
C: sealed::SerializerConfig,
{
#[inline(always)]
fn is_named(&self) -> bool {
false
}
#[inline(always)]
fn is_human_readable(&self) -> bool {
self.0.is_human_readable()
}
fn bytes(&self) -> BytesMode {
self.0.bytes()
}
}
/// Config wrapper that overrides `Serializer::is_human_readable` and
/// `Deserializer::is_human_readable` to return `true`.
#[derive(Copy, Clone, Debug)]
pub struct HumanReadableConfig<C>(C);
impl<C> HumanReadableConfig<C> {
/// Creates a `HumanReadableConfig` inheriting unchanged configuration options from the given configuration.
#[inline]
pub fn new(inner: C) -> Self {
Self(inner)
}
}
impl<C> sealed::SerializerConfig for HumanReadableConfig<C>
where
C: sealed::SerializerConfig,
{
#[inline(always)]
fn is_named(&self) -> bool {
self.0.is_named()
}
#[inline(always)]
fn is_human_readable(&self) -> bool {
true
}
fn bytes(&self) -> BytesMode {
self.0.bytes()
}
}
/// Config wrapper that overrides `Serializer::is_human_readable` and
/// `Deserializer::is_human_readable` to return `false`.
#[derive(Copy, Clone, Debug)]
pub struct BinaryConfig<C>(C);
impl<C> BinaryConfig<C> {
/// Creates a `BinaryConfig` inheriting unchanged configuration options from the given configuration.
#[inline(always)]
pub fn new(inner: C) -> Self {
Self(inner)
}
}
impl<C> sealed::SerializerConfig for BinaryConfig<C>
where
C: sealed::SerializerConfig,
{
#[inline(always)]
fn is_named(&self) -> bool {
self.0.is_named()
}
#[inline(always)]
fn is_human_readable(&self) -> bool {
false
}
fn bytes(&self) -> BytesMode {
self.0.bytes()
}
}

1193
third_party/rust/rmp-serde/src/decode.rs vendored Normal file
View File

File diff suppressed because it is too large Load Diff

1271
third_party/rust/rmp-serde/src/encode.rs vendored Normal file
View File

File diff suppressed because it is too large Load Diff

337
third_party/rust/rmp-serde/src/lib.rs vendored Normal file
View File

@@ -0,0 +1,337 @@
#![doc = include_str!("../README.md")]
#![forbid(unsafe_code)]
#![warn(missing_debug_implementations, missing_docs)]
use std::fmt::{self, Display, Formatter};
use std::str::{self, Utf8Error};
use serde::de;
use serde::{Deserialize, Serialize};
#[allow(deprecated)]
pub use crate::decode::from_read_ref;
pub use crate::decode::{from_read, Deserializer};
pub use crate::encode::{to_vec, to_vec_named, Serializer};
pub use crate::decode::from_slice;
mod bytes;
pub mod config;
pub mod decode;
pub mod encode;
/// Hack used to serialize MessagePack Extension types.
///
/// A special `ExtStruct` type is used to represent
/// extension types. This struct is renamed in serde.
///
/// Name of Serde newtype struct to Represent Msgpack's Ext
/// Msgpack Ext: `Ext(tag, binary)`
/// Serde data model: `_ExtStruct((tag, binary))`
///
/// Example Serde impl for custom type:
///
/// ```ignore
/// #[derive(Debug, PartialEq, Serialize, Deserialize)]
/// #[serde(rename = "_ExtStruct")]
/// struct ExtStruct((i8, serde_bytes::ByteBuf));
///
/// test_round(ExtStruct((2, serde_bytes::ByteBuf::from(vec![5]))),
/// Value::Ext(2, vec![5]));
/// ```
pub const MSGPACK_EXT_STRUCT_NAME: &str = "_ExtStruct";
/// Helper that allows both to encode and decode strings no matter whether they contain valid or
/// invalid UTF-8.
///
/// Regardless of validity the UTF-8 content this type will always be serialized as a string.
#[derive(Clone, Debug, PartialEq)]
#[doc(hidden)]
pub struct Raw {
s: Result<String, (Vec<u8>, Utf8Error)>,
}
impl Raw {
/// Constructs a new `Raw` from the UTF-8 string.
#[inline]
#[must_use]
pub fn new(v: String) -> Self {
Self { s: Ok(v) }
}
/// DO NOT USE. See <https://github.com/3Hren/msgpack-rust/issues/305>
#[deprecated(note = "This feature has been removed")]
#[must_use]
pub fn from_utf8(v: Vec<u8>) -> Self {
match String::from_utf8(v) {
Ok(v) => Raw::new(v),
Err(err) => {
let e = err.utf8_error();
Self {
s: Err((err.into_bytes(), e)),
}
}
}
}
/// Returns `true` if the raw is valid UTF-8.
#[inline]
#[must_use]
pub fn is_str(&self) -> bool {
self.s.is_ok()
}
/// Returns `true` if the raw contains invalid UTF-8 sequence.
#[inline]
#[must_use]
pub fn is_err(&self) -> bool {
self.s.is_err()
}
/// Returns the string reference if the raw is valid UTF-8, or else `None`.
#[inline]
#[must_use]
pub fn as_str(&self) -> Option<&str> {
match self.s {
Ok(ref s) => Some(s.as_str()),
Err(..) => None,
}
}
/// Returns the underlying `Utf8Error` if the raw contains invalid UTF-8 sequence, or
/// else `None`.
#[inline]
#[must_use]
pub fn as_err(&self) -> Option<&Utf8Error> {
match self.s {
Ok(..) => None,
Err((_, ref err)) => Some(err),
}
}
/// Returns a byte slice of this raw's contents.
#[inline]
#[must_use]
pub fn as_bytes(&self) -> &[u8] {
match self.s {
Ok(ref s) => s.as_bytes(),
Err(ref err) => &err.0[..],
}
}
/// Consumes this object, yielding the string if the raw is valid UTF-8, or else `None`.
#[inline]
#[must_use]
pub fn into_str(self) -> Option<String> {
self.s.ok()
}
/// Converts a `Raw` into a byte vector.
#[inline]
#[must_use]
pub fn into_bytes(self) -> Vec<u8> {
match self.s {
Ok(s) => s.into_bytes(),
Err(err) => err.0,
}
}
}
impl Serialize for Raw {
fn serialize<S>(&self, se: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self.s {
Ok(ref s) => se.serialize_str(s),
Err((ref b, ..)) => se.serialize_bytes(b),
}
}
}
struct RawVisitor;
impl<'de> de::Visitor<'de> for RawVisitor {
type Value = Raw;
#[cold]
fn expecting(&self, fmt: &mut Formatter<'_>) -> Result<(), fmt::Error> {
"string or bytes".fmt(fmt)
}
#[inline]
fn visit_string<E>(self, v: String) -> Result<Self::Value, E> {
Ok(Raw { s: Ok(v) })
}
#[inline]
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where E: de::Error
{
Ok(Raw { s: Ok(v.into()) })
}
#[inline]
fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
where E: de::Error
{
let s = match str::from_utf8(v) {
Ok(s) => Ok(s.into()),
Err(err) => Err((v.into(), err)),
};
Ok(Raw { s })
}
#[inline]
fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>
where E: de::Error
{
let s = match String::from_utf8(v) {
Ok(s) => Ok(s),
Err(err) => {
let e = err.utf8_error();
Err((err.into_bytes(), e))
}
};
Ok(Raw { s })
}
}
impl<'de> Deserialize<'de> for Raw {
#[inline]
fn deserialize<D>(de: D) -> Result<Self, D::Error>
where D: de::Deserializer<'de>
{
de.deserialize_any(RawVisitor)
}
}
/// Helper that allows both to encode and decode strings no matter whether they contain valid or
/// invalid UTF-8.
///
/// Regardless of validity the UTF-8 content this type will always be serialized as a string.
#[derive(Clone, Copy, Debug, PartialEq)]
#[doc(hidden)]
pub struct RawRef<'a> {
s: Result<&'a str, (&'a [u8], Utf8Error)>,
}
impl<'a> RawRef<'a> {
/// Constructs a new `RawRef` from the UTF-8 string.
#[inline]
#[must_use]
pub fn new(v: &'a str) -> Self {
Self { s: Ok(v) }
}
#[deprecated(note = "This feature has been removed")]
#[must_use]
pub fn from_utf8(v: &'a [u8]) -> Self {
match str::from_utf8(v) {
Ok(v) => RawRef::new(v),
Err(err) => {
Self {
s: Err((v, err))
}
}
}
}
/// Returns `true` if the raw is valid UTF-8.
#[inline]
#[must_use]
pub fn is_str(&self) -> bool {
self.s.is_ok()
}
/// Returns `true` if the raw contains invalid UTF-8 sequence.
#[inline]
#[must_use]
pub fn is_err(&self) -> bool {
self.s.is_err()
}
/// Returns the string reference if the raw is valid UTF-8, or else `None`.
#[inline]
#[must_use]
pub fn as_str(&self) -> Option<&str> {
match self.s {
Ok(s) => Some(s),
Err(..) => None,
}
}
/// Returns the underlying `Utf8Error` if the raw contains invalid UTF-8 sequence, or
/// else `None`.
#[inline]
#[must_use]
pub fn as_err(&self) -> Option<&Utf8Error> {
match self.s {
Ok(..) => None,
Err((_, ref err)) => Some(err),
}
}
/// Returns a byte slice of this raw's contents.
#[inline]
#[must_use]
pub fn as_bytes(&self) -> &[u8] {
match self.s {
Ok(s) => s.as_bytes(),
Err((bytes, _err)) => bytes,
}
}
}
impl<'a> Serialize for RawRef<'a> {
fn serialize<S>(&self, se: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self.s {
Ok(s) => se.serialize_str(s),
Err((b, ..)) => se.serialize_bytes(b),
}
}
}
struct RawRefVisitor;
impl<'de> de::Visitor<'de> for RawRefVisitor {
type Value = RawRef<'de>;
#[cold]
fn expecting(&self, fmt: &mut Formatter<'_>) -> Result<(), fmt::Error> {
"string or bytes".fmt(fmt)
}
#[inline]
fn visit_borrowed_str<E>(self, v: &'de str) -> Result<Self::Value, E>
where E: de::Error
{
Ok(RawRef { s: Ok(v) })
}
#[inline]
fn visit_borrowed_bytes<E>(self, v: &'de [u8]) -> Result<Self::Value, E>
where E: de::Error
{
let s = match str::from_utf8(v) {
Ok(s) => Ok(s),
Err(err) => Err((v, err)),
};
Ok(RawRef { s })
}
}
impl<'de> Deserialize<'de> for RawRef<'de> {
#[inline]
fn deserialize<D>(de: D) -> Result<Self, D::Error>
where D: de::Deserializer<'de>
{
de.deserialize_any(RawRefVisitor)
}
}

569
third_party/rust/rmp-serde/tests/decode.rs vendored Normal file
View File

@@ -0,0 +1,569 @@
use std::fmt::{self, Formatter};
use std::io::Cursor;
use serde::de;
use serde::Deserialize;
use rmp::Marker;
use rmp_serde::decode::{self, Error};
use rmp_serde::{Deserializer, Raw, RawRef};
#[test]
fn pass_nil() {
let buf = [0xc0];
let mut de = Deserializer::new(&buf[..]);
assert_eq!((), Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn fail_nil_from_reserved() {
let buf = [0xc1];
let mut de = Deserializer::new(&buf[..]);
let res: Result<(), Error> = Deserialize::deserialize(&mut de);
match res.err() {
Some(Error::TypeMismatch(Marker::Reserved)) => (),
other => panic!("unexpected result: {other:?}"),
}
}
#[test]
fn pass_bool() {
let buf = [0xc3, 0xc2];
let mut de = Deserializer::new(&buf[..]);
assert_eq!(true, Deserialize::deserialize(&mut de).unwrap());
assert_eq!(false, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn fail_bool_from_fixint() {
let buf = [0x00];
let cur = Cursor::new(&buf[..]);
let mut deserializer = Deserializer::new(cur);
let res: Result<bool, Error> = Deserialize::deserialize(&mut deserializer);
match res.err().unwrap() {
Error::Syntax(..) => (),
other => panic!("unexpected result: {other:?}"),
}
}
#[test]
fn pass_u64() {
let buf = [0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(18446744073709551615u64, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_u32() {
let buf = [0xce, 0xff, 0xff, 0xff, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(4294967295u32, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn fail_u32_from_u64() {
let buf = [0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let res: Result<u32, Error> = Deserialize::deserialize(&mut de);
match res.err().unwrap() {
Error::Syntax(..) => (),
other => panic!("unexpected result: {other:?}"),
}
}
#[test]
fn pass_u16() {
let buf = [0xcd, 0xff, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(65535u16, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_u8() {
let buf = [0xcc, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(255u8, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_u8_from_64() {
let buf = [0xcf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(42u8, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_usize() {
let buf = [0xcc, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(255usize, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_i64() {
let buf = [0xd3, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(9223372036854775807i64, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_i32() {
let buf = [0xd2, 0x7f, 0xff, 0xff, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(2147483647i32, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_i16() {
let buf = [0xd1, 0x7f, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(32767i16, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_i8() {
let buf = [0xd0, 0x7f];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(127i8, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_isize() {
let buf = [0xd0, 0x7f];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(127isize, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_f32() {
let buf = [0xca, 0x7f, 0x7f, 0xff, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(3.4028234e38_f32, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_f64() {
let buf = [0xcb, 0x40, 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(42f64, Deserialize::deserialize(&mut de).unwrap());
}
// spot check tests for general integers -> float conversions
#[test]
fn pass_i8_as_f32() {
let buf = [0xd0, 0x7f];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(127f32, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_u32_as_f64() {
let buf = [0xce, 0xff, 0xff, 0xff, 0xff];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!(4294967295f64, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_string() {
let buf = [0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual: String = Deserialize::deserialize(&mut de).unwrap();
assert_eq!("le message".to_string(), actual);
}
#[test]
fn pass_tuple() {
let buf = [0x92, 0x2a, 0xce, 0x0, 0x1, 0x88, 0x94];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual: (u32, u32) = Deserialize::deserialize(&mut de).unwrap();
assert_eq!((42, 100500), actual);
}
#[ignore]
#[test]
fn fail_tuple_len_mismatch() {
let buf = [0x92, 0x2a, 0xce, 0x0, 0x1, 0x88, 0x94];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual: Result<(u32,), Error> = Deserialize::deserialize(&mut de);
match actual.err().unwrap() {
Error::LengthMismatch(1) => (),
other => panic!("unexpected result: {other:?}"),
}
}
#[test]
fn pass_option_some() {
let buf = [0x1f];
let mut de = Deserializer::new(&buf[..]);
let actual: Option<u8> = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Some(31), actual);
}
#[test]
fn pass_option_none() {
let buf = [0xc0];
let mut de = Deserializer::new(&buf[..]);
let actual: Option<u8> = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(None, actual);
}
#[test]
fn pass_nested_option_some() {
let buf = [0x1f];
let mut de = Deserializer::new(&buf[..]);
let actual: Option<Option<u8>> = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Some(Some(31)), actual);
}
#[test]
fn pass_nested_option_none() {
let buf = [0xc0];
let mut de = Deserializer::new(&buf[..]);
let actual: Option<Option<u8>> = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(None, actual);
}
#[test]
fn fail_option_u8_from_reserved() {
let buf = [0xc1];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual: Result<Option<u8>, Error> = Deserialize::deserialize(&mut de);
match actual.err() {
Some(Error::TypeMismatch(Marker::Reserved)) => (),
other => panic!("unexpected result: {other:?}"),
}
}
#[test]
fn pass_vector() {
let buf = [0x92, 0x00, 0xcc, 0x80];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual: Vec<u8> = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(vec![0, 128], actual);
}
#[test]
fn pass_map() {
use std::collections::HashMap;
let buf = [
0x82, // 2 (size)
0xa3, 0x69, 0x6e, 0x74, // 'int'
0xcc, 0x80, // 128
0xa3, 0x6b, 0x65, 0x79, // 'key'
0x2a, // 42
];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual = Deserialize::deserialize(&mut de).unwrap();
let mut expected = HashMap::new();
expected.insert("int".to_string(), 128);
expected.insert("key".to_string(), 42);
assert_eq!(expected, actual);
}
// TODO: Merge three of them.
#[test]
fn pass_bin8_into_bytebuf() {
use serde_bytes::ByteBuf;
let buf = [0xc4, 0x02, 0xcc, 0x80];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual: ByteBuf = Deserialize::deserialize(&mut de).unwrap();
assert_eq!([0xcc, 0x80], actual[..]);
}
#[test]
fn pass_bin16_into_bytebuf() {
use serde_bytes::ByteBuf;
let buf = [0xc5, 0x00, 0x02, 0xcc, 0x80];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual: ByteBuf = Deserialize::deserialize(&mut de).unwrap();
assert_eq!([0xcc, 0x80], actual[..]);
}
#[test]
fn pass_bin32_into_bytebuf() {
use serde_bytes::ByteBuf;
let buf = [0xc6, 0x00, 0x00, 0x00, 0x02, 0xcc, 0x80];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual: ByteBuf = Deserialize::deserialize(&mut de).unwrap();
assert_eq!([0xcc, 0x80], actual[..]);
}
#[test]
fn pass_bin8_into_bytebuf_regression_growing_buffer() {
use serde_bytes::ByteBuf;
// Try to deserialize large buf and a small buf
let buf = [0x92, 0xc4, 0x04, 0x71, 0x75, 0x75, 0x78, 0xc4, 0x03, 0x62, 0x61, 0x72];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let (large, small): (ByteBuf, ByteBuf) = Deserialize::deserialize(&mut de).unwrap();
let (large, small): (Vec<u8>, Vec<u8>) = (large.into_vec(), small.into_vec());
assert_eq!((b"quux".to_vec(), b"bar".to_vec()), (large, small));
}
#[test]
fn test_deserialize_numeric() {
#[derive(Debug, PartialEq)]
enum FloatOrInteger {
Float(f64),
Integer(u64),
}
impl<'de> de::Deserialize<'de> for FloatOrInteger {
fn deserialize<D>(de: D) -> Result<FloatOrInteger, D::Error>
where D: de::Deserializer<'de>
{
struct FloatOrIntegerVisitor;
impl<'de> de::Visitor<'de> for FloatOrIntegerVisitor {
type Value = FloatOrInteger;
fn expecting(&self, fmt: &mut Formatter<'_>) -> Result<(), fmt::Error> {
write!(fmt, "either a float or an integer")
}
fn visit_u64<E>(self, value: u64) -> Result<FloatOrInteger, E> {
Ok(FloatOrInteger::Integer(value))
}
fn visit_f64<E>(self, value: f64) -> Result<FloatOrInteger, E> {
Ok(FloatOrInteger::Float(value))
}
}
de.deserialize_any(FloatOrIntegerVisitor)
}
}
let buf = [203, 64, 36, 102, 102, 102, 102, 102, 102]; // 10.2
let mut de = Deserializer::new(&buf[..]);
let x: FloatOrInteger = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(x, FloatOrInteger::Float(10.2));
let buf = [36]; // 36
let mut de = Deserializer::new(&buf[..]);
let x: FloatOrInteger = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(x, FloatOrInteger::Integer(36));
}
#[test]
fn pass_deserializer_get_ref() {
let buf = [0xc0];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!((), Deserialize::deserialize(&mut de).unwrap());
assert_eq!(1, de.get_ref().position());
}
#[test]
fn pass_deserializer_get_mut() {
let buf = [0xc0];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!((), Deserialize::deserialize(&mut de).unwrap());
de.get_mut().set_position(0);
assert_eq!((), Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn pass_deserializer_into_inner() {
let buf = [0xc0];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
assert_eq!((), Deserialize::deserialize(&mut de).unwrap());
let cur = de.into_inner();
assert_eq!(1, cur.position());
}
#[test]
fn pass_deserializer_cursor_position() {
let mut de = Deserializer::new(Cursor::new(vec![0xce, 0xff, 0xff, 0xff, 0xff]));
assert_eq!(4294967295u32, Deserialize::deserialize(&mut de).unwrap());
assert_eq!(5, de.position());
}
#[test]
fn pass_from() {
assert_eq!(2147483647, decode::from_read(&[0xd2, 0x7f, 0xff, 0xff, 0xff][..]).unwrap());
}
#[test]
fn pass_raw_valid_utf8() {
let buf = vec![0xa3, 0x6b, 0x65, 0x79];
let raw: Raw = rmp_serde::from_slice(&buf[..]).unwrap();
assert!(raw.is_str());
assert_eq!("key", raw.as_str().unwrap());
assert_eq!([0x6b, 0x65, 0x79], raw.as_bytes());
}
#[test]
fn pass_raw_invalid_utf8() {
// >>> msgpack.dumps(msgpack.dumps([200, []]))
// '\xa4\x92\xcc\xc8\x90'
let buf = vec![0xa4, 0x92, 0xcc, 0xc8, 0x90];
let raw: Raw = rmp_serde::from_slice(&buf[..]).unwrap();
assert!(raw.is_err());
assert_eq!(0, raw.as_err().unwrap().valid_up_to());
assert_eq!([0x92, 0xcc, 0xc8, 0x90], raw.as_bytes());
}
#[test]
fn pass_raw_ref_valid_utf8() {
let buf = vec![0xa3, 0x6b, 0x65, 0x79];
let raw: RawRef<'_> = rmp_serde::from_slice(&buf[..]).unwrap();
assert!(raw.is_str());
assert_eq!("key", raw.as_str().unwrap());
assert_eq!([0x6b, 0x65, 0x79], raw.as_bytes());
}
#[test]
fn pass_raw_ref_invalid_utf8() {
// >>> msgpack.dumps(msgpack.dumps([200, []]))
// '\xa4\x92\xcc\xc8\x90'
let buf = vec![0xa4, 0x92, 0xcc, 0xc8, 0x90];
let raw: RawRef<'_> = rmp_serde::from_slice(&buf[..]).unwrap();
assert!(raw.is_err());
assert_eq!(0, raw.as_err().unwrap().valid_up_to());
assert_eq!([0x92, 0xcc, 0xc8, 0x90], raw.as_bytes());
}
#[test]
fn fail_str_invalid_utf8() {
let buf = vec![0xa4, 0x92, 0xcc, 0xc8, 0x90];
let err: Result<String, decode::Error> = rmp_serde::from_slice(&buf[..]);
assert!(err.is_err());
match err.err().unwrap() {
decode::Error::Utf8Error(err) => assert_eq!(0, err.valid_up_to()),
// decode::Error::Syntax(err) => {}
err => panic!("unexpected error: {:?}", err),
}
}
#[test]
fn fail_depth_limit() {
#[allow(dead_code)]
struct Nested {
sub: Vec<Nested>,
}
impl<'de> de::Deserialize<'de> for Nested {
fn deserialize<D>(de: D) -> Result<Self, D::Error>
where D: de::Deserializer<'de>
{
let nested = Vec::deserialize(de)?;
Ok(Nested { sub: nested })
}
}
let mut data = Vec::new();
for _ in 0..100 {
data.push(0x91u8);
}
let mut reader = rmp_serde::Deserializer::new(Cursor::new(data));
reader.set_max_depth(100);
let res = Nested::deserialize(&mut reader);
match res.err().unwrap() {
decode::Error::DepthLimitExceeded => (),
other => panic!("unexpected result: {other:?}"),
}
}

526
third_party/rust/rmp-serde/tests/decode_derive.rs vendored Normal file
View File

@@ -0,0 +1,526 @@
use std::io::Cursor;
use serde::Deserialize;
use rmp_serde::decode::Error;
use rmp_serde::Deserializer;
#[test]
fn pass_newtype() {
let buf = [0x2a];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
struct Struct(u32);
let mut de = Deserializer::new(cur);
let actual: Struct = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Struct(42), actual);
}
#[test]
fn pass_tuple_struct() {
let buf = [0x92, 0x2a, 0xce, 0x0, 0x1, 0x88, 0x94];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
struct Decoded(u32, u32);
let mut de = Deserializer::new(cur);
let actual: Decoded = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Decoded(42, 100500), actual);
}
#[test]
fn pass_single_field_struct() {
let buf = [0x91, 0x2a];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
struct Struct {
inner: u32,
}
let mut de = Deserializer::new(cur);
let actual: Struct = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Struct { inner: 42 }, actual);
}
#[test]
fn pass_struct() {
let buf = [0x92, 0x2a, 0xce, 0x0, 0x1, 0x88, 0x94];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
struct Decoded {
id: u32,
value: u32,
}
let mut de = Deserializer::new(cur);
let actual: Decoded = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Decoded { id: 42, value: 100500 }, actual);
}
#[test]
fn pass_struct_from_map() {
#[derive(Debug, PartialEq, Deserialize)]
struct Struct {
et: String,
le: u8,
shit: u8,
}
let buf = [
0x83, // 3 (size)
0xa2, 0x65, 0x74, // "et"
0xa5, 0x76, 0x6f, 0x69, 0x6c, 0x61, // "voila"
0xa2, 0x6c, 0x65, // "le"
0x00, // 0
0xa4, 0x73, 0x68, 0x69, 0x74, // "shit"
0x01, // 1
];
let cur = Cursor::new(&buf[..]);
// It appears no special behavior is needed for deserializing structs encoded as maps.
let mut de = Deserializer::new(cur);
let actual: Struct = Deserialize::deserialize(&mut de).unwrap();
let expected = Struct { et: "voila".into(), le: 0, shit: 1 };
assert_eq!(expected, actual);
}
#[test]
fn pass_unit_variant() {
// We expect enums to be encoded as a map {variant_idx => nil}
let buf = [0x81, 0x0, 0xC0, 0x81, 0x1, 0xC0];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
enum Enum {
A,
B,
}
let mut de = Deserializer::new(cur);
let enum_a = Enum::deserialize(&mut de).unwrap();
let enum_b = Enum::deserialize(&mut de).unwrap();
assert_eq!(enum_a, Enum::A);
assert_eq!(enum_b, Enum::B);
assert_eq!(6, de.get_ref().position());
}
#[test]
fn pass_tuple_enum_with_arg() {
// The encoded byte-array is: {1 => 42}.
let buf = [0x81, 0x01, 0x2a];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
enum Enum {
A,
B(u32),
}
let mut de = Deserializer::new(cur);
let actual: Enum = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Enum::B(42), actual);
assert_eq!(3, de.get_ref().position());
}
#[test]
fn pass_tuple_enum_with_args() {
// The encoded bytearray is: {1 => [42, 58]}.
let buf = [0x81, 0x01, 0x92, 0x2a, 0x3a];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
enum Enum {
A,
B(u32, u32),
}
let mut de = Deserializer::new(cur);
let actual: Enum = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Enum::B(42, 58), actual);
assert_eq!(5, de.get_ref().position());
}
#[test]
fn fail_enum_map_mismatch() {
let buf = [0x82, 0x0, 0x24, 0x1, 0x25];
#[derive(Debug, PartialEq, Deserialize)]
enum Enum {
A(i32),
}
let err: Result<Enum, _> = rmp_serde::from_slice(&buf);
match err.unwrap_err() {
Error::LengthMismatch(2) => (),
other => panic!("unexpected result: {other:?}"),
}
}
#[test]
fn fail_enum_overflow() {
// The encoded bytearray is: {1 => [42]}.
let buf = [0x81, 0x01, 0x2a];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
// TODO: Rename to Enum: A, B, C, ...
enum Enum {
A,
}
let mut de = Deserializer::new(cur);
let actual: Result<Enum, Error> = Deserialize::deserialize(&mut de);
match actual.err().unwrap() {
Error::Syntax(..) => (),
other => panic!("unexpected result: {other:?}"),
}
}
#[test]
fn pass_struct_enum_with_arg() {
// The encoded bytearray is: {1 => [42]}.
let buf = [0x81, 0x01, 0x91, 0x2a];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
enum Enum {
A,
B { id: u32 },
}
let mut de = Deserializer::new(cur);
let actual: Enum = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Enum::B { id: 42 }, actual);
assert_eq!(4, de.get_ref().position());
}
#[test]
fn pass_newtype_variant() {
// The encoded bytearray is: {0 => 'le message'}.
let buf = [0x81, 0x0, 0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
struct Newtype(String);
#[derive(Debug, PartialEq, Deserialize)]
enum Enum {
A(Newtype),
}
let mut de = Deserializer::new(cur);
let actual: Enum = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Enum::A(Newtype("le message".into())), actual);
assert_eq!(buf.len() as u64, de.get_ref().position());
}
#[cfg(disabled)] // This test doesn't actually compile anymore
#[test]
fn pass_enum_custom_policy() {
use rmp_serde::decode::VariantVisitor;
use std::io::Read;
// We expect enums to be endoded as id, [...] (without wrapping tuple).
let buf = [0x01, 0x90];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
enum Enum {
A,
B,
}
struct CustomDeserializer<R: Read> {
inner: Deserializer<R>,
}
impl<R: Read> serde::Deserializer for CustomDeserializer<R> {
type Error = Error;
fn deserialize<V>(&mut self, visitor: V) -> Result<V::Value, Error>
where V: serde::de::Visitor
{
self.inner.deserialize(visitor)
}
fn deserialize_enum<V>(&mut self, _enum: &str, _variants: &'static [&'static str], mut visitor: V)
-> Result<V::Value, Error>
where V: serde::de::EnumVisitor
{
visitor.visit(VariantVisitor::new(&mut self.inner))
}
forward_to_deserialize! {
bool usize u8 u16 u32 u64 isize i8 i16 i32 i64 f32 f64 char str string unit seq
seq_fixed_size bytes map tuple_struct unit_struct struct struct_field
tuple option newtype_struct ignored_any
}
}
let mut de = CustomDeserializer {
inner: Deserializer::new(cur),
};
let actual: Enum = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Enum::B, actual);
assert_eq!(2, de.inner.get_ref().position());
}
#[test]
fn pass_struct_variant() {
#[derive(Debug, PartialEq, Deserialize)]
enum Custom {
First { data: u32 },
Second { data: u32 },
}
let out_first = vec![0x81, 0x00, 0x91, 0x2a];
let out_second = vec![0x81, 0x01, 0x91, 0x2a];
for (expected, out) in [(Custom::First{ data: 42 }, out_first), (Custom::Second { data: 42 }, out_second)] {
let mut de = Deserializer::new(Cursor::new(&out[..]));
let val: Custom = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(expected, val);
}
}
#[test]
fn pass_adjacently_tagged_enum() {
// ["Foo", 123]
let buf = [146, 163, 70, 111, 111, 123];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
#[serde(tag = "t", content = "c")]
enum Enum {
Foo(i32),
Bar(u32),
}
let mut de = Deserializer::new(cur);
let actual = Deserialize::deserialize(&mut de);
assert!(actual.is_ok());
assert_eq!(Enum::Foo(123), actual.unwrap());
}
#[test]
#[should_panic(expected = "assertion failed")]
fn fail_internally_tagged_enum_tuple() {
// ["Foo", 123]
let buf = [146, 163, 70, 111, 111, 123];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
#[serde(tag = "t")]
enum Enum {
Foo(i32),
Bar(u32),
}
let mut de = Deserializer::new(cur);
let actual: Result<Enum, Error> = Deserialize::deserialize(&mut de);
assert!(actual.is_ok());
}
#[test]
fn pass_internally_tagged_enum_struct() {
let buf = [130, 161, 116, 163, 70, 111, 111, 165, 118, 97, 108, 117, 101, 123];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
#[serde(tag = "t")]
enum Enum {
Foo { value: i32 },
Bar { value: u32 },
}
let mut de = Deserializer::new(cur);
let actual: Result<Enum, Error> = Deserialize::deserialize(&mut de);
assert!(actual.is_ok());
assert_eq!(Enum::Foo { value: 123 }, actual.unwrap());
}
#[test]
fn pass_enum_with_one_arg() {
// The encoded bytearray is: {0 => [1, 2]}.
let buf = [0x81, 0x0, 0x92, 0x01, 0x02];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
enum Enum {
V1(Vec<u32>),
}
let mut de = Deserializer::new(cur);
let actual: Enum = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Enum::V1(vec![1, 2]), actual);
assert_eq!(buf.len() as u64, de.get_ref().position());
}
#[test]
fn pass_struct_with_nested_options() {
// The encoded bytearray is: [null, 13].
let buf = [0x92, 0xc0, 0x0D];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
struct Struct {
f1: Option<Option<u32>>,
f2: Option<Option<u32>>,
}
let mut de = Deserializer::new(cur);
let actual: Struct = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(Struct { f1: None, f2: Some(Some(13)) }, actual);
assert_eq!(buf.len() as u64, de.get_ref().position());
}
#[test]
fn pass_struct_with_flattened_map_field() {
use std::collections::BTreeMap;
// The encoded bytearray is: { "f1": 0, "f2": { "german": "Hallo Welt!" }, "english": "Hello World!" }.
let buf = [
0x83, 0xA2, 0x66, 0x31, 0x00, 0xA2, 0x66, 0x32, 0x81, 0xA6, 0x67, 0x65, 0x72, 0x6D, 0x61, 0x6E, 0xAB,
0x48, 0x61, 0x6C, 0x6C, 0x6F, 0x20, 0x57, 0x65, 0x6C, 0x74, 0x21, 0xA7, 0x65, 0x6E, 0x67, 0x6C, 0x69,
0x73, 0x68, 0xAC, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21,
];
let cur = Cursor::new(&buf[..]);
#[derive(Debug, PartialEq, Deserialize)]
struct Struct {
f1: u32,
// not flattend!
f2: BTreeMap<String, String>,
#[serde(flatten)]
f3: BTreeMap<String, String>,
}
let expected = Struct {
f1: 0,
f2: {
let mut map = BTreeMap::new();
map.insert("german".to_string(), "Hallo Welt!".to_string());
map
},
f3: {
let mut map = BTreeMap::new();
map.insert("english".to_string(), "Hello World!".to_string());
map
},
};
let mut de = Deserializer::new(cur);
let actual: Struct = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(expected, actual);
assert_eq!(buf.len() as u64, de.get_ref().position());
}
#[test]
fn pass_struct_with_flattened_struct_field() {
#[derive(Debug, PartialEq, Deserialize)]
struct Struct {
f1: u32,
// not flattend!
f2: InnerStruct,
#[serde(flatten)]
f3: InnerStruct,
}
#[derive(Debug, PartialEq, Deserialize)]
struct InnerStruct {
f4: u32,
f5: u32,
}
let expected = Struct {
f1: 0,
f2: InnerStruct { f4: 8, f5: 13 },
f3: InnerStruct { f4: 21, f5: 34 },
};
// struct-as-tuple
{
// The encoded bytearray is: { "f1": 0, "f2": [8, 13], "f4": 21, "f5": 34 }.
let buf = [
0x84, 0xA2, 0x66, 0x31, 0x00, 0xA2, 0x66, 0x32, 0x92, 0x08, 0x0D, 0xA2, 0x66, 0x34, 0x15, 0xA2, 0x66, 0x35,
0x22,
];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual: Struct = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(expected, actual);
assert_eq!(buf.len() as u64, de.get_ref().position());
}
// struct-as-map
{
// The encoded bytearray is: { "f1": 0, "f2": { "f4": 8, "f5": 13 }, "f4": 21, "f5": 34 }.
let buf = [
0x84, 0xA2, 0x66, 0x31, 0x00, 0xA2, 0x66, 0x32, 0x82, 0xA2, 0x66, 0x34, 0x08,
0xA2, 0x66, 0x35, 0x0D, 0xA2, 0x66, 0x34, 0x15, 0xA2, 0x66, 0x35, 0x22,
];
let cur = Cursor::new(&buf[..]);
let mut de = Deserializer::new(cur);
let actual: Struct = Deserialize::deserialize(&mut de).unwrap();
assert_eq!(expected, actual);
assert_eq!(buf.len() as u64, de.get_ref().position());
}
}
#[test]
fn pass_from_slice() {
let buf = [0x93, 0xa4, 0x4a, 0x6f, 0x68, 0x6e, 0xa5, 0x53, 0x6d, 0x69, 0x74, 0x68, 0x2a];
#[derive(Debug, PartialEq, Deserialize)]
struct Person<'a> {
name: &'a str,
surname: &'a str,
age: u8,
}
assert_eq!(Person { name: "John", surname: "Smith", age: 42 }, rmp_serde::from_slice(&buf[..]).unwrap());
}
#[test]
#[allow(deprecated)]
fn pass_from_ref() {
let buf = [0x92, 0xa5, 0x42, 0x6f, 0x62, 0x62, 0x79, 0x8];
#[derive(Debug, Deserialize, PartialEq)]
struct Dog<'a> {
name: &'a str,
age: u8,
}
assert_eq!(Dog { name: "Bobby", age: 8 }, rmp_serde::from_read_ref(&buf).unwrap());
}

409
third_party/rust/rmp-serde/tests/encode.rs vendored Normal file
View File

@@ -0,0 +1,409 @@
extern crate rmp_serde as rmps;
use std::io::Cursor;
use rmps::config::BytesMode;
use serde::Serialize;
use rmp_serde::encode::{self, Error};
use rmp_serde::{Raw, RawRef, Serializer};
#[test]
fn pass_null() {
let mut buf = [0x00];
let val = ();
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xc0], buf);
}
#[test]
fn fail_null() {
let mut buf = [];
let val = ();
match val.serialize(&mut Serializer::new(&mut &mut buf[..])) {
Err(Error::InvalidValueWrite(..)) => (),
other => panic!("unexpected result: {other:?}"),
}
}
#[test]
fn pass_bool() {
let mut buf = [0x00, 0x00];
{
let mut cur = Cursor::new(&mut buf[..]);
let mut encoder = Serializer::new(&mut cur);
let val = true;
val.serialize(&mut encoder).ok().unwrap();
let val = false;
val.serialize(&mut encoder).ok().unwrap();
}
assert_eq!([0xc3, 0xc2], buf);
}
#[test]
fn pass_usize() {
let mut buf = [0x00, 0x00];
let val = 255usize;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xcc, 0xff], buf);
}
#[test]
fn pass_u8() {
let mut buf = [0x00, 0x00];
let val = 255u8;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xcc, 0xff], buf);
}
#[test]
fn pass_u16() {
let mut buf = [0x00, 0x00, 0x00];
let val = 65535u16;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xcd, 0xff, 0xff], buf);
}
#[test]
fn pass_u32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
let val = 4294967295u32;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xce, 0xff, 0xff, 0xff, 0xff], buf);
}
#[test]
fn pass_u64() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let val = 18446744073709551615u64;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], buf);
}
#[test]
fn pass_isize() {
let mut buf = [0x00, 0x00];
let val = -128isize;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xd0, 0x80], buf);
}
#[test]
fn pass_i8() {
let mut buf = [0x00, 0x00];
let val = -128i8;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xd0, 0x80], buf);
}
#[test]
fn pass_i16() {
let mut buf = [0x00, 0x00, 0x00];
let val = -32768i16;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xd1, 0x80, 0x00], buf);
}
#[test]
fn pass_i32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
let val = -2147483648i32;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xd2, 0x80, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_i64() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let val = -9223372036854775808i64;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xd3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_i64_most_effective() {
let mut buf = [0x00, 0x00];
// This value can be represented using 2 bytes although it's i64.
let val = 128i64;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).unwrap();
assert_eq!([0xcc, 0x80], buf);
}
#[test]
fn pass_f32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
let val = 3.4028234e38_f32;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xca, 0x7f, 0x7f, 0xff, 0xff], buf);
}
#[test]
fn pass_f64() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let val = 42f64;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xcb, 0x40, 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_char() {
let mut buf = [0x00, 0x00];
let val = '!';
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xa1, 0x21], buf);
}
#[test]
fn pass_string() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let val = "le message";
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65], buf);
}
#[test]
fn pass_tuple() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let val = (42u32, 100500u32);
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0x92, 0x2a, 0xce, 0x0, 0x1, 0x88, 0x94], buf);
}
#[test]
fn pass_tuple_not_bytes() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let val = (42u32, 100500u32);
val.serialize(&mut Serializer::new(&mut &mut buf[..]).with_bytes(BytesMode::ForceAll)).ok().unwrap();
assert_eq!([0x92, 0x2a, 0xce, 0x0, 0x1, 0x88, 0x94], buf);
}
#[test]
fn pass_tuple_bytes() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let val = (1u8, 100u8, 200u8, 254u8);
val.serialize(&mut Serializer::new(&mut &mut buf[..]).with_bytes(BytesMode::ForceAll)).ok().unwrap();
assert_eq!([196, 4, 1, 100, 200, 254], buf);
}
#[test]
fn pass_hash_array_bytes() {
use std::collections::HashSet;
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let val = [[255u8; 3], [1u8; 3]].into_iter().collect::<HashSet<[u8;3]>>();
val.serialize(&mut Serializer::new(&mut &mut buf[..]).with_bytes(BytesMode::ForceAll)).ok().unwrap();
}
#[test]
fn pass_tuple_low_bytes() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
let val = (1u8, 2, 3, 127);
val.serialize(&mut Serializer::new(&mut &mut buf[..]).with_bytes(BytesMode::ForceAll)).ok().unwrap();
assert_eq!([148, 1, 2, 3, 127], buf);
}
#[test]
fn pass_option_some() {
let mut buf = [0x00];
let val = Some(100u32);
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0x64], buf);
}
#[test]
fn pass_option_none() {
let mut buf = [0x00];
let val: Option<u32> = None;
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0xc0], buf);
}
#[test]
fn pass_seq() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let val = vec!["le", "shit"];
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
assert_eq!([0x92, 0xa2, 0x6c, 0x65, 0xa4, 0x73, 0x68, 0x69, 0x74], buf);
}
#[test]
fn pass_map() {
use std::collections::BTreeMap;
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let mut val = BTreeMap::new();
val.insert(0u8, "le");
val.insert(1u8, "shit");
val.serialize(&mut Serializer::new(&mut &mut buf[..])).ok().unwrap();
let out = [
0x82, // 2 (size)
0x00, // 0
0xa2, 0x6c, 0x65, // "le"
0x01, // 1
0xa4, 0x73, 0x68, 0x69, 0x74, // "shit"
];
assert_eq!(out, buf);
}
#[test]
fn pass_empty_map() {
use std::collections::BTreeMap;
let mut buf = vec![];
let val: BTreeMap<u64, u64> = BTreeMap::new();
val.serialize(&mut Serializer::new(&mut buf)).ok().unwrap();
let out = vec![
0x80, // (size: 0)
];
assert_eq!(out, buf);
}
#[test]
fn pass_encoding_struct_into_vec() {
let val = (42u8, "the Answer");
let mut buf: Vec<u8> = Vec::new();
val.serialize(&mut Serializer::new(&mut buf)).unwrap();
assert_eq!(vec![0x92, 0x2a, 0xaa, 0x74, 0x68, 0x65, 0x20, 0x41, 0x6e, 0x73, 0x77, 0x65, 0x72], buf);
}
#[test]
fn pass_bin() {
use serde_bytes::Bytes;
let mut buf = Vec::new();
let val = Bytes::new(&[0xcc, 0x80]);
val.serialize(&mut Serializer::new(&mut buf)).ok().unwrap();
assert_eq!(vec![0xc4, 0x02, 0xcc, 0x80], buf);
}
#[test]
fn pass_to_vec() {
assert_eq!(vec![0xc0], encode::to_vec(&()).unwrap());
assert_eq!(vec![0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65],
encode::to_vec("le message").unwrap());
}
#[test]
fn get_mut() {
let mut se = Serializer::new(Vec::new());
true.serialize(&mut se).unwrap();
assert_eq!(&vec![0xc3], se.get_ref());
se.get_mut().push(42);
assert_eq!(vec![0xc3, 42], se.into_inner());
}
#[test]
fn pass_raw_valid_utf8() {
let raw = Raw::new("key".into());
let mut buf = Vec::new();
raw.serialize(&mut Serializer::new(&mut buf)).unwrap();
assert_eq!(vec![0xa3, 0x6b, 0x65, 0x79], buf);
}
#[test]
#[allow(deprecated)]
fn pass_raw_invalid_utf8() {
// >>> msgpack.dumps(msgpack.dumps([200, []]))
// '\xa4\x92\xcc\xc8\x90'
let raw = Raw::from_utf8(vec![0x92, 0xcc, 0xc8, 0x90]);
let mut buf = Vec::new();
raw.serialize(&mut Serializer::new(&mut buf)).unwrap();
assert_eq!(vec![196, 4, 146, 204, 200, 144], buf);
}
#[test]
fn pass_raw_ref_valid_utf8() {
let raw = RawRef::new("key");
let mut buf = Vec::new();
raw.serialize(&mut Serializer::new(&mut buf)).unwrap();
assert_eq!(vec![0xa3, 0x6b, 0x65, 0x79], buf);
}
#[test]
#[allow(deprecated)]
fn pass_raw_ref_invalid_utf8() {
// >>> msgpack.dumps(msgpack.dumps([200, []]))
// '\xa4\x92\xcc\xc8\x90'
let b = &[0x92, 0xcc, 0xc8, 0x90];
let raw = RawRef::from_utf8(b);
let mut buf = Vec::new();
raw.serialize(&mut Serializer::new(&mut buf)).unwrap();
assert_eq!(vec![196, 4, 146, 204, 200, 144], buf);
}
#[test]
fn serializer_one_type_arg() {
let _s: rmp_serde::Serializer<&mut dyn std::io::Write>;
}

355
third_party/rust/rmp-serde/tests/encode_derive.rs vendored Normal file
View File

@@ -0,0 +1,355 @@
use rmp_serde::Serializer;
use serde::Serialize;
#[test]
fn pass_unit_struct() {
#[derive(Serialize)]
struct Unit;
let mut buf = Vec::new();
Unit.serialize(&mut Serializer::new(&mut buf)).unwrap();
// Expect: [].
assert_eq!(vec![0x90], buf);
}
#[test]
fn pass_unit_variant() {
#[derive(Serialize)]
enum Enum {
V1,
V2,
}
let mut buf = Vec::new();
Enum::V1.serialize(&mut Serializer::new(&mut buf)).unwrap();
Enum::V2.serialize(&mut Serializer::new(&mut buf)).unwrap();
// Expect: "V1", "V2"
assert_eq!(vec![0xa2, 0x56, 0x31, 0xa2, 0x56, 0x32], buf);
}
#[test]
fn pass_newtype_struct() {
#[derive(Serialize)]
struct Struct(u64);
let val = Struct(42);
let mut buf = Vec::new();
val.serialize(&mut Serializer::new(&mut buf)).unwrap();
assert_eq!(vec![0x2a], buf);
}
#[test]
fn pass_newtype_variant() {
#[derive(Serialize)]
enum Enum {
V2(u64),
}
let mut buf = Vec::new();
Enum::V2(42).serialize(&mut Serializer::new(&mut buf)).unwrap();
// Expect: {"V2" => 42}
assert_eq!(buf, vec![0x81, 0xa2, 0x56, 0x32, 42]);
}
#[test]
fn pass_untagged_newtype_variant() {
#[derive(Serialize)]
#[serde(untagged)]
enum Enum1 {
A(u64),
B(Enum2),
}
#[derive(Serialize)]
enum Enum2 {
C,
}
let buf1 = rmp_serde::to_vec(&Enum1::A(123)).unwrap();
let buf2 = rmp_serde::to_vec(&Enum1::B(Enum2::C)).unwrap();
assert_eq!(buf1, [123]);
// Expect: "C"
assert_eq!(buf2, [0xa1, 0x43]);
}
#[test]
fn pass_tuple_struct() {
#[derive(Serialize)]
struct Struct(u32, u64);
let val = Struct(42, 100500);
let mut buf = Vec::new();
val.serialize(&mut Serializer::new(&mut buf)).unwrap();
// Expect: [42, 100500].
assert_eq!(vec![0x92, 0x2a, 0xce, 0x00, 0x01, 0x88, 0x94], buf);
}
#[test]
fn pass_tuple_variant() {
#[derive(Serialize)]
enum Enum {
V1,
V2(u32, u64),
}
let mut buf = Vec::new();
Enum::V1.serialize(&mut Serializer::new(&mut buf)).unwrap();
Enum::V2(42, 100500).serialize(&mut Serializer::new(&mut buf)).unwrap();
// Expect: {0 => nil} {1 => [42, 100500]}
// Expect: "V1", {"V2" => [42, 100500] }
assert_eq!(
vec![0xa2, 0x56, 0x31, 0x81, 0xa2, 0x56, 0x32, 0x92, 0x2a, 0xce, 0x0, 0x1, 0x88, 0x94],
buf
);
}
#[test]
fn pass_struct() {
#[derive(Serialize)]
struct Struct {
f1: u32,
f2: u32,
}
let val = Struct {
f1: 42,
f2: 100500,
};
let mut buf = Vec::new();
val.serialize(&mut Serializer::new(&mut buf)).unwrap();
// Expect: [42, 100500].
assert_eq!(vec![0x92, 0x2a, 0xce, 0x0, 0x1, 0x88, 0x94], buf);
}
#[test]
fn serialize_struct_variant() {
#[derive(Serialize)]
enum Enum {
V1 { f1: u32 },
V2 { f1: u32 },
}
let mut buf = Vec::new();
Enum::V1 { f1: 42 }.serialize(&mut Serializer::new(&mut buf)).unwrap();
Enum::V2 { f1: 43 }.serialize(&mut Serializer::new(&mut buf)).unwrap();
// Expect: { "V1" => [42] } { "V2" => [43] }
assert_eq!(
vec![0x81, 0xa2, 0x56, 0x31, 0x91, 0x2a, 0x81, 0xa2, 0x56, 0x32, 0x91, 0x2b],
buf
);
}
#[test]
fn serialize_struct_variant_as_map() {
#[derive(Serialize)]
enum Enum {
V1 { f1: u32 },
}
let mut se = Serializer::new(Vec::new()).with_struct_map();
Enum::V1 { f1: 42 }.serialize(&mut se).unwrap();
// Expect: {"V1" => {"f1": 42}}.
assert_eq!(
vec![0x81, 0xa2, 0x56, 0x31, 0x81, 0xa2, 0x66, 0x31, 0x2a],
se.into_inner()
);
}
#[test]
fn serialize_struct_with_flattened_map_field() {
use std::collections::BTreeMap;
#[derive(Serialize)]
struct Struct {
f1: u32,
// not flattend!
f2: BTreeMap<String, String>,
#[serde(flatten)]
f3: BTreeMap<String, String>,
}
let mut se = Serializer::new(Vec::new());
Struct {
f1: 0,
f2: {
let mut map = BTreeMap::new();
map.insert("german".to_string(), "Hallo Welt!".to_string());
map
},
f3: {
let mut map = BTreeMap::new();
map.insert("english".to_string(), "Hello World!".to_string());
map
},
}
.serialize(&mut se).unwrap();
// Expect: { "f1": 0, "f2": { "german": "Hallo Welt!" }, "english": "Hello World!" }.
assert_eq!(
vec![
0x83, 0xA2, 0x66, 0x31, 0x00, 0xA2, 0x66, 0x32, 0x81, 0xA6, 0x67, 0x65, 0x72, 0x6D, 0x61, 0x6E, 0xAB,
0x48, 0x61, 0x6C, 0x6C, 0x6F, 0x20, 0x57, 0x65, 0x6C, 0x74, 0x21, 0xA7, 0x65, 0x6E, 0x67, 0x6C, 0x69,
0x73, 0x68, 0xAC, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21,
],
se.into_inner()
);
}
#[test]
fn serialize_struct_with_flattened_struct_field() {
#[derive(Serialize)]
struct Struct {
f1: u32,
// not flattend!
f2: InnerStruct,
#[serde(flatten)]
f3: InnerStruct,
}
#[derive(Serialize)]
struct InnerStruct {
f4: u32,
f5: u32,
}
let strct = Struct {
f1: 0,
f2: InnerStruct { f4: 8, f5: 13 },
f3: InnerStruct { f4: 21, f5: 34 },
};
// struct-as-tuple
{
let mut se = Serializer::new(Vec::new());
strct.serialize(&mut se).unwrap();
// Expect: { "f1": 0, "f2": [8, 13], "f4": 21, "f5": 34 }.
assert_eq!(vec![
0x84, 0xA2, 0x66, 0x31, 0x00, 0xA2, 0x66, 0x32, 0x92, 0x08, 0x0D, 0xA2, 0x66, 0x34, 0x15, 0xA2, 0x66, 0x35, 0x22,
],
se.into_inner());
}
// struct-as-map
{
let mut se = Serializer::new(Vec::new()).with_struct_map();
strct.serialize(&mut se).unwrap();
// Expect: { "f1": 0, "f2": { "f4": 8, "f5": 13 }, "f4": 21, "f5": 34 }.
assert_eq!(
vec![
0x84, 0xA2, 0x66, 0x31, 0x00, 0xA2, 0x66, 0x32, 0x82, 0xA2, 0x66, 0x34, 0x08,
0xA2, 0x66, 0x35, 0x0D, 0xA2, 0x66, 0x34, 0x15, 0xA2, 0x66, 0x35, 0x22,
],
se.into_inner()
);
}
}
#[test]
fn pass_struct_as_map_using_ext() {
#[derive(Serialize)]
struct Dog<'a> {
name: &'a str,
age: u16,
}
let dog = Dog {
name: "Bobby",
age: 8,
};
let mut se = Serializer::new(Vec::new()).with_struct_map();
dog.serialize(&mut se).unwrap();
// Expect: {"name": "Bobby", "age": 8}.
assert_eq!(vec![0x82, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xa5, 0x42, 0x6f, 0x62, 0x62, 0x79, 0xa3, 0x61, 0x67, 0x65, 0x08],
se.into_inner());
}
#[test]
fn pass_struct_as_tuple_using_double_ext() {
#[derive(Serialize)]
struct Dog<'a> {
name: &'a str,
age: u16,
}
let dog = Dog {
name: "Bobby",
age: 8,
};
let mut se = Serializer::new(Vec::new())
.with_struct_map()
.with_struct_tuple();
dog.serialize(&mut se).unwrap();
assert_eq!(vec![0x92, 0xa5, 0x42, 0x6f, 0x62, 0x62, 0x79, 0x08],
se.into_inner());
}
#[test]
fn pass_struct_as_map_using_triple_ext() {
#[derive(Serialize)]
struct Dog<'a> {
name: &'a str,
age: u16,
}
let dog = Dog {
name: "Bobby",
age: 8,
};
let mut se = Serializer::new(Vec::new())
.with_struct_map()
.with_struct_tuple()
.with_struct_map();
dog.serialize(&mut se).unwrap();
// Expect: {"name": "Bobby", "age": 8}.
assert_eq!(vec![0x82, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xa5, 0x42, 0x6f, 0x62, 0x62, 0x79, 0xa3, 0x61, 0x67, 0x65, 0x08],
se.into_inner());
}
#[test]
fn pass_struct_as_map_using_triple_ext_many_times() {
#[derive(Serialize)]
struct Dog<'a> {
name: &'a str,
age: u16,
}
let dog = Dog {
name: "Bobby",
age: 8,
};
let mut se = Serializer::new(Vec::new())
.with_struct_map()
.with_struct_tuple()
.with_struct_map()
.with_struct_map()
.with_struct_map()
.with_struct_map();
dog.serialize(&mut se).unwrap();
// Expect: {"name": "Bobby", "age": 8}.
assert_eq!(vec![0x82, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xa5, 0x42, 0x6f, 0x62, 0x62, 0x79, 0xa3, 0x61, 0x67, 0x65, 0x08],
se.into_inner());
}

764
third_party/rust/rmp-serde/tests/round.rs vendored Normal file
View File

@@ -0,0 +1,764 @@
use rmp_serde::config::{DefaultConfig, SerializerConfig};
use rmp_serde::decode::ReadReader;
use rmp_serde::{Deserializer, Serializer};
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::io::Cursor;
#[test]
#[ignore]
fn issue_250() {
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(tag = "type", content = "payload")]
pub enum Example {
Unit1,
Unit2,
HasValue { x: u32 },
TupleWithValue(u32, u32),
InnerValue(SomeInnerValue),
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
pub struct SomeInnerValue {
pub a: u32,
pub b: String,
}
let v = rmp_serde::to_vec(&Example::HasValue { x: 3 }).unwrap();
// Fails unwrap with Syntax("invalid type: sequence,
// expected struct variant Example::HasValue")
let ex: Example = rmp_serde::from_slice(&v).unwrap();
assert_eq!(Example::HasValue { x: 3 }, ex);
let v = rmp_serde::to_vec(&Example::Unit1).unwrap();
// Fails unwrap with Syntax("invalid length 1,
// expected adjacently tagged enum Example")
let ex: Example = rmp_serde::from_slice(&v).unwrap();
assert_eq!(Example::Unit1, ex);
}
#[test]
fn round_trip_option() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Foo {
v: Option<Vec<u8>>,
}
let expected = Foo { v: None };
let mut buf = Vec::new();
expected.serialize(&mut Serializer::new(&mut buf)).unwrap();
let mut de = Deserializer::new(Cursor::new(&buf[..]));
assert_eq!(expected, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn round_trip_nested_option() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Struct {
f1: Option<Option<u32>>,
f2: Option<Option<u32>>,
}
let expected = Struct {
f1: Some(Some(13)),
f2: None,
};
let mut buf = Vec::new();
expected.serialize(&mut Serializer::new(&mut buf)).unwrap();
let mut de = Deserializer::new(Cursor::new(&buf[..]));
assert_eq!(expected, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn round_trip_optional_enum() {
#[derive(Serialize, Deserialize, Debug, PartialEq)]
pub enum SimpleEnum {
Variant,
}
let expected = Some(SimpleEnum::Variant);
let mut buf = Vec::new();
expected.serialize(&mut Serializer::new(&mut buf)).unwrap();
let mut de = Deserializer::new(Cursor::new(&buf[..]));
assert_eq!(expected, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn round_trip_cow() {
#[derive(Serialize, Deserialize, Debug, PartialEq)]
struct Foo<'a> {
v: Cow<'a, [u8]>,
}
let expected = Foo {
v: Cow::Borrowed(&[]),
};
let mut buf = Vec::new();
expected.serialize(&mut Serializer::new(&mut buf)).unwrap();
let mut de = Deserializer::new(Cursor::new(&buf[..]));
assert_eq!(expected, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn round_trip_option_cow() {
use serde::Serialize;
use std::borrow::Cow;
use std::io::Cursor;
#[derive(Serialize, Deserialize, Debug, PartialEq)]
struct Foo<'a> {
v: Option<Cow<'a, [u8]>>,
}
let expected = Foo { v: None };
let mut buf = Vec::new();
expected.serialize(&mut Serializer::new(&mut buf)).unwrap();
let mut de = Deserializer::new(Cursor::new(&buf[..]));
assert_eq!(expected, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn round_struct_like_enum() {
use serde::Serialize;
#[derive(Serialize, Deserialize, Debug, PartialEq)]
enum Enum {
A { data: u32 },
}
let expected = Enum::A { data: 42 };
let mut buf = Vec::new();
expected.serialize(&mut Serializer::new(&mut buf)).unwrap();
let mut de = Deserializer::new(&buf[..]);
assert_eq!(expected, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn round_struct_like_enum_with_struct_map() {
use serde::Serialize;
#[derive(Serialize, Deserialize, Debug, PartialEq)]
enum Enum {
A { data: u32 },
}
let expected = Enum::A { data: 42 };
let mut buf = Vec::new();
expected
.serialize(&mut Serializer::new(&mut buf).with_struct_map())
.unwrap();
let mut de = Deserializer::new(&buf[..]);
assert_eq!(expected, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn round_struct_like_enum_with_struct_tuple() {
use serde::Serialize;
#[derive(Serialize, Deserialize, Debug, PartialEq)]
enum Enum {
A { data: u32 },
}
let expected = Enum::A { data: 42 };
let mut buf = Vec::new();
expected
.serialize(&mut Serializer::new(&mut buf).with_struct_tuple())
.unwrap();
let mut de = Deserializer::new(&buf[..]);
assert_eq!(expected, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn round_enum_with_newtype_struct() {
use serde::Serialize;
#[derive(Serialize, Deserialize, Debug, PartialEq)]
struct Newtype(String);
#[derive(Serialize, Deserialize, Debug, PartialEq)]
enum Enum {
A(Newtype),
}
let expected = Enum::A(Newtype("le message".into()));
let mut buf = Vec::new();
expected.serialize(&mut Serializer::new(&mut buf)).unwrap();
let mut de = Deserializer::new(&buf[..]);
assert_eq!(expected, Deserialize::deserialize(&mut de).unwrap());
}
#[test]
fn round_trip_untagged_enum_with_enum_associated_data() {
#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[serde(untagged)]
enum Foo {
A(Bar),
}
#[derive(Serialize, Deserialize, Debug, PartialEq)]
enum Bar {
B,
C(String),
D(u64, u64, u64),
E { f1: String },
}
let data1_1 = Foo::A(Bar::B);
let bytes_1 = rmp_serde::to_vec(&data1_1).unwrap();
let data1_2 = rmp_serde::from_slice(&bytes_1).unwrap();
assert_eq!(data1_1, data1_2);
let data2_1 = Foo::A(Bar::C("Hello".into()));
let bytes_2 = rmp_serde::to_vec(&data2_1).unwrap();
let data2_2 = rmp_serde::from_slice(&bytes_2).unwrap();
assert_eq!(data2_1, data2_2);
let data3_1 = Foo::A(Bar::D(1, 2, 3));
let bytes_3 = rmp_serde::to_vec(&data3_1).unwrap();
let data3_2 = rmp_serde::from_slice(&bytes_3).unwrap();
assert_eq!(data3_1, data3_2);
let data4_1 = Foo::A(Bar::E { f1: "Hello".into() });
let bytes_4 = rmp_serde::to_vec(&data4_1).unwrap();
let data4_2 = rmp_serde::from_slice(&bytes_4).unwrap();
assert_eq!(data4_1, data4_2);
}
// Checks whether deserialization and serialization can both work with structs as maps
#[test]
fn round_struct_as_map() {
use rmp_serde::decode::from_slice;
use rmp_serde::to_vec_named;
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct Dog1 {
name: String,
age: u16,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct Dog2 {
age: u16,
name: String,
}
let dog1 = Dog1 {
name: "Frankie".into(),
age: 42,
};
let serialized: Vec<u8> = to_vec_named(&dog1).unwrap();
let deserialized: Dog2 = from_slice(&serialized).unwrap();
let check = Dog1 {
age: deserialized.age,
name: deserialized.name,
};
assert_eq!(dog1, check);
}
#[test]
fn round_struct_as_map_in_vec() {
// See: issue #205
use rmp_serde::decode::from_slice;
use rmp_serde::to_vec_named;
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct Dog1 {
name: String,
age: u16,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct Dog2 {
age: u16,
name: String,
}
let dog1 = Dog1 {
name: "Frankie".into(),
age: 42,
};
let data = vec![dog1];
let serialized: Vec<u8> = to_vec_named(&data).unwrap();
let deserialized: Vec<Dog2> = from_slice(&serialized).unwrap();
let dog2 = &deserialized[0];
assert_eq!(dog2.name, "Frankie");
assert_eq!(dog2.age, 42);
}
#[test]
fn round_trip_unit_struct() {
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct Message1 {
data: u8,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct Message2;
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
enum Messages {
Message1(Message1),
Message2(Message2),
}
let msg2 = Messages::Message2(Message2);
// struct-as-tuple
{
let serialized: Vec<u8> = rmp_serde::to_vec(&msg2).unwrap();
let deserialized: Messages = rmp_serde::from_slice(&serialized).unwrap();
assert_eq!(deserialized, msg2);
}
// struct-as-map
{
let serialized: Vec<u8> = rmp_serde::to_vec_named(&msg2).unwrap();
let deserialized: Messages = rmp_serde::from_slice(&serialized).unwrap();
assert_eq!(deserialized, msg2);
}
}
#[test]
#[ignore]
fn round_trip_unit_struct_untagged_enum() {
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct UnitStruct;
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct MessageA {
some_int: i32,
unit: UnitStruct,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(untagged)]
enum Messages {
MessageA(MessageA),
}
let msga = Messages::MessageA(MessageA {
some_int: 32,
unit: UnitStruct,
});
// struct-as-tuple
{
let serialized: Vec<u8> = rmp_serde::to_vec(&msga).unwrap();
let deserialized: Messages = rmp_serde::from_slice(&serialized).unwrap();
assert_eq!(deserialized, msga);
}
// struct-as-map
{
let serialized: Vec<u8> = rmp_serde::to_vec_named(&msga).unwrap();
let deserialized: Messages = rmp_serde::from_slice(&serialized).unwrap();
assert_eq!(deserialized, msga);
}
}
#[test]
fn round_trip_struct_with_flattened_map_field() {
use std::collections::BTreeMap;
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct Struct {
f1: u32,
// not flattend!
f2: BTreeMap<String, String>,
#[serde(flatten)]
f3: BTreeMap<String, String>,
}
let strct = Struct {
f1: 0,
f2: {
let mut map = BTreeMap::new();
map.insert("german".to_string(), "Hallo Welt!".to_string());
map
},
f3: {
let mut map = BTreeMap::new();
map.insert("english".to_string(), "Hello World!".to_string());
map
},
};
let serialized: Vec<u8> = rmp_serde::to_vec(&strct).unwrap();
let deserialized: Struct = rmp_serde::from_slice(&serialized).unwrap();
assert_eq!(deserialized, strct);
}
#[test]
fn round_trip_struct_with_flattened_struct_field() {
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct Struct {
f1: u32,
// not flattend!
f2: InnerStruct,
#[serde(flatten)]
f3: InnerStruct,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct InnerStruct {
f4: u32,
f5: u32,
}
let strct = Struct {
f1: 0,
f2: InnerStruct { f4: 8, f5: 13 },
f3: InnerStruct { f4: 21, f5: 34 },
};
// struct-as-tuple
{
let serialized: Vec<u8> = rmp_serde::to_vec(&strct).unwrap();
let deserialized: Struct = rmp_serde::from_slice(&serialized).unwrap();
assert_eq!(deserialized, strct);
}
// struct-as-map
{
let serialized: Vec<u8> = rmp_serde::to_vec_named(&strct).unwrap();
let deserialized: Struct = rmp_serde::from_slice(&serialized).unwrap();
assert_eq!(deserialized, strct);
}
}
// Checks whether deserialization and serialization can both work with enum variants as strings
#[test]
fn round_variant_string() {
use rmp_serde::decode::from_slice;
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
enum Animal1 {
Dog { breed: String },
Cat,
Emu,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
enum Animal2 {
Emu,
Dog { breed: String },
Cat,
}
// use helper macro so that we can test many combinations at once. Needs to be a macro to deal
// with the serializer owning a reference to the Vec.
macro_rules! do_test {
($ser:expr) => {
{
let animal1 = Animal1::Dog { breed: "Pitbull".to_owned() };
let expected = Animal2::Dog { breed: "Pitbull".to_owned() };
let mut buf = Vec::new();
animal1.serialize(&mut $ser(&mut buf)).unwrap();
let deserialized: Animal2 = from_slice(&buf).unwrap();
assert_eq!(deserialized, expected);
}
}
}
do_test!(Serializer::new);
do_test!(|b| Serializer::new(b).with_struct_map());
do_test!(|b| Serializer::new(b).with_struct_tuple());
do_test!(|b| Serializer::new(b).with_struct_map());
do_test!(|b| Serializer::new(b).with_struct_tuple());
do_test!(|b| {
Serializer::new(b)
.with_struct_tuple()
.with_struct_map()
.with_struct_tuple()
.with_struct_map()
});
}
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
#[test]
fn roundtrip_tuples_arrays() {
assert_roundtrips((1i32,100,1000,10000,100000,1000000,10000000));
assert_roundtrips((0u8,1u8,11u8,111u8,255u8));
assert_roundtrips((0u8,1i8,11u16,111i32,255i64));
assert_roundtrips((0i8,1,11,111,-1,-11,-111));
assert_roundtrips((0u128, 1111111u128));
assert_roundtrips([1i32,100,1000,10000,100000,1000000,10000000]);
assert_roundtrips([0u8,1,11,111,255]);
assert_roundtrips([0i8,1,11,111,-1,-11,-111]);
assert_roundtrips([(0u128, 1111111u128)]);
}
#[test]
fn roundtrip_vec() {
assert_roundtrips(vec![1i32,100,1000,10000,100000,1000000,10000000]);
assert_roundtrips(vec![0u8,1,11,111,255]);
assert_roundtrips(vec![0i8,1,11,111,-1,-11,-111]);
assert_roundtrips(vec![(0u8, 1u8)]);
assert_roundtrips(vec![(0u8, 1u32)]);
assert_roundtrips(vec![] as Vec<String>);
assert_roundtrips(vec![] as Vec<u8>);
assert_roundtrips(vec![] as Vec<()>);
assert_roundtrips(vec![] as Vec<Vec<()>>);
assert_roundtrips(vec![vec![1u128,2,3]]);
assert_roundtrips(vec![vec![Some(3u16),None,Some(10000)]]);
}
#[test]
fn roundtrip_hashsets() {
use std::collections::HashSet;
assert_roundtrips([1i32,100,1000,10000,100000,1000000,10000000].into_iter().collect::<HashSet<_>>());
assert_roundtrips([0u8,1,11,111,255].into_iter().collect::<HashSet<_>>());
assert_roundtrips([0i8,1,11,111,-1,-11,-111].into_iter().collect::<HashSet<_>>());
}
#[test]
fn roundtrip_ipv4addr() {
assert_roundtrips(Ipv4Addr::new(127, 0, 0, 1));
}
#[test]
fn roundtrip_ipv6addr() {
assert_roundtrips(Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8));
}
#[test]
fn roundtrip_ipaddr_ipv4addr() {
assert_roundtrips(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));
}
#[test]
fn roundtrip_ipaddr_ipv6addr() {
assert_roundtrips(IpAddr::V6(Ipv6Addr::new(1, 2, 3, 4, 5, 6, 7, 8)));
}
#[test]
fn roundtrip_result_ipv4addr() {
let val: Result<Ipv4Addr, ()> = Ok(Ipv4Addr::new(127, 0, 0, 1));
assert_roundtrips(val);
}
#[test]
fn roundtrip_result_num() {
assert_roundtrips(Ok::<u32, u32>(42));
assert_roundtrips(Err::<(), _>(222));
}
#[test]
fn roundtrip_simple_enum() {
#[derive(PartialEq, Debug, Serialize, Deserialize)]
enum SimpleEnum {
V1(u32),
V2(String),
}
assert_roundtrips(SimpleEnum::V1(42));
assert_roundtrips(SimpleEnum::V2("hello".into()));
}
#[test]
fn roundtrip_some() {
#[derive(PartialEq, Debug, Serialize, Deserialize)]
struct Wrapper<T>(T);
assert_roundtrips(Some(99));
assert_roundtrips(Wrapper(Some(99)));
assert_roundtrips(Some(Wrapper(99)));
assert_roundtrips(Some("hi".to_string()));
}
/// Some types don't fully consume their input SeqAccess, leading to incorrect
/// deserializes.
///
/// https://github.com/3Hren/msgpack-rust/issues/287
#[test]
fn checked_seq_access_len() {
#[derive(Serialize)]
struct Input {
a: [&'static str; 4],
d: &'static str,
}
#[allow(dead_code)]
#[derive(Deserialize, Debug)]
struct Output {
a: [String; 2],
c: String,
}
let mut buffer = Vec::new();
let mut serializer = Serializer::new(&mut buffer)
.with_binary()
.with_struct_map();
// The bug is basically that Output will successfully deserialize from input
// because the [String; 0] deserializer doesn't drain the SeqAccess, and
// the two fields it leaves behind can then be deserialized into `v`
let data = Input {
a: ["b", "b", "c", "c"],
d: "d",
};
data.serialize(&mut serializer)
.expect("failed to serialize");
let mut deserializer = rmp_serde::Deserializer::new(
Cursor::new(&buffer)
).with_binary();
Output::deserialize(&mut deserializer)
.expect_err("Input round tripped into Output; this shouldn't happen");
}
#[test]
fn array_from_bytes() {
let orig = [1u8, 128, 255];
let v = rmp_serde::to_vec(orig.as_slice()).unwrap();
let arr: [u8; 3] = rmp_serde::from_slice(&v).unwrap();
assert_eq!(arr, orig);
let tup: (u8, u8, u8) = rmp_serde::from_slice(&v).unwrap();
assert_eq!(tup, (1, 128, 255));
}
#[test]
fn i128_from_integers() {
let v = rmp_serde::to_vec([0, 1i8, -12i8, 119].as_slice()).unwrap();
let arr: [i128; 4] = rmp_serde::from_slice(&v).unwrap();
assert_eq!(arr, [0, 1i128, -12, 119]);
}
#[ignore]
#[test]
fn roundtrip_some_failures() {
// FIXME
assert_roundtrips(Some(None::<()>));
}
#[cfg(test)]
#[track_caller]
fn assert_roundtrips<T: PartialEq + std::fmt::Debug + Serialize + for<'a> Deserialize<'a>>(val: T) {
use rmp_serde::config::BytesMode;
assert_roundtrips_config(&val, "default", |s| s, |d| d);
assert_roundtrips_config(&val, ".with_struct_map()", |s| s.with_struct_map(), |d| d);
assert_roundtrips_config(
&val,
".with_struct_map()",
|s| s.with_struct_map(),
|d| d,
);
assert_roundtrips_config(
&val,
".with_human_readable()",
|s| s.with_human_readable(),
|d| d.with_human_readable(),
);
assert_roundtrips_config(
&val,
".with_human_readable().with_struct_map()",
|s| s.with_human_readable().with_struct_map(),
|d| d.with_human_readable(),
);
assert_roundtrips_config(
&val,
".with_human_readable()",
|s| s.with_human_readable(),
|d| d.with_human_readable(),
);
assert_roundtrips_config(
&val,
".with_human_readable().with_struct_map()",
|s| {
s.with_human_readable()
.with_struct_map()
},
|d| d.with_human_readable(),
);
assert_roundtrips_config(
&val,
".with_bytes(ForceIterables)",
|s| s.with_bytes(BytesMode::ForceIterables),
|d| d,
);
assert_roundtrips_config(
&val,
".with_bytes(ForceAll)",
|s| s.with_bytes(BytesMode::ForceAll),
|d| d,
);
}
#[cfg(test)]
#[track_caller]
fn assert_roundtrips_config<T, CSF, SC, CDF, DC>(
val: &T,
desc: &str,
config_serializer: CSF,
config_deserializer: CDF,
) where
T: PartialEq + std::fmt::Debug + Serialize + for<'a> Deserialize<'a>,
CSF: FnOnce(Serializer<Vec<u8>, DefaultConfig>) -> Serializer<Vec<u8>, SC>,
SC: SerializerConfig,
CDF: FnOnce(
Deserializer<ReadReader<&[u8]>, DefaultConfig>,
) -> Deserializer<ReadReader<&[u8]>, DC>,
DC: SerializerConfig,
{
let mut serializer = config_serializer(Serializer::new(Vec::new()));
if let Err(e) = val.serialize(&mut serializer) {
panic!(
"Failed to serialize: {}\nConfig: {}\nValue: {:?}\n",
e, desc, val
);
}
let serialized = serializer.into_inner();
let mut deserializer = config_deserializer(Deserializer::new(serialized.as_slice()));
let val2: T = match T::deserialize(&mut deserializer) {
Ok(t) => t,
Err(e) => {
panic!(
"Does not deserialize: {}\nConfig: {}\nSerialized {:?}\nGot {:?}\n",
e,
desc,
val,
rmpv::decode::value::read_value(&mut serialized.as_slice())
.expect("rmp didn't serialize correctly at all")
);
}
};
assert_eq!(val, &val2, "Config: {}", desc);
}

1
third_party/rust/rmp/.cargo-checksum.json vendored Normal file
View File

@@ -0,0 +1 @@
{"files":{"CHANGELOG.md":"7fa7c73b5b677038d450f99773c4f530e8ae5c9b3b4b1bc9797ebe16ad7291f6","Cargo.lock":"86a0f19aa5dd247c3e738a31401ccaa8922cc96c51020b822fcb374fe238c8b9","Cargo.toml":"4455cd4bb9e7d93700662b5efb42f5ccbe1bed8cc3e20592a857b6258e7d6688","LICENSE":"979d35e1d157289100ba2a7eb13fffe4e7a1b93506c755c4c66890918e467c4e","README.md":"5036e0b8f30a2205fb6a7255391359d8d20f47225523144403a9fd23d0bf5992","benches/bench.rs":"21123f22a964555dab88b67d0f31cbcf0219985774d1155934099a55cbbdc200","clippy.toml":"fb31cdc22361b09ec4820bda557b952ce4c14f0c8066fe4e48f9ecb7a844101b","examples/inspect.rs":"f94c487fc30d0ab65fc4a918dbebed33f9143e4cc5180b667c3e0502c8926ff9","src/decode/bytes.rs":"b00f8065c02524accb92920545aadaa90c8daf5f1f33684e493c549438d3b046","src/decode/dec.rs":"6f8e05f386e384734fbd8647ea40cd44734d8fa50042481a26b584792cd0b056","src/decode/ext.rs":"00bbf3daf7fd4b9f4cbecaa057052cbf4dd60cd1f70fce9bfc0f55d9c5a9685f","src/decode/mod.rs":"ed467bfbb61e2dc8f1c31911e02213cca5b9b931ec84c509f336f0fa4a8d074c","src/decode/sint.rs":"6be9951d358b3d20084212cabb6d1952fc3b26a05d5021523f83bb268918072d","src/decode/str.rs":"40d1815b9d78324665b9cb9e9ccd739632b6c6a31cc24beb7178ac036c584511","src/decode/uint.rs":"0d28c59367c02d610c79baabb5791b4c07bc3b4931b98d751ed837bba8893fb5","src/encode/bin.rs":"28dc1da41a29d71b94e40a4ebf502d3112d4c7660bfcc9390b8a6c47157a5dd8","src/encode/buffer.rs":"2e5d953ea4f189eaaa8dfc7e7f99c4756c1a678d5fc757b2b4ef89e077a4ee41","src/encode/dec.rs":"05b6cc0ddbb2bf061408d2b34666f3c341132a2e00b8f8c6a1b2f0bdd483e355","src/encode/ext.rs":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","src/encode/map.rs":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","src/encode/mod.rs":"454d609a06db09e58241d89bbbb8fd5a254ca7e7362ac5e44ae45b5181efa0e8","src/encode/sint.rs":"39615e55027fe5c340ffacd1b4074dfa78fcd49768e32aef2fe6e5d4930486b4","src/encode/str.rs":"32a14d0c2556ae3bd610e9d8daa5aa808582cfaa5b5685abe11e4aea8be7b2cd","src/encode/uint.rs":"9a3e54197b1e8fa3a21a5581e50e406b353d2a01938da4d9d01f083a1e968c9a","src/encode/vec.rs":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","src/errors.rs":"9c0923a47007564efbbb0d4a85ef4bb85ffa1db993f7d05127fa37f5e4148697","src/lib.rs":"691e5a0d3db1e77b4ea1fcfb5a35f57f7454f383d88a920b5c3cf07445a90f62","src/marker.rs":"33b8c9809f0ccee414bb1938a4af6a945ff1fa8a799d053c1769ddaf89389320","tests/func/decode/array.rs":"0c137d58f6457fd2d286673e97f7dcf4492c0c243954a7f810d0e907595e2381","tests/func/decode/bin.rs":"e6b6ca2342199ab945b8417bf771a5240d7d55e3dab7e3ba96eddf51fc4d517f","tests/func/decode/bool.rs":"4cfaecf5cbedbb740bd70e2f82a79b66033cc363999849ac525b5ffd2926ad0a","tests/func/decode/ext.rs":"ef0f9a13eeb67fea84bd6e6e08d45381890cf657406ac77d9a12071e8654e750","tests/func/decode/float.rs":"ad6cecc8a3e33136f61a5a9133e9356638177135b7a77a09a29086d330825c4b","tests/func/decode/map.rs":"c596ecd263d1cb63b189c49883c9fcc81bf71e600c9e3120bf907bba8ead48f0","tests/func/decode/mod.rs":"4abfcc0991c087185790316811edbbbdc51d9d3bc53b629cc26e825d69ec1962","tests/func/decode/null.rs":"822512da3adea38ac185686e25a2738a61735552754ba0f0ba929a48b7a62958","tests/func/decode/sint.rs":"f11c64336a920598df5509e237dc798a05eb6ae449a84e9df1dd41096d43193d","tests/func/decode/string.rs":"6b1c5a11b64d27c6f84afa89f6d8fa82f5ebdca4cc4330c48b6eacd524401edd","tests/func/decode/uint.rs":"150a7c618bb028e8f63fc7a2b1bbff9346a6189f3f10f6ff5f7b64a2047d098f","tests/func/encode/array.rs":"a88ab75a0bc7984fcf02a4b9b90c5589d673970194252080f6d7ee93624ae8af","tests/func/encode/bin.rs":"27c8e82f7bab7fc87141629da70d6cf1f0845662676493b2e2ba35aead1f72c7","tests/func/encode/bool.rs":"5f3a9bd1c3de6289cf6cd55255a8484b3a0c75f7d0c1818da02bba374160710c","tests/func/encode/ext.rs":"2e68f72812645d5da6e679d4bb93576f78901da991553af2654ff4da04d82eba","tests/func/encode/float.rs":"a45a1c54d093353b7d0775ea8e3391a7eb6c5cb7371e0140aef53e932bc1a40a","tests/func/encode/int.rs":"010320f424fe8f437af395018332be1de6404f135429243f2fbdf83bc236fc5a","tests/func/encode/map.rs":"0f332db29b3074a4a1002f9a5975c838268a6736449bf39ec8732227da6ab706","tests/func/encode/mod.rs":"e896f72e17c379b05a298afe5082d636c7f1edce04e1e8d70ace8cd2641c179a","tests/func/encode/null.rs":"ff4ebff2bbf47426358ac806305b7625f43cdb342ef77d19aed398530b5ab637","tests/func/encode/string.rs":"a73cb26cb778c41556f0f55babf36874362c8fde599399fa94c4981c3050d4cb","tests/func/mirror.rs":"6725857df96e2e8db2cb8e2f5c599d121850b9e55a6430d9f494f627e6a7812e","tests/lib.rs":"d1c881751f5ecb0f56f635b88da057073054a2cee0049c7f2d2badda7c74a287"},"package":"228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4"}

177
third_party/rust/rmp/CHANGELOG.md vendored Normal file
View File

@@ -0,0 +1,177 @@
# Change Log
All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](http://semver.org/).
## Unreleased (0.8.11)
### Added
- Implemeneted support for `#![no_std]` in `rmpv`
- Adds new `feature="std"` (on by default)
- Introduces new `RmpRead` and `RmpWrite` traits.
- Needed because `std::io::Read` (and Write) are missing on `#![no_std]`
- Introduces new `Bytes` and `ByteBuf` wrappers, that implement RmpRead/RmpWrite for no\_std targets.
## 0.8.6 - 2017-04-23
### Added
- New `rmp::decode::read_str_from_slice` function for zero-copy reading strings from slices.
### Changed
- Deprecate `rmp::decode::read_str_ref`, because it's useless and inconvenient.
## 0.8.5 - 2017-03-13
### Fixed
- Fix compilation on rustc 1.13.0.
## 0.8.4 - 2017-03-09
### Added
- Derive `Debug` for `MarkerReadError`.
## 0.8.3 - 2017-03-04
### Fixed
- Fixed `write_sint`, so it should mark positive values as unsigned integers.
Before this change it marked signed integers larger than `4294967296` with I64 marker, which is not consistent with other MessagePack implementations.
Now it should mark such integers with U64 marker.
## 0.8.2 - 2017-02-01
### Added
- Conversion from `ValueWriteError` into I/O error.
## 0.8.1 - 2017-01-05
### Changed
- Fixed docs link.
## 0.8.0 - 2017-01-05
### Added
- Marker now implements `From` and `Into` traits for `u8`.
- Add `read_int` function, which allows to read integer values and automatically cast to the expected result type even if they aren't the same. An additional `OutOfRange` error will be returned in the case of failed numeric cast.
- Add `NumValueReadError` enum with additional `OutOfRange` variant to be able to detect whether integer decoding failed because of out of range.
### Changed
- Update `byteorder` dependency to 1.0.
- Unexpected EOF variant has been merged with the default one in the I/O Error enum.
- Function `write_sint` now encodes 64-bit signed integers using the most compact representation.
- Function `write_uint` now encodes 64-bit unsigned integers using the most compact representation.
- Rename `read_array_size` function to `read_array_len` for consistency.
- Rename `read_map_size` function to `read_map_len` for consistency.
- Make `FixedValueWriteError` struct private. All functions, that previously returned such kind of error now return the Standard I/O error.
### Removed
- Move `Value` and `ValueRef` enums and associated functions into the separate `rmpv` crate.
- Remove conversions from `byteorder` crate errors, because since 0.5 there are no such errors.
- Remove `write_sint_eff` function - its functionality can now be done using `write_sint` instead.
- Remove `write_uint_eff` function - its functionality can now be done using `write_uint` instead.
- Integral functions like `read_*_loosely` and `read_*_fit` were removed in favor of generic `read_int` function, which allows to read integral values and cast them to the specified result type even if they aren't the same.
- Remove `read_bin_borrow` function.
## 0.7.5 - 2016-07-24
### Added
- Add `is_*` methods for Value for checking its underlying value without explicit matching.
- Add `as_*` methods for Value for borrowing its underlying value.
- Value is now indexable by integer.
## 0.7.4 - 2016-07-18
### Added
- Value now can be initialized from primitive values using From trait.
## 0.7.3 - 2015-09-23
### Changed
- Restricted upper version limit for dependencies.
## 0.7.2 - 2015-09-23
### Added
- Implemented `Display` trait for `Value`.
## 0.7.1 - 2015-09-11
### Changed
- Use `to_owned` instead of `to_string` while converting `ValueRef` into `Value`.
This change improves `ValueRef::to_owned()` method performance by approximately 10-20%.s Also after this change it's cheaper to decode directly into `ValueRef` with further converting to owned value rather than decoding directly into `Value`.
## 0.7.0 - 2015-08-24
### Changed
- The big single crate has been refactored, which results in three crates: `rmp`, `rmp-serialize` and `rmp-serde`.
## 0.6.0 - 2015-08-17
### Added
- Initial support for [Serde](https://github.com/serde-rs/serde) serializer and deserializer.
- Efficient bytes serialization with Serde.
- Efficient binaries deserialization with Serde using `ByteBuf`.
- Rust serialize Decoder now can provide the underlying reader both by reference or by value, destroying itself in the last case.
### Changed
- Update well-formness for `BigEndianRead` trait to be implemented only for sized types.
- Renamed `PositiveFixnum` marker to `FixPos`.
- Renamed `NegativeFixnum` marker to `FixNeg`.
- Renamed `FixedString` marker to `FixStr`.
- Renamed `FixedArray` marker to `FixArray`.
- Renamed `FixedMap` to `FixMap`.
- Minor documentation updates and markdown fixes.
## 0.5.1 - 2015-08-10
### Changed
- Now the `rustc_serialize::Encoder` should encode signed integers using the most effective underlying representation.
- Now the `rustc_serialize::Decoder` should properly map integers to the result type if the decoded value fits in
result type's range.
## 0.5.0 - 2015-08-01
### Added
- New `ValueRef` value struct represents MessagePack'ed value, but unlike an owning `Value` it owns nothing except its
structure. It means that all strings and binaries it contains are borrowed from the byte array from which the value
was created.
- New `BorrowRead` trait, which looks like a standard `BufRead` but unlike the standard this has an explicit internal
buffer lifetime, which allows to borrow from underlying buffer while mutating the type.
- Encoding function for `ValueRef` with its own error category.
- Decoding function for `ValueRef` with its own error category.
- Conversion method from `ValueRef` to `Value`.
- More benchmarks and tests.
### Changed
- Derive `Copy` trait for `Integer` and `Float` enums.
## 0.4.0 - 2015-07-17
### Added
- Low level `write_str` function allows to serialize the UTF-8 encoded strings the most efficient way.
- Low level `write_bin` function allows to serialize the binary array the most efficient way.
- Implemented `std::error::Error` trait for error types.
## 0.3.2 - 2015-07-05
### Changed
- Encoder now should return proper error types.
## 0.3.1 - 2015-06-28
### Changed
- Stabilizing enum serialization/deserialization. Now every enum is serialized as [int, [args...]].
- Suppressed some warnings appeared on updated compiler.
## 0.3.0 - 2015-06-25
### Added
- Enum serialization/deserialization.
## 0.2.2 - 2015-06-15
### Changed
- Minor integer decoding performance tweaking.
## 0.2.1 - 2015-05-30
### Added
- Benchmarking module.
### Changed
- Increased string decoding performance by ~30 times.
- Exported `read_value` function to the `rmp::decode` module.
- Exported `Value` struct to the root crate namespace.
## 0.2.0 - 2015-05-27
### Added
- Introducing a `Value` algebraic data type, which represents an owning MessagePack object. It can
be found in `rmp::value` module.
- The Value ADT encoding and decoding functions.
- Low-level ext type decoders.
## 0.1.1 - 2015-05-18
### Changed
- Added documentation and repository site in Cargo.toml.
- Added keywords to ease searching using crates.io.
## 0.1.0 - 2015-05-15
### Added
- Initial commit.
- This CHANGELOG file to hopefully serve as an evolving example of a standardized open source
project CHANGELOG.

158
third_party/rust/rmp/Cargo.lock generated vendored Normal file
View File

@@ -0,0 +1,158 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "aho-corasick"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
dependencies = [
"memchr",
]
[[package]]
name = "autocfg"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "env_logger"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3"
dependencies = [
"log",
"regex",
]
[[package]]
name = "getrandom"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "libc"
version = "0.2.153"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
[[package]]
name = "log"
version = "0.4.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
[[package]]
name = "memchr"
version = "2.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
[[package]]
name = "num-traits"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
dependencies = [
"autocfg",
]
[[package]]
name = "paste"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
[[package]]
name = "quickcheck"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6"
dependencies = [
"env_logger",
"log",
"rand",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "regex"
version = "1.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
[[package]]
name = "rmp"
version = "0.8.14"
dependencies = [
"byteorder",
"num-traits",
"paste",
"quickcheck",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"

53
third_party/rust/rmp/Cargo.toml vendored Normal file
View File

@@ -0,0 +1,53 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2021"
name = "rmp"
version = "0.8.14"
authors = ["Evgeny Safronov <division494@gmail.com>"]
description = "Pure Rust MessagePack serialization implementation"
documentation = "https://docs.rs/rmp"
readme = "README.md"
keywords = [
"msgpack",
"MessagePack",
]
categories = ["encoding"]
license = "MIT"
repository = "https://github.com/3Hren/msgpack-rust"
[package.metadata.release]
tag-prefix = "{{crate_name}}/"
[dependencies.byteorder]
version = "1.4.2"
default-features = false
[dependencies.num-traits]
version = "0.2.14"
default-features = false
[dependencies.paste]
version = "1.0"
[dev-dependencies.quickcheck]
version = "1.0.2"
[features]
default = ["std"]
std = [
"byteorder/std",
"num-traits/std",
]
[badges.maintenance]
status = "looking-for-maintainer"

21
third_party/rust/rmp/LICENSE vendored Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2017 Evgeny Safronov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

142
third_party/rust/rmp/README.md vendored Normal file
View File

@@ -0,0 +1,142 @@
## The Rust MessagePack Library
RMP is a pure Rust [MessagePack](http://msgpack.org) implementation of an efficient binary
serialization format. This crate provides low-level core functionality, writers and readers for
primitive values with direct mapping between binary MessagePack format.
[Looking for Serde support](https://lib.rs/crates/rmp-serde)?
This crate represents the very basic functionality needed to work with MessagePack format.
Ideologically it is developed as a basis for building high-level abstractions.
### Usage
To use `rmp`, first add this to your `Cargo.toml`:
```toml
[dependencies.rmp]
rmp = "0.8"
```
### Features
- **Low-level API**
RMP is designed to be lightweight and straightforward. There are low-level APIs, which give you
full control over the encoding/decoding process. `no-std` environments are supported.
- **Zero-copy value decoding**
RMP allows to decode bytes from a buffer in a zero-copy manner, without any heap allocations.
easily and blazingly fast. Rust static checks guarantee that the data will be valid until buffer lives.
- **Clear error handling**
RMP's error system guarantees that you never receive an error enum with unreachable variant.
- **Robust and tested**
This project is developed using TDD and CI, so any found bugs will be fixed without breaking
existing functionality.
### Detailed
Currently there are two large modules: encode and decode. More detail you can find in the
corresponding sections.
Formally every MessagePack message consists of some marker encapsulating a data type and the
data itself. Sometimes there are no separate data chunk, for example for booleans. In these
cases a marker contains the value. For example, the `true` value is encoded as `0xc3`.
```rust
let mut buf = Vec::new();
rmp::encode::write_bool(&mut buf, true).unwrap();
assert_eq!([0xc3], buf[..]);
```
Sometimes a single value can be encoded in multiple ways. For example a value of `42` can be
represented as: `[0x2a], [0xcc, 0x2a], [0xcd, 0x00, 0x2a]` and so on, and all of them are
considered as valid representations. To allow fine-grained control over encoding such values
the library provides direct mapping functions.
```rust
let mut bufs = vec![vec![]; 5];
rmp::encode::write_pfix(&mut bufs[0], 42).unwrap();
rmp::encode::write_u8(&mut bufs[1], 42).unwrap();
rmp::encode::write_u16(&mut bufs[2], 42).unwrap();
rmp::encode::write_u32(&mut bufs[3], 42).unwrap();
rmp::encode::write_u64(&mut bufs[4], 42).unwrap();
assert_eq!([0x2a], bufs[0][..]);
assert_eq!([0xcc, 0x2a], bufs[1][..]);
assert_eq!([0xcd, 0x00, 0x2a], bufs[2][..]);
assert_eq!([0xce, 0x00, 0x00, 0x00, 0x2a], bufs[3][..]);
assert_eq!([0xcf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a], bufs[4][..]);
```
But they aren't planned to be widely used. Instead we often need to encode bytes compactly to
save space. In these cases RMP provides functions that guarantee that for encoding the most
compact representation will be chosen.
```rust
let mut buf = Vec::new();
rmp::encode::write_sint(&mut buf, 300).unwrap();
assert_eq!([0xcd, 0x1, 0x2c], buf[..]);
```
On the other hand for deserialization it is not matter in which representation the value is
encoded - RMP deals with all of them.
Sometimes you know the exact type representation and want to enforce the deserialization process
to make it strongly type safe.
```rust
let buf = [0xcd, 0x1, 0x2c];
assert_eq!(300, rmp::decode::read_u16(&mut &buf[..]).unwrap());
```
However if you try to decode such bytearray as other integer type, for example `u32`, there will
be type mismatch error.
```rust
let buf = [0xcd, 0x1, 0x2c];
rmp::decode::read_u32(&mut &buf[..]).err().unwrap();
```
But sometimes all you want is just to encode an integer that *must* fit in the specified type
no matter how it was encoded. RMP provides [`such`][read_int] function to ease integration with
other MessagePack libraries.
```rust
let buf = [0xcd, 0x1, 0x2c];
assert_eq!(300i16, rmp::decode::read_int(&mut &buf[..]).unwrap());
assert_eq!(300i32, rmp::decode::read_int(&mut &buf[..]).unwrap());
assert_eq!(300i64, rmp::decode::read_int(&mut &buf[..]).unwrap());
assert_eq!(300u16, rmp::decode::read_int(&mut &buf[..]).unwrap());
assert_eq!(300u32, rmp::decode::read_int(&mut &buf[..]).unwrap());
assert_eq!(300u64, rmp::decode::read_int(&mut &buf[..]).unwrap());
```
### API
Almost all API are represented as pure functions, which accepts a generic `Write` or `Read` and
the value to be encoded/decoded. For example let's do a round trip for π number.
```rust
let pi = std::f64::consts::PI;
let mut buf = Vec::new();
rmp::encode::write_f64(&mut buf, pi).unwrap();
assert_eq!([0xcb, 0x40, 0x9, 0x21, 0xfb, 0x54, 0x44, 0x2d, 0x18], buf[..]);
assert_eq!(pi, rmp::decode::read_f64(&mut &buf[..]).unwrap());
```
License: MIT
[read_int]: https://docs.rs/rmp/latest/rmp/decode/fn.read_int.html

44
third_party/rust/rmp/benches/bench.rs vendored Normal file
View File

@@ -0,0 +1,44 @@
#![feature(test)]
extern crate test;
use test::Bencher;
use rmp::decode::*;
#[bench]
fn from_i64_read_i64(b: &mut Bencher) {
let buf = [0xd3, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];
b.iter(|| {
let res = read_i64(&mut &buf[..]).unwrap();
test::black_box(res);
});
}
#[bench]
fn from_i64_read_int(b: &mut Bencher) {
let buf = [0xd3, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];
b.iter(|| {
let res: i64 = read_int(&mut &buf[..]).unwrap();
test::black_box(res);
});
}
#[bench]
fn from_string_read_str(b: &mut Bencher) {
// Lorem ipsum dolor sit amet.
let buf = [
0xbb, 0x4c, 0x6f, 0x72, 0x65, 0x6d, 0x20, 0x69, 0x70, 0x73,
0x75, 0x6d, 0x20, 0x64, 0x6f, 0x6c, 0x6f, 0x72, 0x20, 0x73,
0x69, 0x74, 0x20, 0x61, 0x6d, 0x65, 0x74, 0x2e
];
let mut out = [0u8; 32];
b.iter(|| {
let res = read_str(&mut &buf[..], &mut out[..]).unwrap();
test::black_box(res);
});
}

1
third_party/rust/rmp/clippy.toml vendored Normal file
View File

@@ -0,0 +1 @@
doc-valid-idents = ["MessagePack"]

135
third_party/rust/rmp/examples/inspect.rs vendored Normal file
View File

@@ -0,0 +1,135 @@
use std::fmt;
use std::io::{self, Read};
use rmp::{decode::*, Marker};
fn main() {
let path = std::env::args_os().nth(1).expect("Specify path to a file with msgpack content");
let data = std::fs::read(&path).expect(&path.to_string_lossy());
dump(&mut Indent { i: 0, start: true }, &mut data.as_slice()).unwrap();
}
fn dump(indent: &mut Indent, rd: &mut &[u8]) -> Result<(), Box<dyn std::error::Error>> {
match read_marker(rd).map_err(ValueReadError::from)? {
Marker::FixPos(n) => print!("U0({n})"),
Marker::FixNeg(n) => print!("I0({n})"),
Marker::Null => print!("Null"),
Marker::True => print!("True"),
Marker::False => print!("False"),
Marker::U8 => print!("U8({})", rd.read_data_u8()?),
Marker::U16 => print!("U16({})", rd.read_data_u16()?),
Marker::U32 => print!("U32({})", rd.read_data_u32()?),
Marker::U64 => print!("U64({})", rd.read_data_u64()?),
Marker::I8 => print!("I8({})", rd.read_data_i8()?),
Marker::I16 => print!("I16({})", rd.read_data_i16()?),
Marker::I32 => print!("I32({})", rd.read_data_i32()?),
Marker::I64 => print!("I64({})", rd.read_data_i64()?),
Marker::F32 => print!("F32({})", rd.read_data_f32()?),
Marker::F64 => print!("F64({})", rd.read_data_f64()?),
Marker::FixStr(len) => print!("Str0(\"{}\")", read_str_data(len.into(), rd)?),
Marker::Str8 => print!("Str8(\"{}\")", read_str_data(rd.read_data_u8()?.into(), rd)?),
Marker::Str16 => print!("Str16(\"{}\")", read_str_data(rd.read_data_u16()?.into(), rd)?),
Marker::Str32 => print!("Str32(\"{}\")", read_str_data(rd.read_data_u32()?.into(), rd)?),
Marker::Bin8 => print!("Bin8({})", HexDump(&read_bin_data(rd.read_data_u8()?.into(), rd)?)),
Marker::Bin16 => print!("Bin16({})", HexDump(&read_bin_data(rd.read_data_u16()?.into(), rd)?)),
Marker::Bin32 => print!("Bin32({})", HexDump(&read_bin_data(rd.read_data_u32()?.into(), rd)?)),
Marker::FixArray(len) => dump_array(indent, 0, len.into(), rd)?,
Marker::Array16 => dump_array(indent, 16, rd.read_data_u16()?.into(), rd)?,
Marker::Array32 => dump_array(indent, 32, rd.read_data_u32()?.into(), rd)?,
Marker::FixMap(len) => dump_map(indent, 0, len.into(), rd)?,
Marker::Map16 => dump_map(indent, 16, rd.read_data_u16()?.into(), rd)?,
Marker::Map32 => dump_map(indent, 32, rd.read_data_u32()?.into(), rd)?,
Marker::FixExt1 => todo!(),
Marker::FixExt2 => todo!(),
Marker::FixExt4 => todo!(),
Marker::FixExt8 => todo!(),
Marker::FixExt16 => todo!(),
Marker::Ext8 => todo!(),
Marker::Ext16 => todo!(),
Marker::Ext32 => todo!(),
Marker::Reserved => todo!(),
}
Ok(())
}
fn dump_map(indent: &mut Indent, ty: u8, len: u32, rd: &mut &[u8]) -> Result<(), Box<dyn std::error::Error>> {
indent.print(format_args!("Map{ty}{{"));
let multiline = len > 1;
if multiline { indent.ln(); } else { print!(" ") }
indent.ind();
for i in 0..len {
indent.print(""); dump(indent, rd)?; print!(": "); dump(indent, rd)?;
if multiline { print!(","); indent.ln(); } else if i+1 != len { print!(", ") }
}
indent.out();
indent.print(format_args!("}}"));
Ok(())
}
fn dump_array(indent: &mut Indent, ty: u8, len: u32, rd: &mut &[u8]) -> Result<(), Box<dyn std::error::Error>> {
indent.print(format_args!("Array{ty}["));
let multiline = len > 1;
if multiline { indent.ln(); } else { print!(" ") }
indent.ind();
for i in 0..len {
indent.print(""); dump(indent, rd)?;
if multiline { print!(","); indent.ln(); } else if i+1 != len { print!(", ") }
}
indent.out();
indent.print("]");
Ok(())
}
fn read_str_data<R: Read>(len: u32, rd: &mut R) -> Result<String, io::Error> {
Ok(String::from_utf8_lossy(&read_bin_data(len, rd)?).into_owned())
}
fn read_bin_data<R: Read>(len: u32, rd: &mut R) -> Result<Vec<u8>, io::Error> {
let mut buf = Vec::with_capacity(len.min(1<<16) as usize);
let bytes_read = rd.take(len as u64).read_to_end(&mut buf)?;
if bytes_read != len as usize {
return Err(io::ErrorKind::UnexpectedEof.into());
}
Ok(buf)
}
struct Indent { i: u16, start: bool }
impl Indent {
fn print(&mut self, args: impl fmt::Display) {
print!("{:w$}{args}", "", w = if self.start { (self.i as usize) * 2 } else { 0 });
self.start = false;
}
pub fn ind(&mut self) {
self.i += 1;
}
pub fn ln(&mut self) {
println!();
self.start = true;
}
pub fn out(&mut self) {
self.i -= 1;
}
}
struct HexDump<'a>(&'a [u8]);
impl fmt::Display for HexDump<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let truncate = self.0.len() > 50;
if truncate {
f.write_fmt(format_args!("{}B ", self.0.len()))?;
}
for &b in &self.0[0.. (if truncate { 50 } else { self.0.len() })] {
f.write_fmt(format_args!("{b:02x}"))?;
}
if truncate {
f.write_str("…")?;
}
Ok(())
}
}

156
third_party/rust/rmp/src/decode/bytes.rs vendored Normal file
View File

@@ -0,0 +1,156 @@
//! Implementation of the [Bytes] type
use super::RmpRead;
use crate::decode::RmpReadErr;
use core::fmt::{Display, Formatter};
/// Indicates that an error occurred reading from [Bytes]
#[derive(Debug)]
#[non_exhaustive]
// NOTE: We can't use thiserror because of no_std :(
pub enum BytesReadError {
/// Indicates that there were not enough bytes.
InsufficientBytes {
expected: usize,
actual: usize,
position: u64,
},
}
impl Display for BytesReadError {
fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result {
match *self {
BytesReadError::InsufficientBytes { expected, actual, position } => {
write!(f, "Expected at least bytes {expected}, but only got {actual} (pos {position})")
}
}
}
}
#[cfg(feature = "std")]
impl std::error::Error for BytesReadError {}
impl RmpReadErr for BytesReadError {}
/// A wrapper around `&[u8]` to read more efficiently.
///
/// This has a specialized implementation of `RmpWrite`
/// and has error type [Infallible](core::convert::Infallible).
///
/// This has the additional benefit of working on `#[no_std]` (unlike the builtin Read trait)
///
/// See also [serde_bytes::Bytes](https://docs.rs/serde_bytes/0.11/serde_bytes/struct.Bytes.html)
///
/// Unlike a plain `&[u8]` this also tracks an internal offset in the input (See [`Self::position`]).
///
/// This is used for (limited) compatibility with [`std::io::Cursor`]. Unlike a [Cursor](std::io::Cursor) it does
/// not support mark/reset.
#[derive(Debug, Copy, Clone, Default, Eq, PartialEq, Hash, Ord, PartialOrd)]
pub struct Bytes<'a> {
/// The internal position of the input buffer.
///
/// This is not required for correctness.
/// It is only used for error reporting (and to implement [`Self::position`])
current_position: u64,
bytes: &'a [u8],
}
impl<'a> Bytes<'a> {
/// Wrap an existing bytes slice.
///
/// This sets the internal position to zero.
#[inline]
#[must_use]
pub fn new(bytes: &'a [u8]) -> Self {
Bytes { bytes, current_position: 0 }
}
/// Get a reference to the remaining bytes in the buffer.
#[inline]
#[must_use]
pub fn remaining_slice(&self) -> &'a [u8] {
self.bytes
}
/// Return the position of the input buffer.
///
/// This is not required for correctness, it only exists to help mimic
/// [`Cursor::position`](std::io::Cursor::position)
#[inline]
#[must_use]
pub fn position(&self) -> u64 {
self.current_position
}
}
impl<'a> From<&'a [u8]> for Bytes<'a> {
#[inline]
fn from(bytes: &'a [u8]) -> Self {
Bytes { bytes, current_position: 0 }
}
}
impl RmpRead for Bytes<'_> {
type Error = BytesReadError;
#[inline]
fn read_u8(&mut self) -> Result<u8, Self::Error> {
if let Some((&first, newly_remaining)) = self.bytes.split_first() {
self.bytes = newly_remaining;
self.current_position += 1;
Ok(first)
} else {
Err(BytesReadError::InsufficientBytes {
expected: 1,
actual: 0,
position: self.current_position,
})
}
}
#[inline]
fn read_exact_buf(&mut self, buf: &mut [u8]) -> Result<(), Self::Error> {
let to_read = buf.len();
if to_read <= self.bytes.len() {
let (src, newly_remaining) = self.bytes.split_at(to_read);
self.bytes = newly_remaining;
self.current_position += to_read as u64;
buf.copy_from_slice(src);
Ok(())
} else {
Err(BytesReadError::InsufficientBytes {
expected: to_read,
actual: self.bytes.len(),
position: self.current_position,
})
}
}
}
#[cfg(not(feature = "std"))]
impl<'a> RmpRead for &'a [u8] {
type Error = BytesReadError;
fn read_u8(&mut self) -> Result<u8, Self::Error> {
if let Some((&first, newly_remaining)) = self.split_first() {
*self = newly_remaining;
Ok(first)
} else {
Err(BytesReadError::InsufficientBytes {
expected: 1,
actual: 0,
position: 0,
})
}
}
fn read_exact_buf(&mut self, buf: &mut [u8]) -> Result<(), Self::Error> {
let to_read = buf.len();
if to_read <= self.len() {
let (src, newly_remaining) = self.split_at(to_read);
*self = newly_remaining;
buf.copy_from_slice(src);
Ok(())
} else {
Err(BytesReadError::InsufficientBytes {
expected: to_read,
actual: self.len(),
position: 0,
})
}
}
}

48
third_party/rust/rmp/src/decode/dec.rs vendored Normal file
View File

@@ -0,0 +1,48 @@
use super::{read_marker, RmpRead, ValueReadError};
use crate::Marker;
/// Attempts to read exactly 5 bytes from the given reader and to decode them as `f32` value.
///
/// The first byte should be the marker and the others should represent the data itself.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_f32<R: RmpRead>(rd: &mut R) -> Result<f32, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::F32 => Ok(rd.read_data_f32()?),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 9 bytes from the given reader and to decode them as `f64` value.
///
/// The first byte should be the marker and the others should represent the data itself.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_f64<R: RmpRead>(rd: &mut R) -> Result<f64, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::F64 => Ok(rd.read_data_f64()?),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}

168
third_party/rust/rmp/src/decode/ext.rs vendored Normal file
View File

@@ -0,0 +1,168 @@
use super::{read_marker, RmpRead, ValueReadError};
use crate::Marker;
/// Attempts to read exactly 3 bytes from the given reader and interpret them as a fixext1 type
/// with data attached.
///
/// According to the MessagePack specification, a fixext1 stores an integer and a byte array whose
/// length is 1 byte. Its marker byte is `0xd4`.
///
/// Note, that this function copies a byte array from the reader to the output `u8` variable.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_fixext1<R: RmpRead>(rd: &mut R) -> Result<(i8, u8), ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::FixExt1 => {
let ty = rd.read_data_i8()?;
let data = rd.read_data_u8()?;
Ok((ty, data))
}
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 4 bytes from the given reader and interpret them as a fixext2 type
/// with data attached.
///
/// According to the MessagePack specification, a fixext2 stores an integer and a byte array whose
/// length is 2 bytes. Its marker byte is `0xd5`.
///
/// Note, that this function copies a byte array from the reader to the output buffer, which is
/// unlikely if you want zero-copy functionality.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
pub fn read_fixext2<R: RmpRead>(rd: &mut R) -> Result<(i8, [u8; 2]), ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::FixExt2 => {
let mut buf = [0; 2];
read_fixext_data(rd, &mut buf).map(|ty| (ty, buf))
}
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 6 bytes from the given reader and interpret them as a fixext4 type
/// with data attached.
///
/// According to the MessagePack specification, a fixext4 stores an integer and a byte array whose
/// length is 4 bytes. Its marker byte is `0xd6`.
///
/// Note, that this function copies a byte array from the reader to the output buffer, which is
/// unlikely if you want zero-copy functionality.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
pub fn read_fixext4<R: RmpRead>(rd: &mut R) -> Result<(i8, [u8; 4]), ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::FixExt4 => {
let mut buf = [0; 4];
read_fixext_data(rd, &mut buf).map(|ty| (ty, buf))
}
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 10 bytes from the given reader and interpret them as a fixext8 type
/// with data attached.
///
/// According to the MessagePack specification, a fixext8 stores an integer and a byte array whose
/// length is 8 bytes. Its marker byte is `0xd7`.
///
/// Note, that this function copies a byte array from the reader to the output buffer, which is
/// unlikely if you want zero-copy functionality.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
pub fn read_fixext8<R: RmpRead>(rd: &mut R) -> Result<(i8, [u8; 8]), ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::FixExt8 => {
let mut buf = [0; 8];
read_fixext_data(rd, &mut buf).map(|ty| (ty, buf))
}
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 18 bytes from the given reader and interpret them as a fixext16 type
/// with data attached.
///
/// According to the MessagePack specification, a fixext16 stores an integer and a byte array whose
/// length is 16 bytes. Its marker byte is `0xd8`.
///
/// Note, that this function copies a byte array from the reader to the output buffer, which is
/// unlikely if you want zero-copy functionality.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
pub fn read_fixext16<R: RmpRead>(rd: &mut R) -> Result<(i8, [u8; 16]), ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::FixExt16 => {
let mut buf = [0; 16];
read_fixext_data(rd, &mut buf).map(|ty| (ty, buf))
}
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
fn read_fixext_data<R: RmpRead>(rd: &mut R, buf: &mut [u8]) -> Result<i8, ValueReadError<R::Error>> {
let id = rd.read_data_i8()?;
match rd.read_exact_buf(buf) {
Ok(()) => Ok(id),
Err(err) => Err(ValueReadError::InvalidDataRead(err)),
}
}
/// Extension type meta information.
///
/// Extension represents a tuple of type information and a byte array where type information is an
/// integer whose meaning is defined by applications.
///
/// Applications can assign 0 to 127 to store application-specific type information.
///
/// # Note
///
/// MessagePack reserves -1 to -128 for future extension to add predefined types which will be
/// described in separated documents.
#[derive(Debug, PartialEq)]
pub struct ExtMeta {
/// Type information.
pub typeid: i8,
/// Byte array size.
pub size: u32,
}
pub fn read_ext_meta<R: RmpRead>(rd: &mut R) -> Result<ExtMeta, ValueReadError<R::Error>> {
let size = match read_marker(rd)? {
Marker::FixExt1 => 1,
Marker::FixExt2 => 2,
Marker::FixExt4 => 4,
Marker::FixExt8 => 8,
Marker::FixExt16 => 16,
Marker::Ext8 => u32::from(rd.read_data_u8()?),
Marker::Ext16 => u32::from(rd.read_data_u16()?),
Marker::Ext32 => rd.read_data_u32()?,
marker => return Err(ValueReadError::TypeMismatch(marker)),
};
let ty = rd.read_data_i8()?;
let meta = ExtMeta { typeid: ty, size };
Ok(meta)
}

465
third_party/rust/rmp/src/decode/mod.rs vendored Normal file
View File

@@ -0,0 +1,465 @@
//! Provides various functions and structs for MessagePack decoding.
//!
//! Most of the function defined in this module will silently handle interruption error (EINTR)
//! received from the given `Read` to be in consistent state with the `Write::write_all` method in
//! the standard library.
//!
//! Any other error would immediately interrupt the parsing process. If your reader can results in
//! I/O error and simultaneously be a recoverable state (for example, when reading from
//! non-blocking socket and it returns EWOULDBLOCK) be sure that you buffer the data externally
//! to avoid data loss (using `BufRead` readers with manual consuming or some other way).
mod dec;
mod ext;
mod sint;
mod str;
mod uint;
pub use self::dec::{read_f32, read_f64};
pub use self::ext::{
read_ext_meta, read_fixext1, read_fixext16, read_fixext2, read_fixext4, read_fixext8, ExtMeta,
};
pub use self::sint::{read_i16, read_i32, read_i64, read_i8, read_nfix};
#[allow(deprecated)]
// While we re-export deprecated items, we don't want to trigger warnings while compiling this crate
pub use self::str::{read_str, read_str_from_slice, read_str_len, read_str_ref, DecodeStringError};
pub use self::uint::{read_pfix, read_u16, read_u32, read_u64, read_u8};
use core::fmt::{self, Debug, Display, Formatter};
#[cfg(feature = "std")]
use std::error;
use num_traits::cast::FromPrimitive;
use crate::Marker;
pub mod bytes;
pub use bytes::Bytes;
#[doc(inline)]
#[allow(deprecated)]
pub use crate::errors::Error;
/// The error type for I/O operations on `RmpRead` and associated traits.
///
/// For [`std::io::Read`], this is [`std::io::Error`]
pub trait RmpReadErr: Display + Debug + crate::errors::MaybeErrBound + 'static {}
#[cfg(feature = "std")]
impl RmpReadErr for std::io::Error {}
impl RmpReadErr for core::convert::Infallible {}
macro_rules! read_byteorder_utils {
($($name:ident => $tp:ident),* $(,)?) => {
$(
#[inline]
#[doc(hidden)]
fn $name(&mut self) -> Result<$tp, ValueReadError<Self::Error>> where Self: Sized {
const SIZE: usize = core::mem::size_of::<$tp>();
let mut buf: [u8; SIZE] = [0u8; SIZE];
self.read_exact_buf(&mut buf).map_err(ValueReadError::InvalidDataRead)?;
Ok(paste::paste! {
<byteorder::BigEndian as byteorder::ByteOrder>::[<read_ $tp>](&mut buf)
})
}
)*
};
}
mod sealed {
pub trait Sealed {}
#[cfg(feature = "std")]
impl<T: ?Sized + std::io::Read> Sealed for T {}
#[cfg(not(feature = "std"))]
impl<'a> Sealed for &'a [u8] {}
impl Sealed for super::Bytes<'_> {}
}
/// A type that `rmp` supports reading from.
///
/// The methods of this trait should be considered an implementation detail (for now).
/// It is currently sealed (can not be implemented by the user).
///
/// See also [`std::io::Read`] and [`byteorder::ReadBytesExt`]
///
/// Its primary implementations are [`std::io::Read`] and [Bytes].
pub trait RmpRead: sealed::Sealed {
type Error: RmpReadErr;
/// Read a single (unsigned) byte from this stream
#[inline]
fn read_u8(&mut self) -> Result<u8, Self::Error> {
let mut buf = [0; 1];
self.read_exact_buf(&mut buf)?;
Ok(buf[0])
}
/// Read the exact number of bytes needed to fill the specified buffer.
///
/// If there are not enough bytes, this will return an error.
///
/// See also [`std::io::Read::read_exact`]
fn read_exact_buf(&mut self, buf: &mut [u8]) -> Result<(), Self::Error>;
// Internal helper functions to map I/O error into the `InvalidDataRead` error.
/// Read a single (unsigned) byte from this stream.
#[inline]
#[doc(hidden)]
fn read_data_u8(&mut self) -> Result<u8, ValueReadError<Self::Error>> {
self.read_u8().map_err(ValueReadError::InvalidDataRead)
}
/// Read a single (signed) byte from this stream.
#[inline]
#[doc(hidden)]
fn read_data_i8(&mut self) -> Result<i8, ValueReadError<Self::Error>> {
self.read_data_u8().map(|b| b as i8)
}
read_byteorder_utils!(
read_data_u16 => u16,
read_data_u32 => u32,
read_data_u64 => u64,
read_data_i16 => i16,
read_data_i32 => i32,
read_data_i64 => i64,
read_data_f32 => f32,
read_data_f64 => f64
);
}
/*
* HACK: rmpv & rmp-erde used the internal read_data_* functions.
*
* Since adding no_std support moved these functions to the RmpRead trait,
* this broke compatiblity (despite changing no public APIs).
*
* In theory, we could update rmpv and rmp-serde to use the new APIS,
* but that would be needless churn (and might surprise users who just want to update rmp proper).
*
* Instead, we emulate these internal APIs for now,
* so that rmpv and rmp-serde continue to compile without issue.
*
*
* TODO: Remove this hack once we release a new version of rmp proper
*/
macro_rules! wrap_data_funcs_for_compatibility {
($($tp:ident),* $(,)?) => {
$(paste::paste! {
#[cfg(feature = "std")]
#[doc(hidden)]
#[deprecated(note = "internal function. rmpv & rmp-serde need to switch to RmpRead")]
pub fn [<read_data_ $tp>] <R: std::io::Read>(buf: &mut R) -> Result<$tp, ValueReadError> {
buf.[<read_data_ $tp>]()
}
})*
};
}
wrap_data_funcs_for_compatibility!(
u8, u16, u32, u64,
i8, i16, i32, i64,
f32, f64
);
#[cfg(feature = "std")]
impl<T: std::io::Read> RmpRead for T {
type Error = std::io::Error;
#[inline]
fn read_exact_buf(&mut self, buf: &mut [u8]) -> Result<(), Self::Error> {
std::io::Read::read_exact(self, buf)
}
}
// An error returned from the `write_marker` and `write_fixval` functions.
struct MarkerWriteError<E: RmpReadErr>(E);
impl<E: RmpReadErr> From<E> for MarkerWriteError<E> {
#[cold]
fn from(err: E) -> Self {
MarkerWriteError(err)
}
}
/// An error that can occur when attempting to read a MessagePack marker from the reader.
#[derive(Debug)]
#[allow(deprecated)] // Needed for backwards compat
pub struct MarkerReadError<E: RmpReadErr = Error>(pub E);
/// An error which can occur when attempting to read a MessagePack value from the reader.
#[derive(Debug)]
#[allow(deprecated)] // Needed for backwards compat
pub enum ValueReadError<E: RmpReadErr = Error> {
/// Failed to read the marker.
InvalidMarkerRead(E),
/// Failed to read the data.
InvalidDataRead(E),
/// The type decoded isn't match with the expected one.
TypeMismatch(Marker),
}
#[cfg(feature = "std")]
impl error::Error for ValueReadError {
#[cold]
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
match *self {
ValueReadError::InvalidMarkerRead(ref err) |
ValueReadError::InvalidDataRead(ref err) => Some(err),
ValueReadError::TypeMismatch(..) => None,
}
}
}
impl Display for ValueReadError {
#[cold]
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
// TODO: This should probably use formatting
f.write_str(match *self {
ValueReadError::InvalidMarkerRead(..) => "failed to read MessagePack marker",
ValueReadError::InvalidDataRead(..) => "failed to read MessagePack data",
ValueReadError::TypeMismatch(..) => {
"the type decoded isn't match with the expected one"
}
})
}
}
impl<E: RmpReadErr> From<MarkerReadError<E>> for ValueReadError<E> {
#[cold]
fn from(err: MarkerReadError<E>) -> ValueReadError<E> {
match err {
MarkerReadError(err) => ValueReadError::InvalidMarkerRead(err),
}
}
}
impl<E: RmpReadErr> From<E> for MarkerReadError<E> {
#[cold]
fn from(err: E) -> MarkerReadError<E> {
MarkerReadError(err)
}
}
/// Attempts to read a single byte from the given reader and to decode it as a MessagePack marker.
#[inline]
pub fn read_marker<R: RmpRead>(rd: &mut R) -> Result<Marker, MarkerReadError<R::Error>> {
Ok(Marker::from_u8(rd.read_u8()?))
}
/// Attempts to read a single byte from the given reader and to decode it as a nil value.
///
/// According to the MessagePack specification, a nil value is represented as a single `0xc0` byte.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading the nil marker,
/// except the EINTR, which is handled internally.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_nil<R: RmpRead>(rd: &mut R) -> Result<(), ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::Null => Ok(()),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read a single byte from the given reader and to decode it as a boolean value.
///
/// According to the MessagePack specification, an encoded boolean value is represented as a single
/// byte.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading the bool marker,
/// except the EINTR, which is handled internally.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_bool<R: RmpRead>(rd: &mut R) -> Result<bool, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::True => Ok(true),
Marker::False => Ok(false),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// An error which can occur when attempting to read a MessagePack numeric value from the reader.
#[derive(Debug)]
#[allow(deprecated)] // Used for compatibility
pub enum NumValueReadError<E: RmpReadErr = Error> {
/// Failed to read the marker.
InvalidMarkerRead(E),
/// Failed to read the data.
InvalidDataRead(E),
/// The type decoded isn't match with the expected one.
TypeMismatch(Marker),
/// Out of range integral type conversion attempted.
OutOfRange,
}
#[cfg(feature = "std")]
impl error::Error for NumValueReadError {
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
match *self {
NumValueReadError::InvalidMarkerRead(ref err) |
NumValueReadError::InvalidDataRead(ref err) => Some(err),
NumValueReadError::TypeMismatch(..) |
NumValueReadError::OutOfRange => None,
}
}
}
impl<E: RmpReadErr> Display for NumValueReadError<E> {
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
f.write_str(match *self {
NumValueReadError::InvalidMarkerRead(..) => "failed to read MessagePack marker",
NumValueReadError::InvalidDataRead(..) => "failed to read MessagePack data",
NumValueReadError::TypeMismatch(..) => {
"the type decoded isn't match with the expected one"
}
NumValueReadError::OutOfRange => "out of range integral type conversion attempted",
})
}
}
impl<E: RmpReadErr> From<MarkerReadError<E>> for NumValueReadError<E> {
#[cold]
fn from(err: MarkerReadError<E>) -> NumValueReadError<E> {
match err {
MarkerReadError(err) => NumValueReadError::InvalidMarkerRead(err),
}
}
}
impl<E: RmpReadErr> From<ValueReadError<E>> for NumValueReadError<E> {
#[cold]
fn from(err: ValueReadError<E>) -> NumValueReadError<E> {
match err {
ValueReadError::InvalidMarkerRead(err) => NumValueReadError::InvalidMarkerRead(err),
ValueReadError::InvalidDataRead(err) => NumValueReadError::InvalidDataRead(err),
ValueReadError::TypeMismatch(err) => NumValueReadError::TypeMismatch(err),
}
}
}
/// Attempts to read up to 9 bytes from the given reader and to decode them as integral `T` value.
///
/// This function will try to read up to 9 bytes from the reader (1 for marker and up to 8 for data)
/// and interpret them as a big-endian `T`.
///
/// Unlike `read_*`, this function weakens type restrictions, allowing you to safely decode packed
/// values even if you aren't sure about the actual integral type.
///
/// # Errors
///
/// This function will return `NumValueReadError` on any I/O error while reading either the marker
/// or the data.
///
/// It also returns `NumValueReadError::OutOfRange` if the actual type is not an integer or it does
/// not fit in the given numeric range.
///
/// # Examples
///
/// ```
/// let buf = [0xcd, 0x1, 0x2c];
///
/// assert_eq!(300u16, rmp::decode::read_int(&mut &buf[..]).unwrap());
/// assert_eq!(300i16, rmp::decode::read_int(&mut &buf[..]).unwrap());
/// assert_eq!(300u32, rmp::decode::read_int(&mut &buf[..]).unwrap());
/// assert_eq!(300i32, rmp::decode::read_int(&mut &buf[..]).unwrap());
/// assert_eq!(300u64, rmp::decode::read_int(&mut &buf[..]).unwrap());
/// assert_eq!(300i64, rmp::decode::read_int(&mut &buf[..]).unwrap());
/// assert_eq!(300usize, rmp::decode::read_int(&mut &buf[..]).unwrap());
/// assert_eq!(300isize, rmp::decode::read_int(&mut &buf[..]).unwrap());
/// ```
pub fn read_int<T: FromPrimitive, R: RmpRead>(rd: &mut R) -> Result<T, NumValueReadError<R::Error>> {
let val = match read_marker(rd)? {
Marker::FixPos(val) => T::from_u8(val),
Marker::FixNeg(val) => T::from_i8(val),
Marker::U8 => T::from_u8(rd.read_data_u8()?),
Marker::U16 => T::from_u16(rd.read_data_u16()?),
Marker::U32 => T::from_u32(rd.read_data_u32()?),
Marker::U64 => T::from_u64(rd.read_data_u64()?),
Marker::I8 => T::from_i8(rd.read_data_i8()?),
Marker::I16 => T::from_i16(rd.read_data_i16()?),
Marker::I32 => T::from_i32(rd.read_data_i32()?),
Marker::I64 => T::from_i64(rd.read_data_i64()?),
marker => return Err(NumValueReadError::TypeMismatch(marker)),
};
val.ok_or(NumValueReadError::OutOfRange)
}
/// Attempts to read up to 5 bytes from the given reader and to decode them as a big-endian u32
/// array size.
///
/// Array format family stores a sequence of elements in 1, 3, or 5 bytes of extra bytes in addition
/// to the elements.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
// TODO: Docs.
// NOTE: EINTR is managed internally.
pub fn read_array_len<R>(rd: &mut R) -> Result<u32, ValueReadError<R::Error>>
where
R: RmpRead,
{
match read_marker(rd)? {
Marker::FixArray(size) => Ok(u32::from(size)),
Marker::Array16 => Ok(u32::from(rd.read_data_u16()?)),
Marker::Array32 => Ok(rd.read_data_u32()?),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read up to 5 bytes from the given reader and to decode them as a big-endian u32
/// map size.
///
/// Map format family stores a sequence of elements in 1, 3, or 5 bytes of extra bytes in addition
/// to the elements.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
// TODO: Docs.
pub fn read_map_len<R: RmpRead>(rd: &mut R) -> Result<u32, ValueReadError<R::Error>> {
let marker = read_marker(rd)?;
marker_to_len(rd, marker)
}
pub fn marker_to_len<R: RmpRead>(rd: &mut R, marker: Marker) -> Result<u32, ValueReadError<R::Error>> {
match marker {
Marker::FixMap(size) => Ok(u32::from(size)),
Marker::Map16 => Ok(u32::from(rd.read_data_u16()?)),
Marker::Map32 => Ok(rd.read_data_u32()?),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read up to 5 bytes from the given reader and to decode them as Binary array length.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
// TODO: Docs.
pub fn read_bin_len<R: RmpRead>(rd: &mut R) -> Result<u32, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::Bin8 => Ok(u32::from(rd.read_data_u8()?)),
Marker::Bin16 => Ok(u32::from(rd.read_data_u16()?)),
Marker::Bin32 => Ok(rd.read_data_u32()?),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}

119
third_party/rust/rmp/src/decode/sint.rs vendored Normal file
View File

@@ -0,0 +1,119 @@
use super::{read_marker, RmpRead, ValueReadError};
use crate::Marker;
/// Attempts to read a single byte from the given reader and to decode it as a negative fixnum
/// value.
///
/// According to the MessagePack specification, a negative fixed integer value is represented using
/// a single byte in `[0xe0; 0xff]` range inclusively, prepended with a special marker mask.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading the marker,
/// except the EINTR, which is handled internally.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_nfix<R: RmpRead>(rd: &mut R) -> Result<i8, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::FixNeg(val) => Ok(val),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 2 bytes from the given reader and to decode them as `i8` value.
///
/// The first byte should be the marker and the second one should represent the data itself.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_i8<R: RmpRead>(rd: &mut R) -> Result<i8, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::I8 => rd.read_data_i8(),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 3 bytes from the given reader and to decode them as `i16` value.
///
/// The first byte should be the marker and the others should represent the data itself.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_i16<R: RmpRead>(rd: &mut R) -> Result<i16, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::I16 => rd.read_data_i16(),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 5 bytes from the given reader and to decode them as `i32` value.
///
/// The first byte should be the marker and the others should represent the data itself.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_i32<R: RmpRead>(rd: &mut R) -> Result<i32, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::I32 => rd.read_data_i32(),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 9 bytes from the given reader and to decode them as `i64` value.
///
/// The first byte should be the marker and the others should represent the data itself.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_i64<R: RmpRead>(rd: &mut R) -> Result<i64, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::I64 => rd.read_data_i64(),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}

192
third_party/rust/rmp/src/decode/str.rs vendored Normal file
View File

@@ -0,0 +1,192 @@
#[cfg(feature = "std")]
use std::error;
use core::fmt::{self, Display, Formatter};
use core::str::{from_utf8, Utf8Error};
use super::{read_marker, RmpRead, RmpReadErr, ValueReadError};
use crate::Marker;
#[derive(Debug)]
#[allow(deprecated)] // Only for compatibility
pub enum DecodeStringError<'a, E: RmpReadErr = super::Error> {
InvalidMarkerRead(E),
InvalidDataRead(E),
TypeMismatch(Marker),
/// The given buffer is not large enough to accumulate the specified amount of bytes.
BufferSizeTooSmall(u32),
InvalidUtf8(&'a [u8], Utf8Error),
}
#[cfg(feature = "std")]
impl<'a, E: RmpReadErr> error::Error for DecodeStringError<'a, E> {
#[cold]
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
match *self {
DecodeStringError::InvalidMarkerRead(ref err) |
DecodeStringError::InvalidDataRead(ref err) => Some(err),
DecodeStringError::TypeMismatch(..) |
DecodeStringError::BufferSizeTooSmall(..) => None,
DecodeStringError::InvalidUtf8(_, ref err) => Some(err),
}
}
}
impl<'a, E: RmpReadErr> Display for DecodeStringError<'a, E> {
#[cold]
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
f.write_str("error while decoding string")
}
}
impl<'a, E: RmpReadErr> From<ValueReadError<E>> for DecodeStringError<'a, E> {
#[cold]
fn from(err: ValueReadError<E>) -> DecodeStringError<'a, E> {
match err {
ValueReadError::InvalidMarkerRead(err) => DecodeStringError::InvalidMarkerRead(err),
ValueReadError::InvalidDataRead(err) => DecodeStringError::InvalidDataRead(err),
ValueReadError::TypeMismatch(marker) => DecodeStringError::TypeMismatch(marker),
}
}
}
/// Attempts to read up to 9 bytes from the given reader and to decode them as a string `u32` size
/// value.
///
/// According to the MessagePack specification, the string format family stores an byte array in 1,
/// 2, 3, or 5 bytes of extra bytes in addition to the size of the byte array.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
#[inline]
pub fn read_str_len<R: RmpRead>(rd: &mut R) -> Result<u32, ValueReadError<R::Error>> {
Ok(read_str_len_with_nread(rd)?.0)
}
fn read_str_len_with_nread<R>(rd: &mut R) -> Result<(u32, usize), ValueReadError<R::Error>>
where R: RmpRead
{
match read_marker(rd)? {
Marker::FixStr(size) => Ok((u32::from(size), 1)),
Marker::Str8 => Ok((u32::from(rd.read_data_u8()?), 2)),
Marker::Str16 => Ok((u32::from(rd.read_data_u16()?), 3)),
Marker::Str32 => Ok((rd.read_data_u32()?, 5)),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read a string data from the given reader and copy it to the buffer provided.
///
/// On success returns a borrowed string type, allowing to view the copied bytes as properly utf-8
/// string.
/// According to the spec, the string's data must to be encoded using utf-8.
///
/// # Errors
///
/// Returns `Err` in the following cases:
///
/// - if any IO error (including unexpected EOF) occurs, while reading an `rd`, except the EINTR,
/// which is handled internally.
/// - if the `out` buffer size is not large enough to keep all the data copied.
/// - if the data is not utf-8, with a description as to why the provided data is not utf-8 and
/// with a size of bytes actually copied to be able to get them from `out`.
///
/// # Examples
/// ```
/// use rmp::decode::read_str;
///
/// let buf = [0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65];
/// let mut out = [0u8; 16];
///
/// assert_eq!("le message", read_str(&mut &buf[..], &mut &mut out[..]).unwrap());
/// ```
///
/// # Unstable
///
/// This function is **unstable**, because it needs review.
// TODO: Stabilize. Mark error values for each error case (in docs).
pub fn read_str<'r, R>(rd: &mut R, buf: &'r mut [u8]) -> Result<&'r str, DecodeStringError<'r, R::Error>>
where
R: RmpRead,
{
let len = read_str_len(rd)?;
let ulen = len as usize;
if buf.len() < ulen {
return Err(DecodeStringError::BufferSizeTooSmall(len));
}
read_str_data(rd, len, &mut buf[0..ulen])
}
pub fn read_str_data<'r, R>(rd: &mut R,
len: u32,
buf: &'r mut [u8])
-> Result<&'r str, DecodeStringError<'r, R::Error>>
where R: RmpRead
{
debug_assert_eq!(len as usize, buf.len());
// Trying to copy exact `len` bytes.
match rd.read_exact_buf(buf) {
Ok(()) => match from_utf8(buf) {
Ok(decoded) => Ok(decoded),
Err(err) => Err(DecodeStringError::InvalidUtf8(buf, err)),
},
Err(err) => Err(DecodeStringError::InvalidDataRead(err)),
}
}
/// Attempts to read and decode a string value from the reader, returning a borrowed slice from it.
///
// TODO: Also it's possible to implement all borrowing functions for all `BufRead` implementors.
#[deprecated(since = "0.8.6", note = "useless, use `read_str_from_slice` instead")]
pub fn read_str_ref(rd: &[u8]) -> Result<&[u8], DecodeStringError<'_, super::bytes::BytesReadError>> {
let mut cur = super::Bytes::new(rd);
let len = read_str_len(&mut cur)?;
Ok(&cur.remaining_slice()[..len as usize])
}
/// Attempts to read and decode a string value from the reader, returning a borrowed slice from it.
///
/// # Examples
///
/// ```
/// use rmp::encode::write_str;
/// use rmp::decode::read_str_from_slice;
///
/// let mut buf = Vec::new();
/// write_str(&mut buf, "Unpacking").unwrap();
/// write_str(&mut buf, "multiple").unwrap();
/// write_str(&mut buf, "strings").unwrap();
///
/// let mut chunks = Vec::new();
/// let mut unparsed = &buf[..];
/// while let Ok((chunk, tail)) = read_str_from_slice(unparsed) {
/// chunks.push(chunk);
/// unparsed = tail;
/// }
///
/// assert_eq!(vec!["Unpacking", "multiple", "strings"], chunks);
/// ```
pub fn read_str_from_slice<T: ?Sized + AsRef<[u8]>>(
buf: &T,
) -> Result<(&str, &[u8]), DecodeStringError<'_, super::bytes::BytesReadError>> {
let buf = buf.as_ref();
let (len, nread) = read_str_len_with_nread(&mut super::Bytes::new(buf))?;
let ulen = len as usize;
if buf[nread..].len() >= ulen {
let (head, tail) = buf.split_at(nread + ulen);
match from_utf8(&head[nread..]) {
Ok(val) => Ok((val, tail)),
Err(err) => Err(DecodeStringError::InvalidUtf8(buf, err)),
}
} else {
Err(DecodeStringError::BufferSizeTooSmall(len))
}
}

114
third_party/rust/rmp/src/decode/uint.rs vendored Normal file
View File

@@ -0,0 +1,114 @@
use super::{read_marker, RmpRead, ValueReadError};
use crate::Marker;
/// Attempts to read a single byte from the given reader and to decode it as a positive fixnum
/// value.
///
/// According to the MessagePack specification, a positive fixed integer value is represented using
/// a single byte in `[0x00; 0x7f]` range inclusively, prepended with a special marker mask.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading the marker,
/// except the EINTR, which is handled internally.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_pfix<R: RmpRead>(rd: &mut R) -> Result<u8, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::FixPos(val) => Ok(val),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 2 bytes from the given reader and to decode them as `u8` value.
///
/// The first byte should be the marker and the second one should represent the data itself.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
pub fn read_u8<R: RmpRead>(rd: &mut R) -> Result<u8, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::U8 => rd.read_data_u8(),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 3 bytes from the given reader and to decode them as `u16` value.
///
/// The first byte should be the marker and the others should represent the data itself.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_u16<R: RmpRead>(rd: &mut R) -> Result<u16, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::U16 => rd.read_data_u16(),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 5 bytes from the given reader and to decode them as `u32` value.
///
/// The first byte should be the marker and the others should represent the data itself.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_u32<R: RmpRead>(rd: &mut R) -> Result<u32, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::U32 => rd.read_data_u32(),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}
/// Attempts to read exactly 9 bytes from the given reader and to decode them as `u64` value.
///
/// The first byte should be the marker and the others should represent the data itself.
///
/// # Errors
///
/// This function will return `ValueReadError` on any I/O error while reading either the marker or
/// the data.
///
/// It also returns `ValueReadError::TypeMismatch` if the actual type is not equal with the
/// expected one, indicating you with the actual type.
///
/// # Note
///
/// This function will silently retry on every EINTR received from the underlying `Read` until
/// successful read.
pub fn read_u64<R: RmpRead>(rd: &mut R) -> Result<u64, ValueReadError<R::Error>> {
match read_marker(rd)? {
Marker::U64 => rd.read_data_u64(),
marker => Err(ValueReadError::TypeMismatch(marker)),
}
}

45
third_party/rust/rmp/src/encode/bin.rs vendored Normal file
View File

@@ -0,0 +1,45 @@
use super::RmpWrite;
use crate::encode::{write_marker, ValueWriteError};
use crate::Marker;
/// Encodes and attempts to write the most efficient binary array length implementation to the given
/// write, returning the marker used.
///
/// This function is useful when you want to get full control for writing the data itself, for
/// example, when using non-blocking socket.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_bin_len<W: RmpWrite>(wr: &mut W, len: u32) -> Result<Marker, ValueWriteError<W::Error>> {
let marker = if len < 256 {
Marker::Bin8
} else if len <= u16::MAX as u32 {
Marker::Bin16
} else {
Marker::Bin32
};
write_marker(&mut *wr, marker)?;
if marker == Marker::Bin8 {
wr.write_data_u8(len as u8)?;
} else if marker == Marker::Bin16 {
wr.write_data_u16(len as u16)?;
} else if marker == Marker::Bin32 {
wr.write_data_u32(len)?;
}
Ok(marker)
}
/// Encodes and attempts to write the most efficient binary implementation to the given `Write`.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
// TODO: Docs, range check, example, visibility.
pub fn write_bin<W: RmpWrite>(wr: &mut W, data: &[u8]) -> Result<(), ValueWriteError<W::Error>> {
write_bin_len(wr, data.len() as u32)?;
wr.write_bytes(data)
.map_err(ValueWriteError::InvalidDataWrite)
}

189
third_party/rust/rmp/src/encode/buffer.rs vendored Normal file
View File

@@ -0,0 +1,189 @@
//! Implementation of the [`ByteBuf`] type
use super::RmpWrite;
use alloc::vec::Vec;
#[cfg(not(feature = "std"))]
use core::fmt::{self, Display, Formatter};
/// An error returned from writing to `&mut [u8]` (a byte buffer of fixed capacity) on no_std
///
/// In feature="std", capacity overflow in `<&mut [u8] as std::io::Write>::write_exact()`
/// currently returns [`ErrorKind::WriteZero`](https://doc.rust-lang.org/std/io/enum.ErrorKind.html#variant.WriteZero).
///
/// Since no_std doesn't have std::io::Error we use this instead ;)
///
/// This is specific to `#[cfg(not(feature = "std"))]` so it is `#[doc(hidden)]`
#[derive(Debug)]
#[cfg(not(feature = "std"))]
#[doc(hidden)]
pub struct FixedBufCapacityOverflow {
_priv: (),
}
/// An error returned from writing to `&mut [u8]`
///
/// Aliased for compatibility with `no_std` mode.
#[cfg(feature = "std")]
#[doc(hidden)]
pub type FixedBufCapacityOverflow = std::io::Error;
#[cfg(not(feature = "std"))]
impl Display for FixedBufCapacityOverflow {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
// This is intentionally vauge because std::io::Error is
// Doesn't make sense for no_std to have bettetr errors than std
f.write_str("Capacity overflow for fixed-size byte buffer")
}
}
#[cfg(not(feature = "std"))]
impl crate::encode::RmpWriteErr for FixedBufCapacityOverflow {}
/// Fallback implementation for fixed-capacity buffers
///
/// Only needed for no-std because we don't have
/// the blanket impl for `std::io::Write`
#[cfg(not(feature = "std"))]
impl<'a> RmpWrite for &'a mut [u8] {
type Error = FixedBufCapacityOverflow;
#[inline]
fn write_bytes(&mut self, buf: &[u8]) -> Result<(), Self::Error> {
let to_write = buf.len();
let remaining = self.len();
if to_write <= remaining {
self[..to_write].copy_from_slice(buf);
unsafe {
//Cant use split_at or re-borrowing due to lifetime errors :(
*self = core::slice::from_raw_parts_mut(
self.as_mut_ptr().add(to_write),
remaining - to_write,
)
}
Ok(())
} else {
Err(FixedBufCapacityOverflow { _priv: () })
}
}
}
/// A wrapper around `Vec<u8>` to serialize more efficiently.
///
/// This has a specialized implementation of `RmpWrite`
/// It gives `std::convert::Infailable` for errors.
/// This is because writing to `Vec<T>` can only fail due to allocation.
///
/// This has the additional benefit of working on `#[no_std]`
///
/// See also [serde_bytes::ByteBuf](https://docs.rs/serde_bytes/0.11/serde_bytes/struct.ByteBuf.html)
#[derive(Debug, Clone, Default, Eq, PartialEq, Hash, Ord, PartialOrd)]
pub struct ByteBuf {
bytes: Vec<u8>,
}
impl ByteBuf {
/// Construct a new empty buffer
#[inline]
#[must_use]
pub fn new() -> Self {
ByteBuf { bytes: Vec::new() }
}
/// Construct a new buffer with the specified capacity
///
/// See [`Vec::with_capacity`] for details
#[inline]
#[must_use]
pub fn with_capacity(capacity: usize) -> Self {
ByteBuf {
bytes: Vec::with_capacity(capacity),
}
}
/// Unwrap the underlying buffer of this vector
#[inline]
#[must_use]
pub fn into_vec(self) -> Vec<u8> {
self.bytes
}
/// Wrap the specified vector as a [`ByteBuf`]
#[inline]
#[must_use]
pub fn from_vec(bytes: Vec<u8>) -> Self {
ByteBuf { bytes }
}
/// Get a reference to this type as a [Vec]
#[inline]
#[must_use]
pub fn as_vec(&self) -> &Vec<u8> {
&self.bytes
}
/// Get a mutable reference to this type as a [Vec]
#[inline]
pub fn as_mut_vec(&mut self) -> &mut Vec<u8> {
&mut self.bytes
}
/// Get a reference to this type as a slice of bytes (`&[u8]`)
#[inline]
#[must_use]
pub fn as_slice(&self) -> &[u8] {
&self.bytes
}
}
impl AsRef<[u8]> for ByteBuf {
fn as_ref(&self) -> &[u8] {
&self.bytes
}
}
impl AsRef<Vec<u8>> for ByteBuf {
#[inline]
fn as_ref(&self) -> &Vec<u8> {
&self.bytes
}
}
impl AsMut<Vec<u8>> for ByteBuf {
#[inline]
fn as_mut(&mut self) -> &mut Vec<u8> {
&mut self.bytes
}
}
impl From<ByteBuf> for Vec<u8> {
#[inline]
fn from(buf: ByteBuf) -> Self {
buf.bytes
}
}
impl From<Vec<u8>> for ByteBuf {
#[inline]
fn from(bytes: Vec<u8>) -> Self {
ByteBuf { bytes }
}
}
impl RmpWrite for ByteBuf {
type Error = core::convert::Infallible;
#[inline]
fn write_u8(&mut self, val: u8) -> Result<(), Self::Error> {
self.bytes.push(val);
Ok(())
}
#[inline]
fn write_bytes(&mut self, buf: &[u8]) -> Result<(), Self::Error> {
self.bytes.extend_from_slice(buf);
Ok(())
}
}
#[cfg(not(feature = "std"))]
impl<'a> RmpWrite for Vec<u8> {
type Error = core::convert::Infallible;
#[inline]
fn write_u8(&mut self, val: u8) -> Result<(), Self::Error> {
self.push(val);
Ok(())
}
#[inline]
fn write_bytes(&mut self, buf: &[u8]) -> Result<(), Self::Error> {
self.extend_from_slice(buf);
Ok(())
}
}

31
third_party/rust/rmp/src/encode/dec.rs vendored Normal file
View File

@@ -0,0 +1,31 @@
use super::{write_marker, RmpWrite};
use crate::encode::ValueWriteError;
use crate::Marker;
/// Encodes and attempts to write an `f32` value as a 5-byte sequence into the given write.
///
/// The first byte becomes the `f32` marker and the others will represent the data itself.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_f32<W: RmpWrite>(wr: &mut W, val: f32) -> Result<(), ValueWriteError<W::Error>> {
write_marker(wr, Marker::F32)?;
wr.write_data_f32(val)?;
Ok(())
}
/// Encodes and attempts to write an `f64` value as a 9-byte sequence into the given write.
///
/// The first byte becomes the `f64` marker and the others will represent the data itself.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_f64<W: RmpWrite>(wr: &mut W, val: f64) -> Result<(), ValueWriteError<W::Error>> {
write_marker(wr, Marker::F64)?;
wr.write_data_f64(val)?;
Ok(())
}

1
third_party/rust/rmp/src/encode/ext.rs vendored Normal file
View File

@@ -0,0 +1 @@

1
third_party/rust/rmp/src/encode/map.rs vendored Normal file
View File

@@ -0,0 +1 @@

337
third_party/rust/rmp/src/encode/mod.rs vendored Normal file
View File

@@ -0,0 +1,337 @@
//! Provides various functions and structs for MessagePack encoding.
mod bin;
mod dec;
mod ext;
mod map;
mod sint;
mod str;
mod uint;
mod vec;
pub use self::bin::{write_bin, write_bin_len};
pub use self::dec::{write_f32, write_f64};
pub use self::sint::{write_i16, write_i32, write_i64, write_i8, write_nfix, write_sint};
pub use self::str::{write_str, write_str_len};
pub use self::uint::{write_pfix, write_u16, write_u32, write_u64, write_u8, write_uint, write_uint8};
use core::fmt::{self, Debug, Display, Formatter};
#[cfg(feature = "std")]
use std::error;
use crate::Marker;
pub mod buffer;
pub use buffer::ByteBuf;
#[doc(inline)]
#[allow(deprecated)]
pub use crate::errors::Error;
/// The error type for operations on the [`RmpWrite`] trait.
///
/// For [`std::io::Write`], this is [`std::io::Error`]
/// For [`ByteBuf`], this is [`core::convert::Infallible`]
pub trait RmpWriteErr: Display + Debug + crate::errors::MaybeErrBound + 'static {}
#[cfg(feature = "std")]
impl RmpWriteErr for std::io::Error {}
impl RmpWriteErr for core::convert::Infallible {}
// An error returned from the `write_marker` and `write_fixval` functions.
struct MarkerWriteError<E: RmpWriteErr>(E);
impl<E: RmpWriteErr> From<E> for MarkerWriteError<E> {
#[cold]
fn from(err: E) -> Self {
MarkerWriteError(err)
}
}
/// Attempts to write the given marker into the writer.
fn write_marker<W: RmpWrite>(wr: &mut W, marker: Marker) -> Result<(), MarkerWriteError<W::Error>> {
wr.write_u8(marker.to_u8()).map_err(MarkerWriteError)
}
/// An error returned from primitive values write functions.
#[doc(hidden)]
pub struct DataWriteError<E: RmpWriteErr>(E);
impl<E: RmpWriteErr> From<E> for DataWriteError<E> {
#[cold]
#[inline]
fn from(err: E) -> DataWriteError<E> {
DataWriteError(err)
}
}
/// Encodes and attempts to write a nil value into the given write.
///
/// According to the MessagePack specification, a nil value is represented as a single `0xc0` byte.
///
/// # Errors
///
/// This function will return `Error` on any I/O error occurred while writing the nil marker.
///
/// # Examples
///
/// ```
/// let mut buf = Vec::new();
///
/// rmp::encode::write_nil(&mut buf).unwrap();
///
/// assert_eq!(vec![0xc0], buf);
/// ```
#[inline]
pub fn write_nil<W: RmpWrite>(wr: &mut W) -> Result<(), W::Error> {
write_marker(wr, Marker::Null).map_err(|e| e.0)
}
/// Encodes and attempts to write a bool value into the given write.
///
/// According to the MessagePack specification, an encoded boolean value is represented as a single
/// byte.
///
/// # Errors
///
/// Each call to this function may generate an I/O error indicating that the operation could not be
/// completed.
#[inline]
pub fn write_bool<W: RmpWrite>(wr: &mut W, val: bool) -> Result<(), W::Error> {
let marker = if val { Marker::True } else { Marker::False };
write_marker(wr, marker).map_err(|e| e.0)
}
mod sealed {
pub trait Sealed {}
#[cfg(feature = "std")]
impl<T: ?Sized + std::io::Write> Sealed for T {}
#[cfg(not(feature = "std"))]
impl Sealed for &mut [u8] {}
#[cfg(not(feature = "std"))]
impl Sealed for alloc::vec::Vec<u8> {}
impl Sealed for super::ByteBuf {}
}
macro_rules! write_byteorder_utils {
($($name:ident => $tp:ident),* $(,)?) => {
$(
#[inline]
#[doc(hidden)]
fn $name(&mut self, val: $tp) -> Result<(), DataWriteError<Self::Error>> where Self: Sized {
const SIZE: usize = core::mem::size_of::<$tp>();
let mut buf: [u8; SIZE] = [0u8; SIZE];
paste::paste! {
<byteorder::BigEndian as byteorder::ByteOrder>::[<write_ $tp>](&mut buf, val);
}
self.write_bytes(&buf).map_err(DataWriteError)
}
)*
};
}
/// A type that `rmp` supports writing into.
///
/// The methods of this trait should be considered an implementation detail (for now).
/// It is currently sealed (can not be implemented by the user).
///
/// See also [`std::io::Write`] and [`byteorder::WriteBytesExt`]
///
/// Its primary implementations are [`std::io::Write`] and [`ByteBuf`].
pub trait RmpWrite: sealed::Sealed {
type Error: RmpWriteErr;
/// Write a single byte to this stream
#[inline]
fn write_u8(&mut self, val: u8) -> Result<(), Self::Error> {
let buf = [val];
self.write_bytes(&buf)
}
/// Write a slice of bytes to the underlying stream
///
/// This will either write all the bytes or return an error.
/// See also [`std::io::Write::write_all`]
fn write_bytes(&mut self, buf: &[u8]) -> Result<(), Self::Error>;
// Internal helper functions to map I/O error into the `DataWriteError` error.
/// Write a single (signed) byte to this stream.
#[inline]
#[doc(hidden)]
fn write_data_u8(&mut self, val: u8) -> Result<(), DataWriteError<Self::Error>> {
self.write_u8(val).map_err(DataWriteError)
}
/// Write a single (signed) byte to this stream.
#[inline]
#[doc(hidden)]
fn write_data_i8(&mut self, val: i8) -> Result<(), DataWriteError<Self::Error>> {
self.write_data_u8(val as u8)
}
write_byteorder_utils!(
write_data_u16 => u16,
write_data_u32 => u32,
write_data_u64 => u64,
write_data_i16 => i16,
write_data_i32 => i32,
write_data_i64 => i64,
write_data_f32 => f32,
write_data_f64 => f64
);
}
#[cfg(feature = "std")]
impl<T: std::io::Write> RmpWrite for T {
type Error = std::io::Error;
#[inline]
fn write_bytes(&mut self, buf: &[u8]) -> Result<(), Self::Error> {
self.write_all(buf)
}
}
/// An error that can occur when attempting to write multi-byte MessagePack value.
#[derive(Debug)]
#[allow(deprecated)] // TODO: Needed for compatibility
pub enum ValueWriteError<E: RmpWriteErr = Error> {
/// I/O error while writing marker.
InvalidMarkerWrite(E),
/// I/O error while writing data.
InvalidDataWrite(E),
}
impl<E: RmpWriteErr> From<MarkerWriteError<E>> for ValueWriteError<E> {
#[cold]
fn from(err: MarkerWriteError<E>) -> Self {
match err {
MarkerWriteError(err) => ValueWriteError::InvalidMarkerWrite(err),
}
}
}
impl<E: RmpWriteErr> From<DataWriteError<E>> for ValueWriteError<E> {
#[cold]
fn from(err: DataWriteError<E>) -> Self {
match err {
DataWriteError(err) => ValueWriteError::InvalidDataWrite(err),
}
}
}
#[cfg(feature = "std")] // Backwards compatbility ;)
impl From<ValueWriteError<std::io::Error>> for std::io::Error {
#[cold]
fn from(err: ValueWriteError<std::io::Error>) -> std::io::Error {
match err {
ValueWriteError::InvalidMarkerWrite(err) |
ValueWriteError::InvalidDataWrite(err) => err,
}
}
}
#[cfg(feature = "std")]
impl<E: RmpWriteErr> error::Error for ValueWriteError<E> {
#[cold]
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
match *self {
ValueWriteError::InvalidMarkerWrite(ref err) |
ValueWriteError::InvalidDataWrite(ref err) => Some(err),
}
}
}
impl<E: RmpWriteErr> Display for ValueWriteError<E> {
#[cold]
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
f.write_str("error while writing multi-byte MessagePack value")
}
}
/// Encodes and attempts to write the most efficient array length implementation to the given write,
/// returning the marker used.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_array_len<W: RmpWrite>(wr: &mut W, len: u32) -> Result<Marker, ValueWriteError<W::Error>> {
let marker = if len < 16 {
Marker::FixArray(len as u8)
} else if len <= u16::MAX as u32 {
Marker::Array16
} else {
Marker::Array32
};
write_marker(wr, marker)?;
if marker == Marker::Array16 {
wr.write_data_u16(len as u16)?;
} else if marker == Marker::Array32 {
wr.write_data_u32(len)?;
}
Ok(marker)
}
/// Encodes and attempts to write the most efficient map length implementation to the given write,
/// returning the marker used.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_map_len<W: RmpWrite>(wr: &mut W, len: u32) -> Result<Marker, ValueWriteError<W::Error>> {
let marker = if len < 16 {
Marker::FixMap(len as u8)
} else if len <= u16::MAX as u32 {
Marker::Map16
} else {
Marker::Map32
};
write_marker(wr, marker)?;
if marker == Marker::Map16 {
wr.write_data_u16(len as u16)?;
} else if marker == Marker::Map32 {
wr.write_data_u32(len)?;
}
Ok(marker)
}
/// Encodes and attempts to write the most efficient ext metadata implementation to the given
/// write, returning the marker used.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
///
/// # Panics
///
/// Panics if `ty` is negative, because it is reserved for future MessagePack extension including
/// 2-byte type information.
pub fn write_ext_meta<W: RmpWrite>(wr: &mut W, len: u32, ty: i8) -> Result<Marker, ValueWriteError<W::Error>> {
let marker = match len {
1 => Marker::FixExt1,
2 => Marker::FixExt2,
4 => Marker::FixExt4,
8 => Marker::FixExt8,
16 => Marker::FixExt16,
0..=255 => Marker::Ext8,
256..=65535 => Marker::Ext16,
_ => Marker::Ext32,
};
write_marker(wr, marker)?;
if marker == Marker::Ext8 {
wr.write_data_u8(len as u8)?;
} else if marker == Marker::Ext16 {
wr.write_data_u16(len as u16)?;
} else if marker == Marker::Ext32 {
wr.write_data_u32(len)?;
}
wr.write_data_i8(ty)?;
Ok(marker)
}

168
third_party/rust/rmp/src/encode/sint.rs vendored Normal file
View File

@@ -0,0 +1,168 @@
use super::{write_marker, RmpWrite};
use crate::encode::{write_pfix, write_u16, write_u32, write_u64, write_u8, ValueWriteError};
use crate::Marker;
/// Encodes and attempts to write a negative small integer value as a negative fixnum into the
/// given write.
///
/// According to the MessagePack specification, a negative fixed integer value is represented using
/// a single byte in `[0xe0; 0xff]` range inclusively, prepended with a special marker mask.
///
/// The function is **strict** with the input arguments - it is the user's responsibility to check
/// if the value fits in the described range, otherwise it will panic.
///
/// If you are not sure if the value fits in the given range use `write_sint` instead, which
/// automatically selects the most compact integer representation.
///
/// # Errors
///
/// This function will return `FixedValueWriteError` on any I/O error occurred while writing the
/// positive integer marker.
///
/// # Panics
///
/// Panics if `val` does not fit in `[-32; 0)` range.
#[inline]
#[track_caller]
pub fn write_nfix<W: RmpWrite>(wr: &mut W, val: i8) -> Result<(), W::Error> {
assert!(-32 <= val && val < 0);
write_marker(wr, Marker::FixNeg(val)).map_err(|e| e.0)?;
Ok(())
}
/// Encodes and attempts to write an `i8` value as a 2-byte sequence into the given write.
///
/// The first byte becomes the marker and the second one will represent the data itself.
///
/// Note, that this function will encode the given value in 2-byte sequence no matter what, even if
/// the value can be represented using single byte as a fixnum. Also note, that the first byte will
/// always be the i8 marker (`0xd0`).
///
/// If you need to fit the given buffer efficiently use `write_sint` instead, which automatically
/// selects the appropriate integer representation.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
///
/// # Examples
///
/// ```
/// let mut buf = [0x00, 0x00];
///
/// rmp::encode::write_i8(&mut &mut buf[..], 42).ok().unwrap();
/// assert_eq!([0xd0, 0x2a], buf);
///
/// // Note, that -18 can be represented simply as `[0xee]`, but the function emits 2-byte sequence.
/// rmp::encode::write_i8(&mut &mut buf[..], -18).ok().unwrap();
/// assert_eq!([0xd0, 0xee], buf);
/// ```
pub fn write_i8<W: RmpWrite>(wr: &mut W, val: i8) -> Result<(), ValueWriteError<W::Error>> {
write_marker(wr, Marker::I8)?;
wr.write_data_i8(val)?;
Ok(())
}
/// Encodes and attempts to write an `i16` value as a 3-byte sequence into the given write.
///
/// The first byte becomes the marker and the others will represent the data itself.
///
/// Note, that this function will encode the given value in 3-byte sequence no matter what, even if
/// the value can be represented using single byte as a fixnum. Also note, that the first byte will
/// always be the i16 marker (`0xd1`).
///
/// If you need to fit the given buffer efficiently use `write_sint` instead, which automatically
/// selects the appropriate integer representation.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_i16<W: RmpWrite>(wr: &mut W, val: i16) -> Result<(), ValueWriteError<W::Error>> {
write_marker(wr, Marker::I16)?;
wr.write_data_i16(val)?;
Ok(())
}
/// Encodes and attempts to write an `i32` value as a 5-byte sequence into the given write.
///
/// The first byte becomes the marker and the others will represent the data itself.
///
/// Note, that this function will encode the given value in 5-byte sequence no matter what, even if
/// the value can be represented using single byte as a fixnum. Also note, that the first byte will
/// always be the i32 marker (`0xd2`).
///
/// If you need to fit the given buffer efficiently use `write_sint` instead, which automatically
/// selects the appropriate integer representation.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_i32<W: RmpWrite>(wr: &mut W, val: i32) -> Result<(), ValueWriteError<W::Error>> {
write_marker(wr, Marker::I32)?;
wr.write_data_i32(val)?;
Ok(())
}
/// Encodes and attempts to write an `i64` value as a 9-byte sequence into the given write.
///
/// The first byte becomes the marker and the others will represent the data itself.
///
/// Note, that this function will encode the given value in 9-byte sequence no matter what, even if
/// the value can be represented using single byte as a fixnum. Also note, that the first byte will
/// always be the i16 marker (`0xd3`).
///
/// If you need to fit the given buffer efficiently use `write_sint` instead, which automatically
/// selects the appropriate integer representation.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_i64<W: RmpWrite>(wr: &mut W, val: i64) -> Result<(), ValueWriteError<W::Error>> {
write_marker(wr, Marker::I64)?;
wr.write_data_i64(val)?;
Ok(())
}
/// Encodes and attempts to write an `i64` value into the given write using the most efficient
/// representation, returning the marker used.
///
/// This function obeys the MessagePack specification, which requires that the serializer SHOULD use
/// the format which represents the data in the smallest number of bytes, with the exception of
/// sized/unsized types.
///
/// Note, that the function will **always** use signed integer representation even if the value can
/// be more efficiently represented using unsigned integer encoding.
///
/// The first byte becomes the marker and the others (if present, up to 9) will represent the data
/// itself.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_sint<W: RmpWrite>(wr: &mut W, val: i64) -> Result<Marker, ValueWriteError<W::Error>> {
match val {
val if -32 <= val && val < 0 => {
write_nfix(wr, val as i8)
.and(Ok(Marker::FixNeg(val as i8)))
.map_err(ValueWriteError::InvalidMarkerWrite)
}
val if -128 <= val && val < -32 => write_i8(wr, val as i8).and(Ok(Marker::I8)),
val if -32768 <= val && val < -128 => write_i16(wr, val as i16).and(Ok(Marker::I16)),
val if -2147483648 <= val && val < -32768 => write_i32(wr, val as i32).and(Ok(Marker::I32)),
val if val < -2147483648 => write_i64(wr, val).and(Ok(Marker::I64)),
val if 0 <= val && val < 128 => {
write_pfix(wr, val as u8)
.and(Ok(Marker::FixPos(val as u8)))
.map_err(ValueWriteError::InvalidMarkerWrite)
}
val if val < 256 => write_u8(wr, val as u8).and(Ok(Marker::U8)),
val if val < 65536 => write_u16(wr, val as u16).and(Ok(Marker::U16)),
val if val < 4294967296 => write_u32(wr, val as u32).and(Ok(Marker::U32)),
val => write_u64(wr, val as u64).and(Ok(Marker::U64)),
}
}

47
third_party/rust/rmp/src/encode/str.rs vendored Normal file
View File

@@ -0,0 +1,47 @@
use super::{write_marker, RmpWrite};
use crate::encode::ValueWriteError;
use crate::Marker;
/// Encodes and attempts to write the most efficient string length implementation to the given
/// write, returning the marker used.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_str_len<W: RmpWrite>(wr: &mut W, len: u32) -> Result<Marker, ValueWriteError<W::Error>> {
let marker = if len < 32 {
Marker::FixStr(len as u8)
} else if len < 256 {
Marker::Str8
} else if len <= u16::MAX as u32 {
Marker::Str16
} else {
Marker::Str32
};
write_marker(wr, marker)?;
if marker == Marker::Str8 {
wr.write_data_u8(len as u8)?;
}
if marker == Marker::Str16 {
wr.write_data_u16(len as u16)?;
}
if marker == Marker::Str32 {
wr.write_data_u32(len)?;
}
Ok(marker)
}
/// Encodes and attempts to write the most efficient string binary representation to the
/// given `Write`.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
// TODO: Docs, range check, example, visibility.
pub fn write_str<W: RmpWrite>(wr: &mut W, data: &str) -> Result<(), ValueWriteError<W::Error>> {
write_str_len(wr, data.len() as u32)?;
wr.write_bytes(data.as_bytes()).map_err(ValueWriteError::InvalidDataWrite)
}

161
third_party/rust/rmp/src/encode/uint.rs vendored Normal file
View File

@@ -0,0 +1,161 @@
use super::{write_marker, RmpWrite};
use crate::encode::ValueWriteError;
use crate::Marker;
/// Encodes and attempts to write an unsigned small integer value as a positive fixint into the
/// given write.
///
/// According to the MessagePack specification, a positive fixed integer value is represented using
/// a single byte in `[0x00; 0x7f]` range inclusively, prepended with a special marker mask.
///
/// The function is **strict** with the input arguments - it is the user's responsibility to check
/// if the value fits in the described range, otherwise it will panic.
///
/// If you are not sure if the value fits in the given range use `write_uint` instead, which
/// automatically selects the most compact integer representation.
///
/// # Errors
///
/// This function will return `FixedValueWriteError` on any I/O error occurred while writing the
/// positive integer marker.
///
/// # Panics
///
/// Panics if `val` is greater than 127.
#[inline]
pub fn write_pfix<W: RmpWrite>(wr: &mut W, val: u8) -> Result<(), W::Error> {
assert!(val < 128);
write_marker(wr, Marker::FixPos(val)).map_err(|e| e.0)?;
Ok(())
}
/// Encodes and attempts to write an `u8` value as a 2-byte sequence into the given write.
///
/// The first byte becomes the marker and the second one will represent the data itself.
///
/// Note, that this function will encode the given value in 2-byte sequence no matter what, even if
/// the value can be represented using single byte as a positive fixnum.
///
/// If you need to fit the given buffer efficiently use `write_uint` instead, which automatically
/// selects the appropriate integer representation.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
///
/// # Examples
/// ```
/// let mut buf = [0x00, 0x00];
///
/// rmp::encode::write_u8(&mut &mut buf[..], 146).ok().unwrap();
/// assert_eq!([0xcc, 0x92], buf);
///
/// // Note, that 42 can be represented simply as `[0x2a]`, but the function emits 2-byte sequence.
/// rmp::encode::write_u8(&mut &mut buf[..], 42).ok().unwrap();
/// assert_eq!([0xcc, 0x2a], buf);
/// ```
pub fn write_u8<W: RmpWrite>(wr: &mut W, val: u8) -> Result<(), ValueWriteError<W::Error>> {
write_marker(wr, Marker::U8)?;
wr.write_data_u8(val)?;
Ok(())
}
/// Encodes and attempts to write an `u16` value strictly as a 3-byte sequence into the given write.
///
/// The first byte becomes the marker and the others will represent the data itself.
///
/// Note, that this function will encode the given value in 3-byte sequence no matter what, even if
/// the value can be represented using single byte as a positive fixnum.
///
/// If you need to fit the given buffer efficiently use `write_uint` instead, which automatically
/// selects the appropriate integer representation.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_u16<W: RmpWrite>(wr: &mut W, val: u16) -> Result<(), ValueWriteError<W::Error>> {
write_marker(wr, Marker::U16)?;
wr.write_data_u16(val)?;
Ok(())
}
/// Encodes and attempts to write an `u32` value strictly as a 5-byte sequence into the given write.
///
/// The first byte becomes the marker and the others will represent the data itself.
///
/// Note, that this function will encode the given value in 5-byte sequence no matter what, even if
/// the value can be represented using single byte as a positive fixnum.
///
/// If you need to fit the given buffer efficiently use `write_uint` instead, which automatically
/// selects the appropriate integer representation.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_u32<W: RmpWrite>(wr: &mut W, val: u32) -> Result<(), ValueWriteError<W::Error>> {
write_marker(wr, Marker::U32)?;
wr.write_data_u32(val)?;
Ok(())
}
/// Encodes and attempts to write an `u64` value strictly as a 9-byte sequence into the given write.
///
/// The first byte becomes the marker and the others will represent the data itself.
///
/// Note, that this function will encode the given value in 9-byte sequence no matter what, even if
/// the value can be represented using single byte as a positive fixnum.
///
/// If you need to fit the given buffer efficiently use `write_uint` instead, which automatically
/// selects the appropriate integer representation.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_u64<W: RmpWrite>(wr: &mut W, val: u64) -> Result<(), ValueWriteError<W::Error>> {
write_marker(wr, Marker::U64)?;
wr.write_data_u64(val)?;
Ok(())
}
/// Encodes and attempts to write an `u8` value into the given write using the most efficient
/// representation, returning the marker used.
///
/// See [`write_uint`] for more info.
pub fn write_uint8<W: RmpWrite>(wr: &mut W, val: u8) -> Result<Marker, ValueWriteError<W::Error>> {
if val < 128 {
write_pfix(wr, val)
.and(Ok(Marker::FixPos(val)))
.map_err(ValueWriteError::InvalidMarkerWrite)
} else {
write_u8(wr, val).and(Ok(Marker::U8))
}
}
/// Encodes and attempts to write an `u64` value into the given write using the most efficient
/// representation, returning the marker used.
///
/// This function obeys the MessagePack specification, which requires that the serializer SHOULD use
/// the format which represents the data in the smallest number of bytes.
///
/// The first byte becomes the marker and the others (if present, up to 9) will represent the data
/// itself.
///
/// # Errors
///
/// This function will return `ValueWriteError` on any I/O error occurred while writing either the
/// marker or the data.
pub fn write_uint<W: RmpWrite>(wr: &mut W, val: u64) -> Result<Marker, ValueWriteError<W::Error>> {
if val < 256 {
write_uint8(wr, val as u8)
} else if val < 65536 {
write_u16(wr, val as u16).and(Ok(Marker::U16))
} else if val < 4294967296 {
write_u32(wr, val as u32).and(Ok(Marker::U32))
} else {
write_u64(wr, val).and(Ok(Marker::U64))
}
}

1
third_party/rust/rmp/src/encode/vec.rs vendored Normal file
View File

@@ -0,0 +1 @@

34
third_party/rust/rmp/src/errors.rs vendored Normal file
View File

@@ -0,0 +1,34 @@
//! Internal error handling code, shared between encoding and decoding.
//!
//! This is used mainly for backwards compatibility and abstraction over std/no_std.
/// An alias to the "default" error handling type.
///
/// This is problematic because when working on `#[no_std]`, because there is no [`std::error::Error`] trait and also no [`std::io::Error`] type.
///
/// Furthermore, this doesn't abstract over the differences between different implementations of [`RmpRead`](crate::decode::RmpRead)/[`RmpWrite`](crate::encode::RmpWrite).
///
/// When working directly with bytes streams, the error type is actually [Infallible](core::convert::Infallible).
///
/// For these two reasons, this type is deprecated
#[cfg(feature = "std")]
#[deprecated(note = "Doesn't abstract over RmpRead/RmpWrite (or work on no_std), use RmpRead::Error/RmpWrite::Error and RmpReadErr/RmpWriteErr instead")]
pub type Error = ::std::io::Error;
#[cfg(not(feature = "std"))]
#[deprecated(note = "Doesn't work meaningfully on no_std")]
pub type Error = ::core::convert::Infallible;
/// Internal type used to abstract over the [`std::error::Error`] trait
///
/// This is a nop in no-std environments.
#[cfg(feature = "std")]
#[doc(hidden)]
pub trait MaybeErrBound: std::error::Error {}
#[cfg(feature = "std")]
impl<T: ?Sized + std::error::Error> MaybeErrBound for T {}
#[cfg(not(feature = "std"))]
#[doc(hidden)]
pub trait MaybeErrBound {}
#[cfg(not(feature = "std"))]
impl<T: ?Sized> MaybeErrBound for T {}

14
third_party/rust/rmp/src/lib.rs vendored Normal file
View File

@@ -0,0 +1,14 @@
#![doc = include_str!("../README.md")]
#![cfg_attr(not(feature = "std"), no_std)]
extern crate alloc;
pub mod decode;
pub mod encode;
mod errors;
mod marker;
pub use crate::marker::Marker;
/// Version of the MessagePack [spec](http://github.com/msgpack/msgpack/blob/master/spec.md).
pub const MSGPACK_VERSION: u32 = 5;

165
third_party/rust/rmp/src/marker.rs vendored Normal file
View File

@@ -0,0 +1,165 @@
const FIXSTR_SIZE : u8 = 0x1f;
const FIXARRAY_SIZE : u8 = 0x0f;
const FIXMAP_SIZE : u8 = 0x0f;
/// Format markers.
#[derive(Clone, Copy, Debug, PartialEq)]
#[repr(u8)]
pub enum Marker {
FixPos(u8) = 0x00,
FixNeg(i8) = 0xe0,
FixMap(u8) = 0x80,
FixArray(u8) = 0x90,
FixStr(u8) = 0xa0,
Null = 0xc0,
// Marked in MessagePack spec as never used.
Reserved,
False,
True,
Bin8,
Bin16,
Bin32,
Ext8,
Ext16,
Ext32,
F32,
F64,
U8,
U16,
U32,
U64,
I8,
I16,
I32,
I64,
FixExt1,
FixExt2,
FixExt4,
FixExt8,
FixExt16,
Str8,
Str16,
Str32,
Array16,
Array32,
Map16,
Map32,
}
impl Marker {
/// Construct a msgpack marker from a single byte.
#[must_use]
#[inline]
pub fn from_u8(n: u8) -> Marker {
match n {
0x00 ..= 0x7f => Marker::FixPos(n),
0xe0 ..= 0xff => Marker::FixNeg(n as i8),
0x80 ..= 0x8f => Marker::FixMap(n & FIXMAP_SIZE),
0x90 ..= 0x9f => Marker::FixArray(n & FIXARRAY_SIZE),
0xa0 ..= 0xbf => Marker::FixStr(n & FIXSTR_SIZE),
0xc0 => Marker::Null,
// Marked in MessagePack spec as never used.
0xc1 => Marker::Reserved,
0xc2 => Marker::False,
0xc3 => Marker::True,
0xc4 => Marker::Bin8,
0xc5 => Marker::Bin16,
0xc6 => Marker::Bin32,
0xc7 => Marker::Ext8,
0xc8 => Marker::Ext16,
0xc9 => Marker::Ext32,
0xca => Marker::F32,
0xcb => Marker::F64,
0xcc => Marker::U8,
0xcd => Marker::U16,
0xce => Marker::U32,
0xcf => Marker::U64,
0xd0 => Marker::I8,
0xd1 => Marker::I16,
0xd2 => Marker::I32,
0xd3 => Marker::I64,
0xd4 => Marker::FixExt1,
0xd5 => Marker::FixExt2,
0xd6 => Marker::FixExt4,
0xd7 => Marker::FixExt8,
0xd8 => Marker::FixExt16,
0xd9 => Marker::Str8,
0xda => Marker::Str16,
0xdb => Marker::Str32,
0xdc => Marker::Array16,
0xdd => Marker::Array32,
0xde => Marker::Map16,
0xdf => Marker::Map32,
}
}
/// Converts a marker object into a single-byte representation.
#[must_use]
#[inline]
pub fn to_u8(&self) -> u8 {
match *self {
Marker::FixPos(val) => val,
Marker::FixNeg(val) => val as u8,
Marker::Null => 0xc0,
Marker::True => 0xc3,
Marker::False => 0xc2,
Marker::U8 => 0xcc,
Marker::U16 => 0xcd,
Marker::U32 => 0xce,
Marker::U64 => 0xcf,
Marker::I8 => 0xd0,
Marker::I16 => 0xd1,
Marker::I32 => 0xd2,
Marker::I64 => 0xd3,
Marker::F32 => 0xca,
Marker::F64 => 0xcb,
Marker::FixStr(len) => 0xa0 | (len & FIXSTR_SIZE),
Marker::Str8 => 0xd9,
Marker::Str16 => 0xda,
Marker::Str32 => 0xdb,
Marker::Bin8 => 0xc4,
Marker::Bin16 => 0xc5,
Marker::Bin32 => 0xc6,
Marker::FixArray(len) => 0x90 | (len & FIXARRAY_SIZE),
Marker::Array16 => 0xdc,
Marker::Array32 => 0xdd,
Marker::FixMap(len) => 0x80 | (len & FIXMAP_SIZE),
Marker::Map16 => 0xde,
Marker::Map32 => 0xdf,
Marker::FixExt1 => 0xd4,
Marker::FixExt2 => 0xd5,
Marker::FixExt4 => 0xd6,
Marker::FixExt8 => 0xd7,
Marker::FixExt16 => 0xd8,
Marker::Ext8 => 0xc7,
Marker::Ext16 => 0xc8,
Marker::Ext32 => 0xc9,
Marker::Reserved => 0xc1,
}
}
}
impl From<u8> for Marker {
#[inline(always)]
fn from(val: u8) -> Marker {
Marker::from_u8(val)
}
}
impl From<Marker> for u8 {
#[inline(always)]
fn from(val: Marker) -> Self {
val.to_u8()
}
}

91
third_party/rust/rmp/tests/func/decode/array.rs vendored Normal file
View File

@@ -0,0 +1,91 @@
use super::Cursor;
use rmp::decode::*;
use rmp::Marker;
#[test]
fn from_empty_array_read_size() {
let buf: &[u8] = &[0x90];
let mut cur = Cursor::new(buf);
assert_eq!(0, read_array_len(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_fixarray_max_read_size() {
let buf: &[u8] = &[
0x9f,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e
];
let mut cur = Cursor::new(buf);
assert_eq!(15, read_array_len(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_array16_min_read_size() {
let buf: &[u8] = &[0xdc, 0x00, 0x10];
let mut cur = Cursor::new(buf);
assert_eq!(16, read_array_len(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_array16_max_read_size() {
let buf: &[u8] = &[0xdc, 0xff, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(65535, read_array_len(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_array16_unexpected_eof_read_size() {
let buf: &[u8] = &[0xdc, 0xff];
let mut cur = Cursor::new(buf);
read_array_len(&mut cur).err().unwrap();
assert!(cur.position() >= 1);
}
#[test]
fn from_array32_min_read_size() {
let buf: &[u8] = &[0xdd, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(0, read_array_len(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_array32_max_read_size() {
let buf: &[u8] = &[0xdd, 0xff, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(4294967295, read_array_len(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_array32_unexpected_eof_read_size() {
let buf: &[u8] = &[0xdd, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(buf);
read_array_len(&mut cur).err().unwrap();
assert!(cur.position() >= 1);
}
#[test]
fn from_null_read_array_len() {
let buf: &[u8] = &[0xc0];
let mut cur = Cursor::new(buf);
match read_array_len(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}

61
third_party/rust/rmp/tests/func/decode/bin.rs vendored Normal file
View File

@@ -0,0 +1,61 @@
use super::Cursor;
use rmp::decode::*;
use rmp::Marker;
#[test]
fn from_bin8_min_read_len() {
let buf: &[u8] = &[0xc4, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(0, read_bin_len(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_bin8_max_read_len() {
let buf: &[u8] = &[0xc4, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(255, read_bin_len(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_bin8_eof_read_len() {
let buf: &[u8] = &[0xc4];
let mut cur = Cursor::new(buf);
read_bin_len(&mut cur).err().unwrap();
assert_eq!(1, cur.position());
}
#[test]
fn from_null_read_len() {
let buf: &[u8] = &[0xc0];
let mut cur = Cursor::new(buf);
match read_bin_len(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn from_bin16_max_read_len() {
let buf: &[u8] = &[0xc5, 0xff, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(65535, read_bin_len(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_bin32_max_read_len() {
let buf: &[u8] = &[0xc6, 0xff, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(4294967295, read_bin_len(&mut cur).unwrap());
assert_eq!(5, cur.position());
}

21
third_party/rust/rmp/tests/func/decode/bool.rs vendored Normal file
View File

@@ -0,0 +1,21 @@
use super::Cursor;
use rmp::decode::*;
#[test]
fn from_bool_false() {
let buf = [0xc2];
let mut cur = Cursor::new(&buf[..]);
assert!(!read_bool(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_bool_true() {
let buf = [0xc3];
let mut cur = Cursor::new(&buf[..]);
assert!(read_bool(&mut cur).unwrap());
assert_eq!(1, cur.position());
}

128
third_party/rust/rmp/tests/func/decode/ext.rs vendored Normal file
View File

@@ -0,0 +1,128 @@
use super::Cursor;
use rmp::decode::*;
#[test]
fn from_fixext1_read_fixext1() {
let buf: &[u8] = &[0xd4, 0x01, 0x02];
let mut cur = Cursor::new(buf);
assert_eq!((1, 2), read_fixext1(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_fixext2_read_fixext2() {
let buf = [0xd5, 0x01, 0x00, 0x02];
let mut cur = Cursor::new(&buf[..]);
assert_eq!((1, [0x00, 0x02]), read_fixext2(&mut cur).unwrap());
assert_eq!(4, cur.position());
}
#[test]
fn from_fixext4_read_fixext4() {
let buf = [0xd6, 0x01, 0x00, 0x00, 0x00, 0x02];
let mut cur = Cursor::new(&buf[..]);
assert_eq!((1, [0x00, 0x00, 0x00, 0x02]), read_fixext4(&mut cur).unwrap());
assert_eq!(6, cur.position());
}
#[test]
fn from_fixext8_read_fixext8() {
let buf = [0xd7, 0x01, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08];
let mut cur = Cursor::new(&buf[..]);
assert_eq!((1, [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]),
read_fixext8(&mut cur).unwrap());
assert_eq!(10, cur.position());
}
#[test]
fn from_fixext16_read_fixext16() {
let buf = [
0xd8,
0x01,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08
];
let mut cur = Cursor::new(&buf[..]);
assert_eq!((1, [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]),
read_fixext16(&mut cur).unwrap());
assert_eq!(18, cur.position());
}
#[test]
fn from_fixext1_read_ext_meta() {
let buf: &[u8] = &[0xd4, 0x01];
let mut cur = Cursor::new(buf);
assert_eq!(ExtMeta { typeid: 1, size: 1 }, read_ext_meta(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_fixext2_read_ext_meta() {
let buf: &[u8] = &[0xd5, 0x01];
let mut cur = Cursor::new(buf);
assert_eq!(ExtMeta { typeid: 1, size: 2 }, read_ext_meta(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_fixext4_read_ext_meta() {
let buf: &[u8] = &[0xd6, 0x01];
let mut cur = Cursor::new(buf);
assert_eq!(ExtMeta { typeid: 1, size: 4 }, read_ext_meta(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_fixext8_read_ext_meta() {
let buf: &[u8] = &[0xd7, 0x01];
let mut cur = Cursor::new(buf);
assert_eq!(ExtMeta { typeid: 1, size: 8 }, read_ext_meta(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_fixext16_read_ext_meta() {
let buf: &[u8] = &[0xd8, 0x01];
let mut cur = Cursor::new(buf);
assert_eq!(ExtMeta { typeid: 1, size: 16 }, read_ext_meta(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_ext8_read_ext_meta() {
let buf: &[u8] = &[0xc7, 0xff, 0x01];
let mut cur = Cursor::new(buf);
assert_eq!(ExtMeta { typeid: 1, size: 255 }, read_ext_meta(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_ext16_read_ext_meta() {
let buf: &[u8] = &[0xc8, 0xff, 0xff, 0x01];
let mut cur = Cursor::new(buf);
assert_eq!(ExtMeta { typeid: 1, size: 65535 }, read_ext_meta(&mut cur).unwrap());
assert_eq!(4, cur.position());
}
#[test]
fn from_ext32_read_ext_meta() {
let buf: &[u8] = &[0xc9, 0xff, 0xff, 0xff, 0xff, 0x01];
let mut cur = Cursor::new(buf);
assert_eq!(ExtMeta { typeid: 1, size: 4294967295 }, read_ext_meta(&mut cur).unwrap());
assert_eq!(6, cur.position());
}

108
third_party/rust/rmp/tests/func/decode/float.rs vendored Normal file
View File

@@ -0,0 +1,108 @@
use super::Cursor;
use rmp::decode::*;
use rmp::Marker;
#[test]
fn from_f32_zero_plus() {
let buf: &[u8] = &[0xca, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(0.0, read_f32(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_f32_max() {
let buf: &[u8] = &[0xca, 0x7f, 0x7f, 0xff, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(3.4028234e38_f32, read_f32(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_f32_inf() {
use std::f32;
let buf: &[u8] = &[0xca, 0x7f, 0x80, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(f32::INFINITY, read_f32(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_f32_neg_inf() {
use std::f32;
let buf: &[u8] = &[0xca, 0xff, 0x80, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(f32::NEG_INFINITY, read_f32(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_null_read_f32() {
let buf: &[u8] = &[0xc0];
let mut cur = Cursor::new(buf);
match read_f32(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn from_f64_zero_plus() {
let buf: &[u8] = &[0xcb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(0.0, read_f64(&mut cur).unwrap());
assert_eq!(9, cur.position());
}
#[test]
fn from_f64_zero_minus() {
let buf: &[u8] = &[0xcb, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(-0.0, read_f64(&mut cur).unwrap());
assert_eq!(9, cur.position());
}
#[test]
fn from_f64_inf() {
use std::f64;
let buf: &[u8] = &[0xcb, 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(f64::INFINITY, read_f64(&mut cur).unwrap());
assert_eq!(9, cur.position());
}
#[test]
fn from_f64_neg_inf() {
use std::f64;
let buf: &[u8] = &[0xcb, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(f64::NEG_INFINITY, read_f64(&mut cur).unwrap());
assert_eq!(9, cur.position());
}
#[test]
fn from_null_read_f64() {
let buf: &[u8] = &[0xc0];
let mut cur = Cursor::new(buf);
match read_f64(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}

61
third_party/rust/rmp/tests/func/decode/map.rs vendored Normal file
View File

@@ -0,0 +1,61 @@
use super::Cursor;
use rmp::decode::*;
use rmp::Marker;
#[test]
fn from_fixmap_min_read_size() {
let buf: &[u8] = &[0x80];
let mut cur = Cursor::new(buf);
assert_eq!(0, read_map_len(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_fixmap_max_read_size() {
let buf: &[u8] = &[0x8f];
let mut cur = Cursor::new(buf);
assert_eq!(15, read_map_len(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_map16_min_read_size() {
let buf: &[u8] = &[0xde, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(0, read_map_len(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_map16_max_read_size() {
let buf: &[u8] = &[0xde, 0xff, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(65535, read_map_len(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_map32_min_read_size() {
let buf: &[u8] = &[0xdf, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(0, read_map_len(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_null_read_map_len() {
let buf: &[u8] = &[0xc0, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
match read_map_len(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}

15
third_party/rust/rmp/tests/func/decode/mod.rs vendored Normal file
View File

@@ -0,0 +1,15 @@
mod array;
mod bin;
mod bool;
mod ext;
mod float;
mod map;
mod null;
mod sint;
mod string;
mod uint;
#[cfg(feature = "std")]
pub type Cursor<'a> = std::io::Cursor<&'a [u8]>;
#[cfg(not(feature = "std"))]
pub type Cursor<'a> = rmp::decode::Bytes<'a>;

66
third_party/rust/rmp/tests/func/decode/null.rs vendored Normal file
View File

@@ -0,0 +1,66 @@
use super::Cursor;
use rmp::decode::*;
#[test]
fn pass() {
let buf = [0xc0];
let mut cur = Cursor::new(&buf[..]);
assert_eq!((), read_nil(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn fail_invalid_marker() {
let buf = [0xc1];
let mut cur = Cursor::new(&buf[..]);
match read_nil(&mut cur) {
Err(ValueReadError::TypeMismatch(..)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn fail_unexpected_eof() {
let buf = [];
let mut cur = Cursor::new(&buf[..]);
read_nil(&mut cur).err().unwrap();
assert_eq!(0, cur.position());
}
#[test]
#[cfg(feature = "std")]
fn interrupt_safe() {
use std::io::{Error, ErrorKind, Read};
struct MockRead { state_: u8 }
impl MockRead {
fn state(&self) -> u8 { self.state_ }
}
impl Read for MockRead {
fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
if self.state_ == 0 {
self.state_ = 1;
Err(Error::new(ErrorKind::Interrupted, "interrupted"))
} else {
assert!(!buf.is_empty());
buf[0] = 0xc0;
Ok(1)
}
}
}
let mut cur = MockRead { state_: 0 };
// The function is interruption-safe, the first read should succeed.
read_nil(&mut cur).unwrap();
assert_eq!(1, cur.state());
}

280
third_party/rust/rmp/tests/func/decode/sint.rs vendored Normal file
View File

@@ -0,0 +1,280 @@
use super::Cursor;
use rmp::decode::*;
use rmp::Marker;
#[test]
fn from_nfix_min() {
let buf = [0xe0];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(-32, read_nfix(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_nfix_max() {
let buf = [0xff];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(-1, read_nfix(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_nfix_type_mismatch() {
let buf = &[0xc0];
let mut cur = Cursor::new(&buf[..]);
match read_nfix(&mut cur) {
Err(ValueReadError::TypeMismatch(..)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn from_i8_min() {
let buf = [0xd0, 0x80];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(-128, read_i8(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_i8_max() {
let buf = [0xd0, 0x7f];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(127, read_i8(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_i8_type_mismatch() {
let buf = [0xc0, 0x80];
let mut cur = Cursor::new(&buf[..]);
match read_i8(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn from_i8_unexpected_eof() {
let buf = [0xd0];
let mut cur = Cursor::new(&buf[..]);
read_i8(&mut cur).err().unwrap();
assert_eq!(1, cur.position());
}
#[test]
fn from_i16_min() {
let buf = [0xd1, 0x80, 0x00];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(-32768, read_i16(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_i16_max() {
let buf = [0xd1, 0x7f, 0xff];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(32767, read_i16(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_i16_type_mismatch() {
let buf = [0xc0, 0x80, 0x00];
let mut cur = Cursor::new(&buf[..]);
match read_i16(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn from_i16_unexpected_eof() {
let buf = [0xd1, 0x7f];
let mut cur = Cursor::new(&buf[..]);
read_i16(&mut cur).err().unwrap();
assert!(cur.position() >= 1);
}
#[test]
fn from_i32_min() {
let buf = [0xd2, 0x80, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(-2_147_483_648, read_i32(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_i32_max() {
let buf = &[0xd2, 0x7f, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(2147483647, read_i32(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_i32_type_mismatch() {
let buf = &[0xc0, 0x80, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(&buf[..]);
match read_i32(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn from_i32_unexpected_eof() {
let buf = &[0xd2, 0x7f, 0xff, 0xff];
let mut cur = Cursor::new(&buf[..]);
read_i32(&mut cur).err().unwrap();
assert!(cur.position() >= 1);
}
#[test]
fn from_i64_min() {
let buf = [0xd3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(-9223372036854775808, read_i64(&mut cur).unwrap());
assert_eq!(9, cur.position());
}
#[test]
fn from_i64_max() {
let buf = [0xd3, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(9223372036854775807, read_i64(&mut cur).unwrap());
assert_eq!(9, cur.position());
}
#[test]
fn from_i64_type_mismatch() {
let buf = [0xc0, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(&buf[..]);
match read_i64(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn from_i64_unexpected_eof() {
let buf = [0xd3, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(&buf[..]);
read_i64(&mut cur).err().unwrap();
assert!(cur.position() >= 1);
}
#[test]
fn from_nfix_min_read_int() {
let buf: &[u8] = &[0xe0];
let mut cur = Cursor::new(buf);
assert_eq!(-32, read_int(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_nfix_max_read_int() {
let buf: &[u8] = &[0xff];
let mut cur = Cursor::new(buf);
assert_eq!(-1, read_int(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_i8_min_read_int() {
let buf: &[u8] = &[0xd0, 0x80];
let mut cur = Cursor::new(buf);
assert_eq!(-128, read_int(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_i8_max_read_int() {
let buf: &[u8] = &[0xd0, 0x7f];
let mut cur = Cursor::new(buf);
assert_eq!(127, read_int(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_i16_min_read_int() {
let buf: &[u8] = &[0xd1, 0x80, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(-32768, read_int(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_i16_max_read_int() {
let buf: &[u8] = &[0xd1, 0x7f, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(32767, read_int(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_i32_min_read_int() {
let buf: &[u8] = &[0xd2, 0x80, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(-2147483648, read_int(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_i32_max_read_int() {
let buf: &[u8] = &[0xd2, 0x7f, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(2147483647, read_int(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_i64_min_read_int() {
let buf: &[u8] = &[0xd3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(-9223372036854775808i64, read_int(&mut cur).unwrap());
assert_eq!(9, cur.position());
}
#[test]
fn from_i64_max_read_int() {
let buf: &[u8] = &[0xd3, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(9223372036854775807i64, read_int(&mut cur).unwrap());
assert_eq!(9, cur.position());
}

242
third_party/rust/rmp/tests/func/decode/string.rs vendored Normal file
View File

@@ -0,0 +1,242 @@
use super::Cursor;
use rmp::decode::*;
use rmp::Marker;
#[test]
fn from_fixstr_min_read_str_len() {
let buf: &[u8] = &[0xa0];
let mut cur = Cursor::new(buf);
assert_eq!(0, read_str_len(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_fixstr_rnd_read_str_len() {
let buf: &[u8] = &[0xaa];
let mut cur = Cursor::new(buf);
assert_eq!(10, read_str_len(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_fixstr_max_read_str_len() {
let buf: &[u8] = &[0xbf];
let mut cur = Cursor::new(buf);
assert_eq!(31, read_str_len(&mut cur).unwrap());
assert_eq!(1, cur.position());
}
#[test]
fn from_str8_min_read_str_len() {
let buf: &[u8] = &[0xd9, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(0, read_str_len(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_str8_rnd_read_str_len() {
let buf: &[u8] = &[0xd9, 0x0a];
let mut cur = Cursor::new(buf);
assert_eq!(10, read_str_len(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_str8_read_str_len_eof() {
let buf: &[u8] = &[0xd9];
let mut cur = Cursor::new(buf);
read_str_len(&mut cur).err().unwrap();
assert_eq!(1, cur.position());
}
#[test]
fn from_str8_max_read_str_len() {
let buf: &[u8] = &[0xd9, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(255, read_str_len(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_str16_min_read_str_len() {
let buf: &[u8] = &[0xda, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(0, read_str_len(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_str16_max_read_str_len() {
let buf: &[u8] = &[0xda, 0xff, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(65535, read_str_len(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_str16_read_str_len_eof() {
let buf: &[u8] = &[0xda, 0x00];
let mut cur = Cursor::new(buf);
read_str_len(&mut cur).err().unwrap();
assert!(cur.position() >= 1);
}
#[test]
fn from_str32_min_read_str_len() {
let buf: &[u8] = &[0xdb, 0x00, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
assert_eq!(0, read_str_len(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_str32_max_read_str_len() {
let buf: &[u8] = &[0xdb, 0xff, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(buf);
assert_eq!(4294967295, read_str_len(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_str32_read_str_len_eof() {
let buf: &[u8] = &[0xdb, 0x00, 0x00, 0x00];
let mut cur = Cursor::new(buf);
read_str_len(&mut cur).err().unwrap();
assert!(cur.position() >= 1);
}
#[test]
fn from_null_read_str_len() {
let buf: &[u8] = &[0xc0];
let mut cur = Cursor::new(buf);
match read_str_len(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn from_str_strfix() {
let buf: &[u8] = &[0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65];
let mut cur = Cursor::new(buf);
let out: &mut [u8] = &mut [0u8; 16];
assert_eq!("le message", read_str(&mut cur, out).unwrap());
assert_eq!(11, cur.position());
}
#[test]
fn from_str_strfix_extra_data() {
let buf: &[u8] = &[0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x00];
let mut cur = Cursor::new(buf);
let out: &mut [u8] = &mut [0u8; 16];
assert_eq!("le message", read_str(&mut cur, out).unwrap());
assert_eq!(11, cur.position());
}
#[test]
fn from_str_strfix_exact_buffer() {
let buf: &[u8] = &[0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65];
let mut cur = Cursor::new(buf);
let out: &mut [u8] = &mut [0u8; 10];
assert_eq!("le message", read_str(&mut cur, out).unwrap());
assert_eq!(11, cur.position());
}
#[test]
fn from_str_strfix_invalid_utf8() {
// Invalid 2 Octet Sequence.
let buf: &[u8] = &[0xa2, 0xc3, 0x28];
let mut cur = Cursor::new(buf);
let out: &mut [u8] = &mut [0u8; 16];
match read_str(&mut cur, out) {
Err(DecodeStringError::InvalidUtf8(raw, _)) => {
assert_eq!(&[0xc3, 0x28], raw);
}
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(3, cur.position());
}
#[test]
fn from_str_strfix_buffer_too_small() {
let buf: &[u8] = &[0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65];
let mut cur = Cursor::new(buf);
let out: &mut [u8] = &mut [0u8; 9];
match read_str(&mut cur, out) {
Err(DecodeStringError::BufferSizeTooSmall(10)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn from_str_strfix_decode_from_slice() {
// Wrap an incomplete buffer into the Cursor to see how many bytes were consumed.
let mut buf = vec![0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73];
assert!(read_str_from_slice(&buf).is_err());
// ... complete the buffer and try to parse again.
buf.append(&mut vec![0x73, 0x61, 0x67, 0x65]);
assert_eq!(("le message", &[][..]), read_str_from_slice(&buf).unwrap());
}
#[test]
fn from_str_strfix_decode_from_slice_with_trailing_bytes() {
let buf = vec![
0xaa, 0x6c, 0x65, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x01, 0x02, 0x03,
];
assert_eq!(("le message", &[0x01, 0x02, 0x03][..]), read_str_from_slice(&buf).unwrap());
}
#[test]
fn example_process_sequence_of_strings() {
// Encoded: 'Unpacking', 'multiple', 'strings'.
let vec = [
0xa9, 0x55, 0x6e, 0x70, 0x61, 0x63, 0x6b, 0x69, 0x6e, 0x67,
0xa8, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0xa7,
0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x73
];
let mut chunks = Vec::new();
let mut unparsed = &vec[..];
loop {
match read_str_from_slice(unparsed) {
Ok((chunk, tail)) => {
chunks.push(chunk);
unparsed = tail;
}
Err(..) => break,
}
}
assert_eq!(["Unpacking", "multiple", "strings"], chunks[..]);
}

202
third_party/rust/rmp/tests/func/decode/uint.rs vendored Normal file
View File

@@ -0,0 +1,202 @@
use super::Cursor;
use rmp::decode::*;
use rmp::Marker;
#[test]
fn from_positive_fixnum() {
let buf = [0x00, 0x7f, 0x20];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(0u8, read_pfix(&mut cur).unwrap());
assert_eq!(1, cur.position());
assert_eq!(127u8, read_pfix(&mut cur).unwrap());
assert_eq!(2, cur.position());
assert_eq!(32u8, read_pfix(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_u8_min() {
let buf = [0xcc, 0x00];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(0, read_u8(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_u8_max() {
let buf = [0xcc, 0xff];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(255, read_u8(&mut cur).unwrap());
assert_eq!(2, cur.position());
}
#[test]
fn from_u8_type_mismatch() {
let buf = [0xc0, 0x80];
let mut cur = Cursor::new(&buf[..]);
match read_u8(&mut cur) {
Err(ValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}"),
}
assert_eq!(1, cur.position());
}
#[test]
fn from_u8_unexpected_eof() {
let buf = [0xcc];
let mut cur = Cursor::new(&buf[..]);
read_u8(&mut cur).err().unwrap();
assert_eq!(1, cur.position());
}
#[test]
fn from_u16_min() {
let buf = [0xcd, 0x00, 0x00];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(0, read_u16(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_u32_max() {
let buf = [0xce, 0xff, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(4294967295, read_u32(&mut cur).unwrap());
assert_eq!(5, cur.position());
}
#[test]
fn from_unsigned_fixnum_read_int() {
let buf = [0x00, 0x7f, 0x20];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(0u64, read_int(&mut cur).unwrap());
assert_eq!(1, cur.position());
assert_eq!(127u64, read_int(&mut cur).unwrap());
assert_eq!(2, cur.position());
assert_eq!(32u64, read_int(&mut cur).unwrap());
assert_eq!(3, cur.position());
}
#[test]
fn from_unsigned_u8_read_int() {
let buf = [0xcc, 0x80, 0xcc, 0xff];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(128u64, read_int(&mut cur).unwrap());
assert_eq!(2, cur.position());
assert_eq!(255u64, read_int(&mut cur).unwrap());
assert_eq!(4, cur.position());
}
#[test]
fn from_unsigned_u8_incomplete_read_int() {
let buf = [0xcc];
let mut cur = Cursor::new(&buf[..]);
read_int::<u64, _>(&mut cur).err().unwrap();
assert_eq!(1, cur.position());
}
#[test]
fn from_unsigned_u16_read_int() {
let buf = [0xcd, 0x01, 0x00, 0xcd, 0xff, 0xff];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(256u64, read_int(&mut cur).unwrap());
assert_eq!(3, cur.position());
assert_eq!(65535u64, read_int(&mut cur).unwrap());
assert_eq!(6, cur.position());
}
#[test]
fn from_unsigned_u16_incomplete_read_int() {
let buf = [0xcd];
let mut cur = Cursor::new(&buf[..]);
read_int::<u64, _>(&mut cur).err().unwrap();
assert_eq!(1, cur.position());
}
#[test]
fn from_unsigned_u32_read_int() {
let buf = [0xce, 0x00, 0x01, 0x00, 0x00, 0xce, 0xff, 0xff, 0xff, 0xff];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(65536u64, read_int(&mut cur).unwrap());
assert_eq!(5, cur.position());
assert_eq!(4294967295u64, read_int(&mut cur).unwrap());
assert_eq!(10, cur.position());
}
#[test]
fn from_unsigned_u32_incomplete_read_int() {
let buf = [0xce];
let mut cur = Cursor::new(&buf[..]);
read_int::<u64, _>(&mut cur).err().unwrap();
assert_eq!(1, cur.position());
}
#[test]
fn from_unsigned_u64_read_int() {
let buf = [
0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
];
let mut cur = Cursor::new(&buf[..]);
assert_eq!(4294967296u64, read_int(&mut cur).unwrap());
assert_eq!(9, cur.position());
assert_eq!(18446744073709551615u64, read_int(&mut cur).unwrap());
assert_eq!(18, cur.position());
}
#[test]
fn from_unsigned_u64_incomplete_read_int() {
let buf = [0xcf];
let mut cur = Cursor::new(&buf[..]);
read_int::<u64, _>(&mut cur).err().unwrap();
assert_eq!(1, cur.position());
}
#[test]
fn from_unsigned_invalid_marker_read_int() {
let buf = [0xc0];
let mut cur = Cursor::new(&buf[..]);
match read_int::<u64, _>(&mut cur) {
Err(NumValueReadError::TypeMismatch(Marker::Null)) => (),
other => panic!("unexpected result: {other:?}")
}
assert_eq!(1, cur.position());
}
#[test]
fn from_unsigned_invalid_unknown_marker_read_int() {
let buf = [0xc1];
let mut cur = Cursor::new(&buf[..]);
match read_int::<u64, _>(&mut cur) {
Err(NumValueReadError::TypeMismatch(Marker::Reserved)) => (),
other => panic!("unexpected result: {other:?}")
}
assert_eq!(1, cur.position());
}

29
third_party/rust/rmp/tests/func/encode/array.rs vendored Normal file
View File

@@ -0,0 +1,29 @@
use rmp::encode::*;
use rmp::Marker;
#[test]
fn pass_pack_len_fix() {
let mut buf = [0x00];
assert_eq!(Marker::FixArray(15), write_array_len(&mut &mut buf[..], 15).unwrap());
assert_eq!([0x9f], buf);
}
#[test]
fn pass_pack_len_u16() {
let mut buf = [0x00, 0x00, 0x00];
assert_eq!(Marker::Array16, write_array_len(&mut &mut buf[..], 65535).unwrap());
assert_eq!([0xdc, 0xff, 0xff], buf);
}
#[test]
fn pass_pack_len_u32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::Array32, write_array_len(&mut &mut buf[..], 4294967295).unwrap());
assert_eq!([0xdd, 0xff, 0xff, 0xff, 0xff], buf);
}

29
third_party/rust/rmp/tests/func/encode/bin.rs vendored Normal file
View File

@@ -0,0 +1,29 @@
use rmp::encode::*;
use rmp::Marker;
#[test]
fn pass_pack_len_u8() {
let mut buf = [0x00, 0x00];
assert_eq!(Marker::Bin8, write_bin_len(&mut &mut buf[..], 255).unwrap());
assert_eq!([0xc4, 0xff], buf);
}
#[test]
fn pass_pack_len_u16() {
let mut buf = [0x00, 0x00, 0x00];
assert_eq!(Marker::Bin16, write_bin_len(&mut &mut buf[..], 65535).unwrap());
assert_eq!([0xc5, 0xff, 0xff], buf);
}
#[test]
fn pass_pack_len_u32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::Bin32, write_bin_len(&mut &mut buf[..], 4294967295).unwrap());
assert_eq!([0xc6, 0xff, 0xff, 0xff, 0xff], buf);
}

19
third_party/rust/rmp/tests/func/encode/bool.rs vendored Normal file
View File

@@ -0,0 +1,19 @@
use rmp::encode::*;
#[test]
fn pass_pack_true() {
let mut buf = [0x00];
write_bool(&mut &mut buf[..], true).unwrap();
assert_eq!([0xc3], buf);
}
#[test]
fn pass_pack_false() {
let mut buf = [0x00];
write_bool(&mut &mut buf[..], false).unwrap();
assert_eq!([0xc2], buf);
}

81
third_party/rust/rmp/tests/func/encode/ext.rs vendored Normal file
View File

@@ -0,0 +1,81 @@
use rmp::encode::*;
use rmp::Marker;
#[test]
fn pass_pack_meta_fix1() {
let mut buf = [0x00, 0x00];
assert_eq!(Marker::FixExt1, write_ext_meta(&mut &mut buf[..], 1, 16).unwrap());
assert_eq!([0xd4, 0x10], buf);
}
#[test]
fn pass_pack_meta_fix2() {
let mut buf = [0x00, 0x00];
assert_eq!(Marker::FixExt2, write_ext_meta(&mut &mut buf[..], 2, 16).unwrap());
assert_eq!([0xd5, 0x10], buf);
}
#[test]
fn pass_pack_meta_fix4() {
let mut buf = [0x00, 0x00];
assert_eq!(Marker::FixExt4, write_ext_meta(&mut &mut buf[..], 4, 16).unwrap());
assert_eq!([0xd6, 0x10], buf);
}
#[test]
fn pass_pack_meta_fix4_timesamp() {
let mut buf = [0x00, 0x00];
assert_eq!(Marker::FixExt4, write_ext_meta(&mut &mut buf[..], 4, -1).unwrap());
assert_eq!([0xd6, 0xff], buf);
}
#[test]
fn pass_pack_meta_fix8() {
let mut buf = [0x00, 0x00];
assert_eq!(Marker::FixExt8, write_ext_meta(&mut &mut buf[..], 8, 16).unwrap());
assert_eq!([0xd7, 0x10], buf);
}
#[test]
fn pass_pack_meta_fix16() {
let mut buf = [0x00, 0x00];
assert_eq!(Marker::FixExt16, write_ext_meta(&mut &mut buf[..], 16, 16).unwrap());
assert_eq!([0xd8, 0x10], buf);
}
#[test]
fn pass_pack_meta_8() {
let mut buf = [0x00, 0x00, 0x00];
assert_eq!(Marker::Ext8, write_ext_meta(&mut &mut buf[..], 255, 16).unwrap());
assert_eq!([0xc7, 0xff, 0x10], buf);
}
#[test]
fn pass_pack_meta_16() {
let mut buf = [0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::Ext16, write_ext_meta(&mut &mut buf[..], 65535, 16).unwrap());
assert_eq!([0xc8, 0xff, 0xff, 0x10], buf);
}
#[test]
fn pass_pack_meta_32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::Ext32, write_ext_meta(&mut &mut buf[..], 4294967295, 16).unwrap());
assert_eq!([0xc9, 0xff, 0xff, 0xff, 0xff, 0x10], buf);
}

21
third_party/rust/rmp/tests/func/encode/float.rs vendored Normal file
View File

@@ -0,0 +1,21 @@
use rmp::encode::*;
#[test]
fn pass_pack_f32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
write_f32(&mut &mut buf[..], 3.4028234e38_f32).ok().unwrap();
assert_eq!([0xca, 0x7f, 0x7f, 0xff, 0xff], buf);
}
#[test]
fn pass_pack_f64() {
use std::f64;
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
write_f64(&mut &mut buf[..], f64::INFINITY).ok().unwrap();
assert_eq!([0xcb, 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}

240
third_party/rust/rmp/tests/func/encode/int.rs vendored Normal file
View File

@@ -0,0 +1,240 @@
use rmp::encode::*;
use rmp::Marker;
#[test]
fn pass_pack_pfix() {
let mut buf = [0x00];
write_pfix(&mut &mut buf[..], 127).ok().unwrap();
assert_eq!([0x7f], buf);
}
#[test]
fn fail_pack_pfix_too_small_buffer() {
let mut buf = [];
write_pfix(&mut &mut buf[..], 127).err().unwrap();
}
#[test]
#[should_panic(expected = "assertion failed")]
fn fail_pack_pfix_too_large() {
let mut buf = [0x00];
write_pfix(&mut &mut buf[..], 128).ok().unwrap();
}
#[test]
fn pass_pack_u8() {
let mut buf = [0x00, 0x00];
write_u8(&mut &mut buf[..], 255).ok().unwrap();
assert_eq!([0xcc, 0xff], buf);
}
#[test]
fn pass_pack_u16() {
let mut buf = [0x00, 0x00, 0x00];
write_u16(&mut &mut buf[..], 65535).ok().unwrap();
assert_eq!([0xcd, 0xff, 0xff], buf);
}
#[test]
fn pass_pack_u32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
write_u32(&mut &mut buf[..], 4294967295).ok().unwrap();
assert_eq!([0xce, 0xff, 0xff, 0xff, 0xff], buf);
}
#[test]
fn pass_pack_u64() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
write_u64(&mut &mut buf[..], 18446744073709551615).ok().unwrap();
assert_eq!([0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], buf);
}
#[test]
fn pass_pack_nfix() {
let mut buf = [0x00];
write_nfix(&mut &mut buf[..], -32).ok().unwrap();
assert_eq!([0xe0], buf);
}
#[test]
#[should_panic(expected = "assertion failed")]
fn fail_pack_nfix_too_large() {
let mut buf = [0x00];
write_nfix(&mut &mut buf[..], 0).ok().unwrap();
}
#[test]
#[should_panic(expected = "assertion failed")]
fn fail_pack_nfix_too_small() {
let mut buf = [0x00];
write_nfix(&mut &mut buf[..], -33).ok().unwrap();
}
#[test]
fn pass_pack_i8() {
let mut buf = [0x00, 0x00];
write_i8(&mut &mut buf[..], -128).ok().unwrap();
assert_eq!([0xd0, 0x80], buf);
}
#[test]
fn pass_pack_i16() {
let mut buf = [0x00, 0x00, 0x00];
write_i16(&mut &mut buf[..], -32768).ok().unwrap();
assert_eq!([0xd1, 0x80, 0x00], buf);
}
#[test]
fn pass_pack_i32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
write_i32(&mut &mut buf[..], -2147483648).ok().unwrap();
assert_eq!([0xd2, 0x80, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_i64() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
write_i64(&mut &mut buf[..], -9223372036854775808).ok().unwrap();
assert_eq!([0xd3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_uint_fix() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::FixPos(127), write_uint(&mut &mut buf[..], 127).ok().unwrap());
assert_eq!([0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_uint_u8() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::U8, write_uint(&mut &mut buf[..], 255).ok().unwrap());
assert_eq!([0xcc, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_uint_u16() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::U16, write_uint(&mut &mut buf[..], 65535).ok().unwrap());
assert_eq!([0xcd, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_uint_u32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::U32, write_uint(&mut &mut buf[..], 4294967295).ok().unwrap());
assert_eq!([0xce, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_uint_u64() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::U64, write_uint(&mut &mut buf[..], 18446744073709551615).ok().unwrap());
assert_eq!([0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], buf);
}
#[test]
fn pass_pack_sint_fix() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::FixNeg(-32), write_sint(&mut &mut buf[..], -32).ok().unwrap());
assert_eq!([0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_sint_i8_min() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::I8, write_sint(&mut &mut buf[..], -128).ok().unwrap());
assert_eq!([0xd0, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_sint_i16_min() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::I16, write_sint(&mut &mut buf[..], -32768).ok().unwrap());
assert_eq!([0xd1, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_sint_i16_max() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::U16, write_sint(&mut &mut buf[..], 32767).ok().unwrap());
assert_eq!([0xcd, 0x7f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_sint_i32_min() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::I32, write_sint(&mut &mut buf[..], -2147483648).ok().unwrap());
assert_eq!([0xd2, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_sint_i32_max() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::U32, write_sint(&mut &mut buf[..], 2147483647).ok().unwrap());
assert_eq!([0xce, 0x7f, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_sint_i64_min() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::I64, write_sint(&mut &mut buf[..], -9223372036854775808).ok().unwrap());
assert_eq!([0xd3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], buf);
}
#[test]
fn pass_pack_sint_i64_max() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::U64, write_sint(&mut &mut buf[..], 9223372036854775807).ok().unwrap());
assert_eq!([0xcf, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], buf);
}

29
third_party/rust/rmp/tests/func/encode/map.rs vendored Normal file
View File

@@ -0,0 +1,29 @@
use rmp::encode::*;
use rmp::Marker;
#[test]
fn pass_pack_len_fix() {
let mut buf = [0x00];
assert_eq!(Marker::FixMap(15), write_map_len(&mut &mut buf[..], 15).unwrap());
assert_eq!([0x8f], buf);
}
#[test]
fn pass_pack_len_u16() {
let mut buf = [0x00, 0x00, 0x00];
assert_eq!(Marker::Map16, write_map_len(&mut &mut buf[..], 65535).unwrap());
assert_eq!([0xde, 0xff, 0xff], buf);
}
#[test]
fn pass_pack_len_u32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::Map32, write_map_len(&mut &mut buf[..], 4294967295).unwrap());
assert_eq!([0xdf, 0xff, 0xff, 0xff, 0xff], buf);
}

9
third_party/rust/rmp/tests/func/encode/mod.rs vendored Normal file
View File

@@ -0,0 +1,9 @@
mod array;
mod bin;
mod bool;
mod ext;
mod float;
mod int;
mod map;
mod null;
mod string;

16
third_party/rust/rmp/tests/func/encode/null.rs vendored Normal file
View File

@@ -0,0 +1,16 @@
use rmp::encode::*;
#[test]
fn pass_pack() {
let mut buf = [0x00];
write_nil(&mut &mut buf[..]).unwrap();
assert_eq!([0xc0], buf);
}
#[test]
fn fail_pack_too_small_buffer() {
let mut buf = [];
write_nil(&mut &mut buf[..]).err().unwrap();
}

38
third_party/rust/rmp/tests/func/encode/string.rs vendored Normal file
View File

@@ -0,0 +1,38 @@
use rmp::encode::*;
use rmp::Marker;
#[test]
fn pass_pack_len_fix() {
let mut buf = [0x00];
assert_eq!(Marker::FixStr(31), write_str_len(&mut &mut buf[..], 31).unwrap());
assert_eq!([0xbf], buf);
}
#[test]
fn pass_pack_len_u8() {
let mut buf = [0x00, 0x00];
assert_eq!(Marker::Str8, write_str_len(&mut &mut buf[..], 255).unwrap());
assert_eq!([0xd9, 0xff], buf);
}
#[test]
fn pass_pack_len_u16() {
let mut buf = [0x00, 0x00, 0x00];
assert_eq!(Marker::Str16, write_str_len(&mut &mut buf[..], 65535).unwrap());
assert_eq!([0xda, 0xff, 0xff], buf);
}
#[test]
fn pass_pack_len_u32() {
let mut buf = [0x00, 0x00, 0x00, 0x00, 0x00];
assert_eq!(Marker::Str32, write_str_len(&mut &mut buf[..], 4294967295).unwrap());
assert_eq!([0xdb, 0xff, 0xff, 0xff, 0xff], buf);
}

35
third_party/rust/rmp/tests/func/mirror.rs vendored Normal file
View File

@@ -0,0 +1,35 @@
use rmp::decode::Bytes;
use rmp::encode::ByteBuf;
use rmp::{decode, encode};
quickcheck! {
fn mirror_uint(xs: u64) -> bool {
let mut buf = ByteBuf::new();
encode::write_uint(&mut buf, xs).unwrap();
xs == decode::read_int(&mut Bytes::new(buf.as_slice())).unwrap()
}
fn mirror_sint(xs: i64) -> bool {
let mut buf = ByteBuf::new();
encode::write_sint(&mut buf, xs).unwrap();
xs == decode::read_int(&mut Bytes::new(buf.as_slice())).unwrap()
}
fn mirror_f32(xs: f32) -> bool {
let mut buf = ByteBuf::new();
encode::write_f32(&mut buf, xs).unwrap();
let res = decode::read_f32(&mut Bytes::new(buf.as_slice())).unwrap();
xs == res || (xs.is_nan() && res.is_nan())
}
fn mirror_f64(xs: f64) -> bool {
let mut buf = ByteBuf::new();
encode::write_f64(&mut buf, xs).expect("write");
let res = decode::read_f64(&mut Bytes::new(buf.as_slice())).expect("read");
true || xs == res || (xs.is_nan() && res.is_nan())
}
}

9
third_party/rust/rmp/tests/lib.rs vendored Normal file
View File

@@ -0,0 +1,9 @@
#[cfg(test)]
#[macro_use]
extern crate quickcheck;
mod func {
mod decode;
mod encode;
mod mirror;
}

2
third_party/rust/suggest/.cargo-checksum.json vendored
View File

@@ -1 +1 @@
{"files":{"Cargo.toml":"b0b1de7601bdddcd829040bb40fa686fa2f08466a30685adef380937265ff260","README.md":"5e28baf874b643d756228bdab345e287bf107d3182dfe6a18aafadcc4b9a3fc9","benches/benchmark_all.rs":"dd15df10d2561e7fdd669e86ef4343c04153682ee5a3943d1de0a54ccf577ac8","build.rs":"78780c5cccfe22c3ff4198624b9e188559c437c3e6fa1c8bb66548eee6aa66bf","src/benchmarks/README.md":"ccee8dbddba8762d0453fa855bd6984137b224b8c019f3dd8e86a3c303f51d71","src/benchmarks/client.rs":"a42eadc20572aa5c0917288971c13f73b076260c8a9ee4ca8cb94a0d7bebe5a2","src/benchmarks/ingest.rs":"6d7eaa3758de5b2c1ca3c6b20484e09c489aef14b0c101415b4f61f92259055b","src/benchmarks/mod.rs":"fe1898ba4d783213525da10d92858ee84cebfd22749bad7aeb461d338fe5504a","src/bin/debug_ingestion_sizes.rs":"ce6e810be7b3fc19e826d75b622b82cfab5a1a99397a6d0833c2c4eebff2d364","src/config.rs":"d40c6e83d8b5faa32c66110803ca9e78611d43507e9d3f1e191a93a7773c37b3","src/db.rs":"8a15aa8bd45c8a7e0a3ba664a96683060979285594ae92f6e97e9cd6ea10fdad","src/error.rs":"5209d1d6d0fd3b155e2198813db47f12e9217978c4a28086b9fe93fc738b64af","src/keyword.rs":"988d0ab021c0df19cfd3c519df7d37f606bf984cd14d0efca4e5a7aff88344dd","src/lib.rs":"73887af023e223188f874d43ddf601784c93e9388a7adece79818d559a3366a1","src/pocket.rs":"1316668840ec9b4ea886223921dc9d3b5a1731d1a5206c0b1089f2a6c45c1b7b","src/provider.rs":"b843e63f7593c9f760be1cfdcb2ccd00f31c3ec7a1c47c46dac65e8ae5f9eccc","src/query.rs":"dd492d2458349036e87236944cf887f4453ab0d225b0b99568d409e6b65036ff","src/rs.rs":"dc4eda420b848f95a0123b73c7766afa0de1a4f99b2be01748f1e2cde9c389dc","src/schema.rs":"3155c9d2092c0e50f25e153508a489c6ee0e37da50efb7bddc5c00b85df2221e","src/store.rs":"a1ac06b23349db7fe3995d8779d07c6decc7ccbccd12cbc8127e221726e09b5a","src/suggest.udl":"57465b28c5756847178189672f0090692c990841ade03e24349eb7a487775935","src/suggestion.rs":"6e3926587b08b006408d399ad2117b6b27d0ecff3668e1da2db93c089831866f","src/testing/client.rs":"ab2eb24c6e7c09847e4df5287adc00d3fe9afc68cd15a47df66c0ed4b9a389ae","src/testing/data.rs":"29e27e7cec0d28f5c53755b2defc22ebfe4c2a4e3be1b337e3532e314964c56e","src/testing/mod.rs":"03663df6a04060e7d18e5a1451cea4edba03d86e0ee7e04741f2b55f64350f06","src/yelp.rs":"bc036ff71b438d53ce8811acd8d650d83ef03faeea476f5b659b403c1e64ff2b","uniffi.toml":"f26317442ddb5b3281245bef6e60ffcb78bb95d29fe4a351a56dbb88d4ec8aab"},"package":null}
{"files":{"Cargo.toml":"2f909cd4eaff25732d1a7c9193d4f8f98348c6933da521c69c34de7bb3a48dff","README.md":"5e28baf874b643d756228bdab345e287bf107d3182dfe6a18aafadcc4b9a3fc9","benches/benchmark_all.rs":"3582f21af9758766ff32ed95f90b69984b32091b1e31e0c0bef307c22fd82f18","build.rs":"78780c5cccfe22c3ff4198624b9e188559c437c3e6fa1c8bb66548eee6aa66bf","metrics.yaml":"0540ab2271aeab7f07335c7ceec12acde942995f9dcb3c29070489aa61899d56","src/benchmarks/README.md":"ccee8dbddba8762d0453fa855bd6984137b224b8c019f3dd8e86a3c303f51d71","src/benchmarks/client.rs":"3b92c350ad396b0e0b5438c7b7b94b08b322702f419ca9b815e6732bd174f8a1","src/benchmarks/ingest.rs":"ca368573591519e6b777d659d5615718bedb6eee1734e25242627f8116b10881","src/benchmarks/mod.rs":"2d7c20d47d6c7e17bc738255a31119bd0c4a4e495419a00c7b10b251ace9ef6b","src/benchmarks/query.rs":"0cac34ce895dd810513b30113b2608fecaece560c74a3286af55e3554f7a7b5a","src/bin/debug_ingestion_sizes.rs":"ce6e810be7b3fc19e826d75b622b82cfab5a1a99397a6d0833c2c4eebff2d364","src/config.rs":"d40c6e83d8b5faa32c66110803ca9e78611d43507e9d3f1e191a93a7773c37b3","src/db.rs":"1fbcdcc2e99692024fcb8a85aafa9dacb35d83703ce509bea1d0c694d379ceb2","src/error.rs":"183a92511565439915275e0705e6740ff513c2549f2ef78fd3055d8aaaf81021","src/fakespot.rs":"03d3aac07b3a3a9ceb8d2c452d4a122bfebf04579829e62e83487877055312d4","src/keyword.rs":"988d0ab021c0df19cfd3c519df7d37f606bf984cd14d0efca4e5a7aff88344dd","src/lib.rs":"455e7ca3cf1c95e04ac567e147a906a07df3e7d548546db399370e3fac3f94c9","src/metrics.rs":"b3b9816b8eda366f808b56e242ac6aa43a5b141ad01d43420fdfcbfca13e1cfc","src/pocket.rs":"1316668840ec9b4ea886223921dc9d3b5a1731d1a5206c0b1089f2a6c45c1b7b","src/provider.rs":"922ff68a4a49da637982d7b480651acf507197f62e0c0a04fce7237c243dc5e2","src/query.rs":"d9a26c024b8ac19ba0dbd6068a3bab881fa60928ecbac080bae80b8cbd2d5320","src/rs.rs":"81c11647b9ee4e52925b98e21adfd7c5c053846918500927502c41850f8f8a09","src/schema.rs":"2f4b0fbf0fd931c1cafa3be8dbddb9791c402125807b687d0bcf4d23b52743fc","src/store.rs":"a848bfbe254e83a6bb9a958a25984357ddb02d679af010cf170cb2498b3c1cd7","src/suggest.udl":"5eedf30402ed121e7f5b052782a55f64d7ca7a690901cd87f657a48f5206228b","src/suggestion.rs":"e24d951b564905f5bcd9b230a28fec78cbd4c29f8ef46bec014b06219902e3f3","src/testing/client.rs":"f059e844f336fd45fe4a335c67779558a98796d86f9ec0f43f96e6a0a6309f69","src/testing/data.rs":"d4fc5227996a8b115d93243fdbd83bc57d73a8c2d4c0b20dffa15bbec27925cb","src/testing/mod.rs":"b6ad90bb951fe0233493a7a1625f9979b6b8a946c5e027342ec25caeb5d1fed9","src/yelp.rs":"bc036ff71b438d53ce8811acd8d650d83ef03faeea476f5b659b403c1e64ff2b","uniffi.toml":"f26317442ddb5b3281245bef6e60ffcb78bb95d29fe4a351a56dbb88d4ec8aab"},"package":null}

5
third_party/rust/suggest/Cargo.toml vendored
View File

@@ -21,8 +21,12 @@ description = "Manages sponsored and web suggestions for Firefox Suggest"
readme = "README.md"
license = "MPL-2.0"
[lib]
bench = false
[[bin]]
name = "debug_ingestion_sizes"
bench = false
required-features = ["benchmark_api"]
[[bench]]
@@ -37,6 +41,7 @@ extend = "1.1"
log = "0.4"
once_cell = "1.5"
parking_lot = ">=0.11,<=0.12"
rmp-serde = "1.3"
serde_json = "1"
thiserror = "1"
uniffi = "0.27.1"

34
third_party/rust/suggest/benches/benchmark_all.rs vendored
View File

@@ -1,13 +1,29 @@
use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
use suggest::benchmarks::{ingest, BenchmarkWithInput};
use criterion::{
criterion_group, criterion_main, measurement::Measurement, BatchSize, BenchmarkGroup, Criterion,
};
use std::sync::Once;
use suggest::benchmarks::{ingest, query, BenchmarkWithInput};
pub fn ingest_single_provider(c: &mut Criterion) {
pub fn ingest(c: &mut Criterion) {
setup_viaduct();
let mut group = c.benchmark_group("ingest");
viaduct_reqwest::use_reqwest_backend();
// This needs to be 10 for now, or else the `ingest-amp-wikipedia` benchmark would take around
// 100s to run which feels like too long. `ingest-amp-mobile` also would take a around 50s.
group.sample_size(10);
for (name, benchmark) in ingest::all_benchmarks() {
run_benchmarks(group, ingest::all_benchmarks())
}
pub fn query(c: &mut Criterion) {
setup_viaduct();
let group = c.benchmark_group("query");
run_benchmarks(group, query::all_benchmarks())
}
fn run_benchmarks<B: BenchmarkWithInput, M: Measurement>(
mut group: BenchmarkGroup<M>,
benchmarks: Vec<(&'static str, B)>,
) {
for (name, benchmark) in benchmarks {
group.bench_function(name.to_string(), |b| {
b.iter_batched(
|| benchmark.generate_input(),
@@ -19,7 +35,13 @@ pub fn ingest_single_provider(c: &mut Criterion) {
);
});
}
group.finish();
}
criterion_group!(benches, ingest_single_provider);
fn setup_viaduct() {
static INIT: Once = Once::new();
INIT.call_once(viaduct_reqwest::use_reqwest_backend);
}
criterion_group!(benches, ingest, query);
criterion_main!(benches);

93
third_party/rust/suggest/metrics.yaml vendored Normal file
View File

@@ -0,0 +1,93 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This file defines the metrics that will be gathered for the Suggest
# component. Changes to these metrics require data review.
#
# We can't record metrics from Rust directly. To work around that we:
# - Define the metrics in application-services
# - Define API calls in application-services that return the metrics
# alongside the normal results.
# - Record the metrics with Glean in the consumer code
---
$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
suggest:
ingest_time:
type: labeled_timing_distribution
description: Time for ingestion (excluding download time), labelled by record type
time_unit: microsecond
labels:
- icon
- data
- amo-suggestions
- pocket-suggestions
- yelp-suggestions
- mdn-suggestions
- weather
- configuration
- amp-mobile-suggestions
- fakespot-suggestions
bugs:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1908397
data_reviews:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1911664
notification_emails:
- disco-team@mozilla.com
- bdk@mozilla.com
expires: "never"
data_sensitivity:
- technical
ingest_download_time:
type: labeled_timing_distribution
description: Download time for ingestion, labelled by record type
time_unit: microsecond
labels:
- icon
- data
- amo-suggestions
- pocket-suggestions
- yelp-suggestions
- mdn-suggestions
- weather
- configuration
- amp-mobile-suggestions
- fakespot-suggestions
bugs:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1908397
data_reviews:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1911664
notification_emails:
- disco-team@mozilla.com
- bdk@mozilla.com
expires: "never"
data_sensitivity:
- technical
query_time:
type: labeled_timing_distribution
description: Time executing queries, labelled by provider type
time_unit: microsecond
labels:
- amp
- ampmobile
- wikipedia
- amo
- pocket
- yelp
- mdn
- weather
- fakespot
bugs:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1908397
data_reviews:
- https://bugzilla.mozilla.org/show_bug.cgi?id=1911664
notification_emails:
- disco-team@mozilla.com
- bdk@mozilla.com
expires: "never"
data_sensitivity:
- technical

124
third_party/rust/suggest/src/benchmarks/client.rs vendored
View File

@@ -2,64 +2,92 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use crate::{rs, Result};
use parking_lot::Mutex;
use std::collections::HashMap;
/// Remotes settings client that runs during the benchmark warm-up phase.
use crate::{db::SuggestDao, error::Error, rs, Result};
/// Remotes settings client for benchmarking
///
/// This should be used to run a full ingestion.
/// Then it can be converted into a [RemoteSettingsBenchmarkClient], which allows benchmark code to exclude the network request time.
/// [RemoteSettingsBenchmarkClient] implements [rs::Client] by getting data from a HashMap rather than hitting the network.
pub struct RemoteSettingsWarmUpClient {
client: rs::RemoteSettingsClient,
pub get_records_responses: Mutex<HashMap<rs::RecordRequest, Vec<rs::Record>>>,
}
impl RemoteSettingsWarmUpClient {
pub fn new() -> Self {
Self {
client: rs::RemoteSettingsClient::new(None, None, None).unwrap(),
get_records_responses: Mutex::new(HashMap::new()),
}
}
}
impl Default for RemoteSettingsWarmUpClient {
fn default() -> Self {
Self::new()
}
}
impl rs::Client for RemoteSettingsWarmUpClient {
fn get_records(&self, request: rs::RecordRequest) -> Result<Vec<rs::Record>> {
let response = self.client.get_records(request.clone())?;
self.get_records_responses
.lock()
.insert(request, response.clone());
Ok(response)
}
}
#[derive(Clone)]
/// This fetches all data in `new`, then implements [rs::Client] by returning the local data.
/// Construct this one before the benchmark is run, then clone it as input for the benchmark. This
/// ensures that network time does not count towards the benchmark time.
#[derive(Clone, Default)]
pub struct RemoteSettingsBenchmarkClient {
pub get_records_responses: HashMap<rs::RecordRequest, Vec<rs::Record>>,
records: Vec<rs::Record>,
attachments: HashMap<String, Vec<u8>>,
}
impl RemoteSettingsBenchmarkClient {
pub fn new() -> Result<Self> {
let mut new_benchmark_client = Self::default();
new_benchmark_client.fetch_data_with_client(
remote_settings::Client::new(remote_settings::RemoteSettingsConfig {
server: None,
bucket_name: None,
collection_name: "quicksuggest".to_owned(),
server_url: None,
})?,
rs::Collection::Quicksuggest,
)?;
new_benchmark_client.fetch_data_with_client(
remote_settings::Client::new(remote_settings::RemoteSettingsConfig {
server: None,
bucket_name: None,
collection_name: "fakespot-suggest-products".to_owned(),
server_url: None,
})?,
rs::Collection::Fakespot,
)?;
Ok(new_benchmark_client)
}
fn fetch_data_with_client(
&mut self,
client: remote_settings::Client,
collection: rs::Collection,
) -> Result<()> {
let response = client.get_records()?;
for r in &response.records {
if let Some(a) = &r.attachment {
self.attachments
.insert(a.location.clone(), client.get_attachment(&a.location)?);
}
}
self.records.extend(
response
.records
.into_iter()
.filter_map(|r| rs::Record::new(r, collection).ok()),
);
Ok(())
}
pub fn total_attachment_size(&self) -> usize {
self.attachments.values().map(|a| a.len()).sum()
}
}
impl rs::Client for RemoteSettingsBenchmarkClient {
fn get_records(&self, request: rs::RecordRequest) -> Result<Vec<rs::Record>> {
fn get_records(
&self,
collection: rs::Collection,
_db: &mut SuggestDao,
) -> Result<Vec<rs::Record>> {
Ok(self
.get_records_responses
.get(&request)
.unwrap_or_else(|| panic!("options not found: {request:?}"))
.clone())
.records
.iter()
.filter(|r| r.collection == collection)
.cloned()
.collect())
}
}
impl From<RemoteSettingsWarmUpClient> for RemoteSettingsBenchmarkClient {
fn from(warm_up_client: RemoteSettingsWarmUpClient) -> Self {
Self {
get_records_responses: warm_up_client.get_records_responses.into_inner(),
fn download_attachment(&self, record: &rs::Record) -> Result<Vec<u8>> {
match &record.attachment {
Some(a) => match self.attachments.get(&a.location) {
Some(data) => Ok(data.clone()),
None => Err(Error::MissingAttachment(record.id.to_string())),
},
None => Err(Error::MissingAttachment(record.id.to_string())),
}
}
}

132
third_party/rust/suggest/src/benchmarks/ingest.rs vendored
View File

@@ -2,18 +2,14 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use std::sync::OnceLock;
use crate::{
benchmarks::{
client::{RemoteSettingsBenchmarkClient, RemoteSettingsWarmUpClient},
BenchmarkWithInput,
},
benchmarks::{client::RemoteSettingsBenchmarkClient, unique_db_filename, BenchmarkWithInput},
rs::SuggestRecordType,
store::SuggestStoreInner,
SuggestIngestionConstraints,
};
use std::sync::atomic::{AtomicU32, Ordering};
static DB_FILE_COUNTER: AtomicU32 = AtomicU32::new(0);
pub struct IngestBenchmark {
temp_dir: tempfile::TempDir,
@@ -22,17 +18,23 @@ pub struct IngestBenchmark {
reingest: bool,
}
/// Get a benchmark client to use for the tests
///
/// Uses OnceLock to ensure we only construct it once.
fn get_benchmark_client() -> RemoteSettingsBenchmarkClient {
static CELL: OnceLock<RemoteSettingsBenchmarkClient> = OnceLock::new();
CELL.get_or_init(|| {
RemoteSettingsBenchmarkClient::new()
.unwrap_or_else(|e| panic!("Error creating benchmark client {e}"))
})
.clone()
}
impl IngestBenchmark {
pub fn new(record_type: SuggestRecordType, reingest: bool) -> Self {
let temp_dir = tempfile::tempdir().unwrap();
let store = SuggestStoreInner::new(
temp_dir.path().join("warmup.sqlite"),
vec![],
RemoteSettingsWarmUpClient::new(),
);
store.benchmark_ingest_records_by_type(record_type);
Self {
client: RemoteSettingsBenchmarkClient::from(store.into_settings_client()),
client: get_benchmark_client(),
temp_dir,
record_type,
reingest,
@@ -49,21 +51,19 @@ impl BenchmarkWithInput for IngestBenchmark {
type Input = InputType;
fn generate_input(&self) -> Self::Input {
let data_path = self.temp_dir.path().join(format!(
"db{}.sqlite",
DB_FILE_COUNTER.fetch_add(1, Ordering::Relaxed)
));
let data_path = self.temp_dir.path().join(unique_db_filename());
let store = SuggestStoreInner::new(data_path, vec![], self.client.clone());
store.ensure_db_initialized();
if self.reingest {
store.force_reingest(self.record_type);
store.ingest_records_by_type(self.record_type);
store.force_reingest();
}
InputType(store)
}
fn benchmarked_code(&self, input: Self::Input) {
let InputType(store) = input;
store.benchmark_ingest_records_by_type(self.record_type);
store.ingest_records_by_type(self.record_type);
}
}
@@ -74,74 +74,82 @@ pub fn all_benchmarks() -> Vec<(&'static str, IngestBenchmark)> {
"ingest-icon",
IngestBenchmark::new(SuggestRecordType::Icon, false),
),
(
"ingest-amp-wikipedia",
IngestBenchmark::new(SuggestRecordType::AmpWikipedia, false),
),
(
"ingest-amo",
IngestBenchmark::new(SuggestRecordType::Amo, false),
),
(
"ingest-pocket",
IngestBenchmark::new(SuggestRecordType::Pocket, false),
),
(
"ingest-yelp",
IngestBenchmark::new(SuggestRecordType::Yelp, false),
),
(
"ingest-mdn",
IngestBenchmark::new(SuggestRecordType::Mdn, false),
),
(
"ingest-weather",
IngestBenchmark::new(SuggestRecordType::Weather, false),
),
(
"ingest-global-config",
IngestBenchmark::new(SuggestRecordType::GlobalConfig, false),
),
(
"ingest-amp-mobile",
IngestBenchmark::new(SuggestRecordType::AmpMobile, false),
),
(
"ingest-again-icon",
IngestBenchmark::new(SuggestRecordType::Icon, true),
),
(
"ingest-amp-wikipedia",
IngestBenchmark::new(SuggestRecordType::AmpWikipedia, false),
),
(
"ingest-again-amp-wikipedia",
IngestBenchmark::new(SuggestRecordType::AmpWikipedia, true),
),
(
"ingest-amo",
IngestBenchmark::new(SuggestRecordType::Amo, false),
),
(
"ingest-again-amo",
IngestBenchmark::new(SuggestRecordType::Amo, true),
),
(
"ingest-pocket",
IngestBenchmark::new(SuggestRecordType::Pocket, false),
),
(
"ingest-again-pocket",
IngestBenchmark::new(SuggestRecordType::Pocket, true),
),
(
"ingest-yelp",
IngestBenchmark::new(SuggestRecordType::Yelp, false),
),
(
"ingest-again-yelp",
IngestBenchmark::new(SuggestRecordType::Yelp, true),
),
(
"ingest-mdn",
IngestBenchmark::new(SuggestRecordType::Mdn, false),
),
(
"ingest-again-mdn",
IngestBenchmark::new(SuggestRecordType::Mdn, true),
),
(
"ingest-weather",
IngestBenchmark::new(SuggestRecordType::Weather, false),
),
(
"ingest-again-weather",
IngestBenchmark::new(SuggestRecordType::Weather, true),
),
(
"ingest-global-config",
IngestBenchmark::new(SuggestRecordType::GlobalConfig, false),
),
(
"ingest-again-global-config",
IngestBenchmark::new(SuggestRecordType::GlobalConfig, true),
),
(
"ingest-amp-mobile",
IngestBenchmark::new(SuggestRecordType::AmpMobile, false),
),
(
"ingest-again-amp-mobile",
IngestBenchmark::new(SuggestRecordType::AmpMobile, true),
),
(
"ingest-fakespot",
IngestBenchmark::new(SuggestRecordType::Fakespot, false),
),
(
"ingest-again-fakespot",
IngestBenchmark::new(SuggestRecordType::Fakespot, true),
),
]
}
@@ -150,25 +158,19 @@ pub fn print_debug_ingestion_sizes() {
let store = SuggestStoreInner::new(
"file:debug_ingestion_sizes?mode=memory&cache=shared",
vec![],
RemoteSettingsWarmUpClient::new(),
RemoteSettingsBenchmarkClient::new().unwrap(),
);
store
.ingest(SuggestIngestionConstraints::default())
.ingest(SuggestIngestionConstraints {
// Uncomment to measure the size for a specific provider
// providers: Some(vec![crate::SuggestionProvider::Fakespot]),
..SuggestIngestionConstraints::default()
})
.unwrap();
let table_row_counts = store.table_row_counts();
let db_size = store.db_size();
let client = store.into_settings_client();
let total_attachment_size: usize = client
.get_records_responses
.lock()
.values()
.flat_map(|records| {
records.iter().map(|r| match &r.attachment_data {
Some(d) => d.len(),
None => 0,
})
})
.sum();
let total_attachment_size: usize = client.total_attachment_size();
println!(
"Total attachment size: {}kb",

8
third_party/rust/suggest/src/benchmarks/mod.rs vendored
View File

@@ -10,8 +10,11 @@
//!
//! All benchmarks are defined as structs that implement either the [Benchmark] or [BenchmarkWithInput]
use std::sync::atomic::{AtomicU32, Ordering};
pub mod client;
pub mod ingest;
pub mod query;
/// Trait for simple benchmarks
///
@@ -38,3 +41,8 @@ pub trait BenchmarkWithInput {
/// Perform the operations that we're benchmarking.
fn benchmarked_code(&self, input: Self::Input);
}
fn unique_db_filename() -> String {
static COUNTER: AtomicU32 = AtomicU32::new(0);
format!("db{}.sqlite", COUNTER.fetch_add(1, Ordering::Relaxed))
}

87
third_party/rust/suggest/src/benchmarks/query.rs vendored Normal file
View File

@@ -0,0 +1,87 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use crate::{
benchmarks::{unique_db_filename, BenchmarkWithInput},
SuggestIngestionConstraints, SuggestStore, SuggestionProvider, SuggestionQuery,
};
pub struct QueryBenchmark {
store: SuggestStore,
provider: SuggestionProvider,
query: &'static str,
}
impl QueryBenchmark {
pub fn new(provider: SuggestionProvider, query: &'static str) -> Self {
let temp_dir = tempfile::tempdir().unwrap();
let data_path = temp_dir.path().join(unique_db_filename());
let store =
SuggestStore::new(&data_path.to_string_lossy(), None).expect("Error building store");
store
.ingest(SuggestIngestionConstraints::all_providers())
.expect("Error during ingestion");
Self {
store,
provider,
query,
}
}
}
// The input for each benchmark a query to pass to the store
pub struct InputType(SuggestionQuery);
impl BenchmarkWithInput for QueryBenchmark {
type Input = InputType;
fn generate_input(&self) -> Self::Input {
InputType(SuggestionQuery {
providers: vec![self.provider],
keyword: self.query.to_string(),
limit: None,
})
}
fn benchmarked_code(&self, input: Self::Input) {
let InputType(query) = input;
self.store
.query(query)
.unwrap_or_else(|e| panic!("Error querying store: {e}"));
}
}
pub fn all_benchmarks() -> Vec<(&'static str, QueryBenchmark)> {
vec![
// Fakespot queries, these attempt to perform prefix matches with various character
// lengths.
//
// The query code will only do a prefix match if the total input length is > 3 chars.
// Therefore, to test shorter prefixes we use 2-term queries.
(
"query-fakespot-hand-s",
QueryBenchmark::new(SuggestionProvider::Fakespot, "hand s"),
),
(
"query-fakespot-hand-sa",
QueryBenchmark::new(SuggestionProvider::Fakespot, "hand sa"),
),
(
"query-fakespot-hand-san",
QueryBenchmark::new(SuggestionProvider::Fakespot, "hand san"),
),
(
"query-fakespot-sani",
QueryBenchmark::new(SuggestionProvider::Fakespot, "sani"),
),
(
"query-fakespot-sanit",
QueryBenchmark::new(SuggestionProvider::Fakespot, "sanit"),
),
(
"query-fakespot-saniti",
QueryBenchmark::new(SuggestionProvider::Fakespot, "saniti"),
),
]
}

211
third_party/rust/suggest/src/db.rs vendored
View File

@@ -3,20 +3,22 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
use std::{collections::HashSet, path::Path, sync::Arc};
use std::{path::Path, sync::Arc};
use interrupt_support::{SqlInterruptHandle, SqlInterruptScope};
use parking_lot::{Mutex, MutexGuard};
use remote_settings::RemoteSettingsResponse;
use rusqlite::{
named_params,
types::{FromSql, ToSql},
Connection, OpenFlags,
Connection, OpenFlags, OptionalExtension,
};
use sql_support::{open_database::open_database_with_flags, ConnExt};
use crate::{
config::{SuggestGlobalConfig, SuggestProviderConfig},
error::RusqliteResultExt,
fakespot,
keyword::full_keyword,
pocket::{split_keyword, KeywordConfidence},
provider::SuggestionProvider,
@@ -158,6 +160,15 @@ impl WriteScope<'_> {
Ok(result)
}
/// Accesses the Suggest database in a transaction for reading only
pub fn read<T>(&mut self, op: impl FnOnce(&SuggestDao) -> Result<T>) -> Result<T> {
let tx = self.conn.transaction()?;
let dao = SuggestDao::new(&tx, &self.scope);
let result = op(&dao)?;
tx.commit()?;
Ok(result)
}
pub fn err_if_interrupted(&self) -> Result<()> {
Ok(self.scope.err_if_interrupted()?)
}
@@ -183,12 +194,11 @@ impl<'a> SuggestDao<'a> {
//
// These methods combine several low-level calls into one logical operation.
pub fn delete_record_data(&mut self, record: &Record) -> Result<()> {
let record_id = SuggestRecordId::from(&record.id);
pub fn delete_record_data(&mut self, record_id: &SuggestRecordId) -> Result<()> {
// Drop either the icon or suggestions, records only contain one or the other
match record_id.as_icon_id() {
Some(icon_id) => self.drop_icon(icon_id)?,
None => self.drop_suggestions(&record_id)?,
None => self.drop_suggestions(record_id)?,
};
Ok(())
}
@@ -197,45 +207,105 @@ impl<'a> SuggestDao<'a> {
//
// These methods implement CRUD operations
pub fn read_cached_rs_data(&self, collection: &str) -> Option<RemoteSettingsResponse> {
match self.try_read_cached_rs_data(collection) {
Ok(result) => result,
Err(e) => {
// Return None on failure . If the cached data is corrupted, maybe because the
// RemoteSettingsResponse schema changed, then we want to just continue on. This also matches
// the proposed API from #6328, so it should be easier to adapt this code once
// that's merged.
error_support::report_error!("suggest-rs-cache-read", "{e}");
None
}
}
}
pub fn write_cached_rs_data(&mut self, collection: &str, data: &RemoteSettingsResponse) {
if let Err(e) = self.try_write_cached_rs_data(collection, data) {
// Return None on failure for the same reason as in [Self::read_cached_rs_data]
error_support::report_error!("suggest-rs-cache-write", "{e}");
}
}
fn try_read_cached_rs_data(&self, collection: &str) -> Result<Option<RemoteSettingsResponse>> {
let mut stmt = self
.conn
.prepare_cached("SELECT data FROM rs_cache WHERE collection = ?")?;
let data = stmt
.query_row((collection,), |row| row.get::<_, Vec<u8>>(0))
.optional()?;
match data {
Some(data) => Ok(Some(rmp_serde::decode::from_slice(data.as_slice())?)),
None => Ok(None),
}
}
fn try_write_cached_rs_data(
&mut self,
collection: &str,
data: &RemoteSettingsResponse,
) -> Result<()> {
let mut stmt = self
.conn
.prepare_cached("INSERT OR REPLACE INTO rs_cache(collection, data) VALUES(?, ?)")?;
stmt.execute((collection, rmp_serde::encode::to_vec(data)?))?;
Ok(())
}
pub fn get_ingested_records(&self) -> Result<Vec<IngestedRecord>> {
let mut stmt = self
.conn
.prepare_cached("SELECT id, collection, type, last_modified FROM ingested_records")?;
let rows = stmt.query_and_then((), IngestedRecord::from_row)?;
rows.collect()
}
pub fn update_ingested_records(
&mut self,
collection: &str,
new_records: &[&Record],
updated_records: &[&Record],
deleted_records: &[&IngestedRecord],
) -> Result<()> {
let mut delete_stmt = self
.conn
.prepare_cached("DELETE FROM ingested_records WHERE collection = ? AND id = ?")?;
for deleted in deleted_records {
delete_stmt.execute((collection, deleted.id.as_str()))?;
}
let mut insert_stmt = self.conn.prepare_cached(
"INSERT OR REPLACE INTO ingested_records(id, collection, type, last_modified) VALUES(?, ?, ?, ?)",
)?;
for record in new_records.iter().chain(updated_records) {
insert_stmt.execute((
record.id.as_str(),
collection,
record.record_type().as_str(),
record.last_modified,
))?;
}
Ok(())
}
/// Update the DB so that we re-ingest all records on the next ingestion.
///
/// We hack this by setting the last_modified time to 1 so that the next time around we always
/// re-ingest the record.
pub fn force_reingest(&mut self) -> Result<()> {
self.conn
.prepare_cached("UPDATE ingested_records SET last_modified=1")?
.execute(())?;
Ok(())
}
pub fn suggestions_table_empty(&self) -> Result<bool> {
Ok(self
.conn
.query_one::<bool>("SELECT NOT EXISTS (SELECT 1 FROM suggestions)")?)
}
/// Fetches suggestions that match the given query from the database.
pub fn fetch_suggestions(&self, query: &SuggestionQuery) -> Result<Vec<Suggestion>> {
let unique_providers = query.providers.iter().collect::<HashSet<_>>();
unique_providers
.iter()
.try_fold(vec![], |mut acc, provider| {
let suggestions = match provider {
SuggestionProvider::Amp => {
self.fetch_amp_suggestions(query, AmpSuggestionType::Desktop)
}
SuggestionProvider::AmpMobile => {
self.fetch_amp_suggestions(query, AmpSuggestionType::Mobile)
}
SuggestionProvider::Wikipedia => self.fetch_wikipedia_suggestions(query),
SuggestionProvider::Amo => self.fetch_amo_suggestions(query),
SuggestionProvider::Pocket => self.fetch_pocket_suggestions(query),
SuggestionProvider::Yelp => self.fetch_yelp_suggestions(query),
SuggestionProvider::Mdn => self.fetch_mdn_suggestions(query),
SuggestionProvider::Weather => self.fetch_weather_suggestions(query),
SuggestionProvider::Fakespot => self.fetch_fakespot_suggestions(query),
}?;
acc.extend(suggestions);
Ok(acc)
})
.map(|mut suggestions| {
suggestions.sort();
if let Some(limit) = query.limit.and_then(|limit| usize::try_from(limit).ok()) {
suggestions.truncate(limit);
}
suggestions
})
}
/// Fetches Suggestions of type Amp provider that match the given query
pub fn fetch_amp_suggestions(
&self,
@@ -692,7 +762,9 @@ impl<'a> SuggestDao<'a> {
f.rating,
f.total_reviews,
i.data,
i.mimetype
i.mimetype,
f.keywords,
f.product_type
FROM
suggestions s
JOIN
@@ -711,10 +783,17 @@ impl<'a> SuggestDao<'a> {
"#,
(&query.fts_query(),),
|row| {
let score = fakespot::FakespotScore::new(
&query.keyword,
row.get(9)?,
row.get(10)?,
row.get(2)?,
)
.as_suggest_score();
Ok(Suggestion::Fakespot {
title: row.get(0)?,
url: row.get(1)?,
score: row.get(2)?,
score,
fakespot_grade: row.get(3)?,
product_id: row.get(4)?,
rating: row.get(5)?,
@@ -1106,33 +1185,6 @@ impl<'a> SuggestDao<'a> {
Ok(())
}
#[cfg(feature = "benchmark_api")]
/// Clears the value for a metadata key.
///
/// This is currently only used for the benchmarks.
pub fn clear_meta(&mut self, key: &str) -> Result<()> {
self.conn.execute_cached(
"DELETE FROM meta WHERE key = :key",
named_params! { ":key": key },
)?;
Ok(())
}
/// Updates the last ingest timestamp if the given last modified time is
/// newer than the existing one recorded.
pub fn put_last_ingest_if_newer(
&mut self,
last_ingest_key: &str,
record_last_modified: u64,
) -> Result<()> {
let last_ingest = self.get_meta::<u64>(last_ingest_key)?.unwrap_or_default();
if record_last_modified > last_ingest {
self.put_meta(last_ingest_key, record_last_modified)?;
}
Ok(())
}
/// Stores global Suggest configuration data.
pub fn put_global_config(&mut self, config: &SuggestGlobalConfig) -> Result<()> {
self.put_meta(GLOBAL_CONFIG_META_KEY, serde_json::to_string(config)?)
@@ -1171,6 +1223,25 @@ impl<'a> SuggestDao<'a> {
}
}
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct IngestedRecord {
pub id: SuggestRecordId,
pub collection: String,
pub record_type: String,
pub last_modified: u64,
}
impl IngestedRecord {
fn from_row(row: &rusqlite::Row) -> Result<Self> {
Ok(Self {
id: SuggestRecordId::new(row.get("id")?),
collection: row.get("collection")?,
record_type: row.get("type")?,
last_modified: row.get("last_modified")?,
})
}
}
/// Helper struct to get full_keyword_ids for a suggestion
///
/// `FullKeywordInserter` handles repeated full keywords efficiently. The first instance will
@@ -1391,11 +1462,13 @@ impl<'conn> FakespotInsertStatement<'conn> {
suggestion_id,
fakespot_grade,
product_id,
keywords,
product_type,
rating,
total_reviews,
icon_id
)
VALUES(?, ?, ?, ?, ?, ?)
VALUES(?, ?, ?, ?, ?, ?, ?, ?)
",
)?))
}
@@ -1414,6 +1487,8 @@ impl<'conn> FakespotInsertStatement<'conn> {
suggestion_id,
&fakespot.fakespot_grade,
&fakespot.product_id,
&fakespot.keywords.to_lowercase(),
&fakespot.product_type.to_lowercase(),
fakespot.rating,
fakespot.total_reviews,
icon_id,

22
third_party/rust/suggest/src/error.rs vendored
View File

@@ -20,8 +20,8 @@ pub enum Error {
context: String,
},
#[error("JSON error: {0}")]
Json(#[from] serde_json::Error),
#[error("Serialization error: {0}")]
Serialization(String),
#[error("Error from Remote Settings: {0}")]
RemoteSettings(#[from] RemoteSettingsError),
@@ -51,6 +51,24 @@ impl From<rusqlite::Error> for Error {
}
}
impl From<serde_json::Error> for Error {
fn from(e: serde_json::Error) -> Self {
Self::Serialization(e.to_string())
}
}
impl From<rmp_serde::decode::Error> for Error {
fn from(e: rmp_serde::decode::Error) -> Self {
Self::Serialization(e.to_string())
}
}
impl From<rmp_serde::encode::Error> for Error {
fn from(e: rmp_serde::encode::Error) -> Self {
Self::Serialization(e.to_string())
}
}
#[extend::ext(name=RusqliteResultExt)]
pub impl<T> Result<T, rusqlite::Error> {
// Convert an rusqlite::Error to our error type, with a context value

225
third_party/rust/suggest/src/fakespot.rs vendored Normal file
View File

@@ -0,0 +1,225 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/// Fakespot-specific logic
/// Score used to order Fakespot suggestions
///
/// FakespotScore contains several components, each in the range of [0, 1]
pub struct FakespotScore {
/// Did the query match the `keywords` field exactly?
keywords_score: f64,
/// How well did the query match the `product_type` field?
product_type_score: f64,
/// Fakespot score from the RS data, this reflects the average review, number of reviews,
/// Fakespot grade, etc.
fakespot_score: f64,
}
impl FakespotScore {
pub fn new(query: &str, keywords: String, product_type: String, fakespot_score: f64) -> Self {
let query = query.to_lowercase();
let query_terms = split_terms(&query);
Self {
keywords_score: calc_keywords_score(&query_terms, &keywords),
product_type_score: calc_product_type_score(&query_terms, &product_type),
fakespot_score,
}
}
/// Convert a FakespotScore into the value to use in `Sugggestion::Fakespot::score`
///
/// This converts FakespotScore into a single float that:
/// - Is > 0.3 so that Fakespot suggestions are preferred to AMP ones
/// - Reflects the Fakespot ordering:
/// - Suggestions with higher keywords_score are greater
/// - If keywords_score is tied, then suggestions with higher product_type_scores are greater
/// - If both are tied, then suggestions with higher fakespot_score are greater
pub fn as_suggest_score(&self) -> f64 {
0.30 + (0.01 * self.keywords_score)
+ (0.001 * self.product_type_score)
+ (0.0001 * self.fakespot_score)
}
}
/// Split a string containing terms into a list of individual terms, normalized to lowercase
fn split_terms(string: &str) -> Vec<&str> {
string.split_whitespace().collect()
}
fn calc_keywords_score(query_terms: &[&str], keywords: &str) -> f64 {
// Note: We can assume keywords is lower-case, since we do that during ingestion
let keyword_terms = split_terms(keywords);
if keyword_terms.is_empty() {
return 0.0;
}
if query_terms == keyword_terms {
1.0
} else {
0.0
}
}
fn calc_product_type_score(query_terms: &[&str], product_type: &str) -> f64 {
// Note: We can assume product_type is lower-case, since we do that during ingestion
let product_type_terms = split_terms(product_type);
if product_type_terms.is_empty() {
return 0.0;
}
let count = product_type_terms
.iter()
.filter(|t| query_terms.contains(t))
.count() as f64;
count / product_type_terms.len() as f64
}
#[cfg(test)]
mod tests {
use super::*;
struct KeywordsTestCase {
keywords: &'static str,
query: &'static str,
expected: f64,
}
impl KeywordsTestCase {
fn test(&self) {
let actual =
calc_keywords_score(&split_terms(&self.query.to_lowercase()), self.keywords);
assert_eq!(
actual, self.expected,
"keywords: {} query: {} expected: {} actual: {actual}",
self.keywords, self.query, self.expected,
);
}
}
#[test]
fn test_keywords_score() {
// Keyword score 1.0 on exact matches, 0.0 otherwise
KeywordsTestCase {
keywords: "apple",
query: "apple",
expected: 1.0,
}
.test();
KeywordsTestCase {
keywords: "apple",
query: "android",
expected: 0.0,
}
.test();
KeywordsTestCase {
keywords: "apple",
query: "apple phone",
expected: 0.0,
}
.test();
// Empty keywords should always score 0.0
KeywordsTestCase {
keywords: "",
query: "",
expected: 0.0,
}
.test();
KeywordsTestCase {
keywords: "",
query: "apple",
expected: 0.0,
}
.test();
// Matching should be case insensitive
KeywordsTestCase {
keywords: "apple",
query: "Apple",
expected: 1.0,
}
.test();
}
struct ProductTypeTestCase {
query: &'static str,
product_type: &'static str,
expected: f64,
}
impl ProductTypeTestCase {
fn test(&self) {
let actual = calc_product_type_score(
&split_terms(&self.query.to_lowercase()),
self.product_type,
);
assert_eq!(
actual, self.expected,
"product_type: {} query: {} expected: {} actual: {actual}",
self.product_type, self.query, self.expected,
);
}
}
#[test]
fn test_product_type_score() {
// Product type scores based on the percentage of terms in the product type that are also
// present in the query
ProductTypeTestCase {
product_type: "standing desk",
query: "standing desk",
expected: 1.0,
}
.test();
ProductTypeTestCase {
product_type: "standing desk",
query: "desk",
expected: 0.5,
}
.test();
ProductTypeTestCase {
product_type: "standing desk",
query: "desk desk desk",
expected: 0.5,
}
.test();
ProductTypeTestCase {
product_type: "standing desk",
query: "standing",
expected: 0.5,
}
.test();
ProductTypeTestCase {
product_type: "standing desk",
query: "phone",
expected: 0.0,
}
.test();
// Extra terms in the query are ignored
ProductTypeTestCase {
product_type: "standing desk",
query: "standing desk for my office",
expected: 1.0,
}
.test();
// Empty product_type should always score 0.0
ProductTypeTestCase {
product_type: "",
query: "",
expected: 0.0,
}
.test();
// Matching should be case insensitive
ProductTypeTestCase {
product_type: "desk",
query: "Desk",
expected: 1.0,
}
.test();
// Extra spaces are ignored
ProductTypeTestCase {
product_type: "desk",
query: " desk ",
expected: 1.0,
}
.test();
}
}

5
third_party/rust/suggest/src/lib.rs vendored
View File

@@ -9,7 +9,9 @@ pub mod benchmarks;
mod config;
mod db;
mod error;
mod fakespot;
mod keyword;
mod metrics;
pub mod pocket;
mod provider;
mod query;
@@ -23,8 +25,9 @@ mod yelp;
pub use config::{SuggestGlobalConfig, SuggestProviderConfig};
pub use error::SuggestApiError;
pub use metrics::{LabeledTimingSample, SuggestIngestionMetrics};
pub use provider::SuggestionProvider;
pub use query::SuggestionQuery;
pub use query::{QueryWithMetricsResult, SuggestionQuery};
pub use store::{InterruptKind, SuggestIngestionConstraints, SuggestStore, SuggestStoreBuilder};
pub use suggestion::{raw_suggestion_url_matches, Suggestion};

97
third_party/rust/suggest/src/metrics.rs vendored Normal file
View File

@@ -0,0 +1,97 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use std::time::Instant;
/// Single sample for a Glean labeled_timing_distribution
pub struct LabeledTimingSample {
pub label: String,
pub value: u64, // time in microseconds
}
impl LabeledTimingSample {
fn new(label: String, value: u64) -> Self {
Self { label, value }
}
}
/// Ingestion metrics
///
/// These are recorded during [crate::Store::ingest] and returned to the consumer to record.
#[derive(Default)]
pub struct SuggestIngestionMetrics {
pub ingestion_times: Vec<LabeledTimingSample>,
pub download_times: Vec<LabeledTimingSample>,
}
impl SuggestIngestionMetrics {
/// Wraps each iteration in `ingest` and records the time for it.
///
/// Passes the closure a DownloadTimer. Use this to measure the times for all
/// downloads that happen during the ingest
pub fn measure_ingest<F, T>(&mut self, record_type: impl Into<String>, operation: F) -> T
where
F: FnOnce(&mut DownloadTimer) -> T,
{
let timer = Instant::now();
let record_type = record_type.into();
let mut download_metrics = DownloadTimer::default();
let result = operation(&mut download_metrics);
let elapsed = timer.elapsed().as_micros() as u64;
self.ingestion_times.push(LabeledTimingSample::new(
record_type.clone(),
elapsed - download_metrics.total_time,
));
self.download_times.push(LabeledTimingSample::new(
record_type,
download_metrics.total_time,
));
result
}
}
/// Records download times for a single loop in ingest
///
/// [Self::measure_download] can be called multiple times. [DownloadTimer] will track the total
/// time for all calls.
#[derive(Default)]
pub struct DownloadTimer {
total_time: u64,
}
impl DownloadTimer {
pub fn measure_download<F, T>(&mut self, operation: F) -> T
where
F: FnOnce() -> T,
{
let timer = Instant::now();
let result = operation();
self.total_time += timer.elapsed().as_micros() as u64;
result
}
}
/// Query metrics
///
/// These are recorded during [crate::Store::query] and returned to the consumer to record.
#[derive(Default)]
pub struct SuggestQueryMetrics {
pub times: Vec<LabeledTimingSample>,
}
impl SuggestQueryMetrics {
pub fn measure_query<F, T>(&mut self, provider: impl Into<String>, operation: F) -> T
where
F: FnOnce() -> T,
{
let provider = provider.into();
let timer = Instant::now();
// Make sure the compiler doesn't reorder/inline in a way that invalidates this
// measurement.
let result = std::hint::black_box(operation());
let elapsed = timer.elapsed().as_micros() as u64;
self.times.push(LabeledTimingSample::new(provider, elapsed));
result
}
}

73
third_party/rust/suggest/src/provider.rs vendored
View File

@@ -3,6 +3,8 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
use std::fmt;
use rusqlite::{
types::{FromSql, FromSqlError, FromSqlResult, ToSql, ToSqlOutput, ValueRef},
Result as RusqliteResult,
@@ -25,6 +27,22 @@ pub enum SuggestionProvider {
Fakespot = 9,
}
impl fmt::Display for SuggestionProvider {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Amp => write!(f, "amp"),
Self::Wikipedia => write!(f, "wikipedia"),
Self::Amo => write!(f, "amo"),
Self::Pocket => write!(f, "pocket"),
Self::Yelp => write!(f, "yelp"),
Self::Mdn => write!(f, "mdn"),
Self::Weather => write!(f, "weather"),
Self::AmpMobile => write!(f, "ampmobile"),
Self::Fakespot => write!(f, "fakespot"),
}
}
}
impl FromSql for SuggestionProvider {
fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
let v = value.as_i64()?;
@@ -66,52 +84,17 @@ impl SuggestionProvider {
}
}
pub(crate) fn records_for_provider(&self) -> Vec<SuggestRecordType> {
pub(crate) fn record_type(&self) -> SuggestRecordType {
match self {
SuggestionProvider::Amp => {
vec![
SuggestRecordType::AmpWikipedia,
SuggestRecordType::Icon,
SuggestRecordType::GlobalConfig,
]
}
SuggestionProvider::Wikipedia => {
vec![
SuggestRecordType::AmpWikipedia,
SuggestRecordType::Icon,
SuggestRecordType::GlobalConfig,
]
}
SuggestionProvider::Amo => {
vec![SuggestRecordType::Amo, SuggestRecordType::GlobalConfig]
}
SuggestionProvider::Pocket => {
vec![SuggestRecordType::Pocket, SuggestRecordType::GlobalConfig]
}
SuggestionProvider::Yelp => {
vec![
SuggestRecordType::Yelp,
SuggestRecordType::Icon,
SuggestRecordType::GlobalConfig,
]
}
SuggestionProvider::Mdn => {
vec![SuggestRecordType::Mdn, SuggestRecordType::GlobalConfig]
}
SuggestionProvider::Weather => {
vec![SuggestRecordType::Weather, SuggestRecordType::GlobalConfig]
}
SuggestionProvider::AmpMobile => {
vec![
SuggestRecordType::AmpMobile,
SuggestRecordType::AmpWikipedia,
SuggestRecordType::Icon,
SuggestRecordType::GlobalConfig,
]
}
SuggestionProvider::Fakespot => {
vec![SuggestRecordType::Fakespot]
}
SuggestionProvider::Amp => SuggestRecordType::AmpWikipedia,
SuggestionProvider::Wikipedia => SuggestRecordType::AmpWikipedia,
SuggestionProvider::Amo => SuggestRecordType::Amo,
SuggestionProvider::Pocket => SuggestRecordType::Pocket,
SuggestionProvider::Yelp => SuggestRecordType::Yelp,
SuggestionProvider::Mdn => SuggestRecordType::Mdn,
SuggestionProvider::Weather => SuggestRecordType::Weather,
SuggestionProvider::AmpMobile => SuggestRecordType::AmpMobile,
SuggestionProvider::Fakespot => SuggestRecordType::Fakespot,
}
}
}

7
third_party/rust/suggest/src/query.rs vendored
View File

@@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use crate::SuggestionProvider;
use crate::{LabeledTimingSample, Suggestion, SuggestionProvider};
/// A query for suggestions to show in the address bar.
#[derive(Clone, Debug, Default)]
@@ -12,6 +12,11 @@ pub struct SuggestionQuery {
pub limit: Option<i32>,
}
pub struct QueryWithMetricsResult {
pub suggestions: Vec<Suggestion>,
pub query_times: Vec<LabeledTimingSample>,
}
impl SuggestionQuery {
// Builder style methods for creating queries (mostly used by the test code)

268
third_party/rust/suggest/src/rs.rs vendored
View File

@@ -31,18 +31,12 @@
//! the new suggestion in their results, and return `Suggestion::T` variants
//! as needed.
use std::{borrow::Cow, fmt};
use std::fmt;
use remote_settings::{Attachment, GetItemsOptions, RemoteSettingsRecord, RsJsonObject, SortOrder};
use remote_settings::{Attachment, RemoteSettingsRecord};
use serde::{Deserialize, Deserializer};
use crate::{error::Error, provider::SuggestionProvider, Result};
/// The maximum number of suggestions in a Suggest record's attachment.
///
/// This should be the same as the `BUCKET_SIZE` constant in the
/// `mozilla-services/quicksuggest-rs` repo.
pub(crate) const SUGGESTIONS_PER_ATTACHMENT: u64 = 200;
use crate::{db::SuggestDao, error::Error, provider::SuggestionProvider, Result};
/// A list of default record types to download if nothing is specified.
/// This defaults to all record types available as-of Fx128.
@@ -60,12 +54,37 @@ pub(crate) const DEFAULT_RECORDS_TYPES: [SuggestRecordType; 9] = [
SuggestRecordType::AmpMobile,
];
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Collection {
Quicksuggest,
Fakespot,
}
impl Collection {
pub fn name(&self) -> &'static str {
match self {
Self::Quicksuggest => "quicksuggest",
Self::Fakespot => "fakespot-suggest-products",
}
}
}
/// A trait for a client that downloads suggestions from Remote Settings.
///
/// This trait lets tests use a mock client.
pub(crate) trait Client {
/// Fetch a list of records and attachment data
fn get_records(&self, request: RecordRequest) -> Result<Vec<Record>>;
/// Get all records from the server
///
/// We use this plus client-side filtering rather than any server-side filtering, as
/// recommended by the remote settings docs
/// (https://remote-settings.readthedocs.io/en/stable/client-specifications.html). This is
/// relatively inexpensive since we use a cache and don't fetch attachments until after the
/// client-side filtering.
///
/// Records that can't be parsed as [SuggestRecord] are ignored.
fn get_records(&self, collection: Collection, dao: &mut SuggestDao) -> Result<Vec<Record>>;
fn download_attachment(&self, record: &Record) -> Result<Vec<u8>>;
}
/// Implements the [Client] trait using a real remote settings client
@@ -99,97 +118,77 @@ impl RemoteSettingsClient {
})
}
fn client_for_record_type(&self, record_type: &str) -> &remote_settings::Client {
match record_type {
"fakespot-suggestions" => &self.fakespot_client,
_ => &self.quicksuggest_client,
fn client_for_collection(&self, collection: Collection) -> &remote_settings::Client {
match collection {
Collection::Fakespot => &self.fakespot_client,
Collection::Quicksuggest => &self.quicksuggest_client,
}
}
}
impl Client for RemoteSettingsClient {
fn get_records(&self, request: RecordRequest) -> Result<Vec<Record>> {
let client = self.client_for_record_type(request.record_type.as_str());
let options = request.into();
client
.get_records_with_options(&options)?
fn get_records(&self, collection: Collection, dao: &mut SuggestDao) -> Result<Vec<Record>> {
// For now, handle the cache manually. Once 6328 is merged, we should be able to delegate
// this to remote_settings.
let client = self.client_for_collection(collection);
let cache = dao.read_cached_rs_data(collection.name());
let last_modified = match &cache {
Some(response) => response.last_modified,
None => 0,
};
let response = match cache {
None => client.get_records()?,
Some(cache) => remote_settings::cache::merge_cache_and_response(
cache,
client.get_records_since(last_modified)?,
),
};
if last_modified != response.last_modified {
dao.write_cached_rs_data(collection.name(), &response);
}
Ok(response
.records
.into_iter()
.map(|record| {
let attachment_data = record
.attachment
.as_ref()
.map(|a| client.get_attachment(&a.location))
.transpose()?;
Ok(Record::new(record, attachment_data))
})
.collect()
.filter_map(|r| Record::new(r, collection).ok())
.collect())
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
pub struct RecordRequest {
pub record_type: String,
pub last_modified: Option<u64>,
pub limit: Option<u64>,
}
impl From<RecordRequest> for GetItemsOptions {
fn from(value: RecordRequest) -> Self {
let mut options = GetItemsOptions::new();
// Remote Settings returns records in descending modification order
// (newest first), but we want them in ascending order (oldest first),
// so that we can eventually resume downloading where we left off.
options.sort("last_modified", SortOrder::Ascending);
options.filter_eq("type", value.record_type);
if let Some(last_modified) = value.last_modified {
options.filter_gt("last_modified", last_modified.to_string());
fn download_attachment(&self, record: &Record) -> Result<Vec<u8>> {
match &record.attachment {
Some(a) => Ok(self
.client_for_collection(record.collection)
.get_attachment(&a.location)?),
None => Err(Error::MissingAttachment(record.id.to_string())),
}
if let Some(limit) = value.limit {
// Each record's attachment has 200 suggestions, so download enough
// records to cover the requested maximum.
options.limit((limit.saturating_sub(1) / SUGGESTIONS_PER_ATTACHMENT) + 1);
}
options
}
}
/// Remote settings record for suggest.
///
/// This is `remote_settings::RemoteSettingsRecord`, plus the downloaded attachment data.
#[derive(Clone, Debug, Default)]
pub struct Record {
pub id: String,
/// This is a `remote_settings::RemoteSettingsRecord` parsed for suggest.
#[derive(Clone, Debug)]
pub(crate) struct Record {
pub id: SuggestRecordId,
pub last_modified: u64,
pub deleted: bool,
pub attachment: Option<Attachment>,
pub fields: RsJsonObject,
pub attachment_data: Option<Vec<u8>>,
pub payload: SuggestRecord,
pub collection: Collection,
}
impl Record {
pub fn new(record: RemoteSettingsRecord, attachment_data: Option<Vec<u8>>) -> Self {
Self {
id: record.id,
deleted: record.deleted,
fields: record.fields,
pub fn new(record: RemoteSettingsRecord, collection: Collection) -> Result<Self> {
Ok(Self {
id: SuggestRecordId::new(record.id),
last_modified: record.last_modified,
attachment: record.attachment,
attachment_data,
}
payload: serde_json::from_value(serde_json::Value::Object(record.fields))?,
collection,
})
}
/// Get the attachment data for this record, returning an error if it's not present.
///
/// This is indented to be used in cases where the attachment data is required.
pub fn require_attachment_data(&self) -> Result<&[u8]> {
self.attachment_data
.as_deref()
.ok_or_else(|| Error::MissingAttachment(self.id.clone()))
pub fn record_type(&self) -> SuggestRecordType {
(&self.payload).into()
}
}
@@ -225,7 +224,7 @@ pub(crate) enum SuggestRecord {
/// Enum for the different record types that can be consumed.
/// Extracting this from the serialization enum so that we can
/// extend it to get type metadata.
#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord)]
#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
pub enum SuggestRecordType {
Icon,
AmpWikipedia,
@@ -239,8 +238,8 @@ pub enum SuggestRecordType {
Fakespot,
}
impl From<SuggestRecord> for SuggestRecordType {
fn from(suggest_record: SuggestRecord) -> Self {
impl From<&SuggestRecord> for SuggestRecordType {
fn from(suggest_record: &SuggestRecord) -> Self {
match suggest_record {
SuggestRecord::Amo => Self::Amo,
SuggestRecord::AmpWikipedia => Self::AmpWikipedia,
@@ -258,25 +257,50 @@ impl From<SuggestRecord> for SuggestRecordType {
impl fmt::Display for SuggestRecordType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Icon => write!(f, "icon"),
Self::AmpWikipedia => write!(f, "data"),
Self::Amo => write!(f, "amo-suggestions"),
Self::Pocket => write!(f, "pocket-suggestions"),
Self::Yelp => write!(f, "yelp-suggestions"),
Self::Mdn => write!(f, "mdn-suggestions"),
Self::Weather => write!(f, "weather"),
Self::GlobalConfig => write!(f, "configuration"),
Self::AmpMobile => write!(f, "amp-mobile-suggestions"),
Self::Fakespot => write!(f, "fakespot-suggestions"),
}
write!(f, "{}", self.as_str())
}
}
impl SuggestRecordType {
/// Return the meta key for the last ingested record.
pub fn last_ingest_meta_key(&self) -> String {
format!("last_quicksuggest_ingest_{}", self)
/// Get all record types to iterate over
///
/// Currently only used by tests
#[cfg(test)]
pub fn all() -> &'static [SuggestRecordType] {
&[
Self::Icon,
Self::AmpWikipedia,
Self::Amo,
Self::Pocket,
Self::Yelp,
Self::Mdn,
Self::Weather,
Self::GlobalConfig,
Self::AmpMobile,
Self::Fakespot,
]
}
pub fn as_str(&self) -> &str {
match self {
Self::Icon => "icon",
Self::AmpWikipedia => "data",
Self::Amo => "amo-suggestions",
Self::Pocket => "pocket-suggestions",
Self::Yelp => "yelp-suggestions",
Self::Mdn => "mdn-suggestions",
Self::Weather => "weather",
Self::GlobalConfig => "configuration",
Self::AmpMobile => "amp-mobile-suggestions",
Self::Fakespot => "fakespot-suggestions",
}
}
pub fn collection(&self) -> Collection {
match self {
Self::Fakespot => Collection::Fakespot,
_ => Collection::Quicksuggest,
}
}
}
@@ -307,9 +331,13 @@ impl<T> SuggestAttachment<T> {
/// The ID of a record in the Suggest Remote Settings collection.
#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd)]
#[serde(transparent)]
pub(crate) struct SuggestRecordId<'a>(Cow<'a, str>);
pub(crate) struct SuggestRecordId(String);
impl SuggestRecordId {
pub fn new(id: String) -> Self {
Self(id)
}
impl<'a> SuggestRecordId<'a> {
pub fn as_str(&self) -> &str {
&self.0
}
@@ -324,12 +352,9 @@ impl<'a> SuggestRecordId<'a> {
}
}
impl<'a, T> From<T> for SuggestRecordId<'a>
where
T: Into<Cow<'a, str>>,
{
fn from(value: T) -> Self {
Self(value.into())
impl fmt::Display for SuggestRecordId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.0)
}
}
@@ -524,6 +549,8 @@ pub(crate) struct DownloadedMdnSuggestion {
pub(crate) struct DownloadedFakespotSuggestion {
pub fakespot_grade: String,
pub product_id: String,
pub keywords: String,
pub product_type: String,
pub rating: f64,
pub score: f64,
pub title: String,
@@ -679,37 +706,4 @@ mod test {
],
);
}
#[test]
fn test_remote_settings_limits() {
fn check_limit(suggestion_limit: Option<u64>, expected_record_limit: Option<&str>) {
let request = RecordRequest {
limit: suggestion_limit,
..RecordRequest::default()
};
let options: GetItemsOptions = request.into();
let actual_record_limit = options
.iter_query_pairs()
.find_map(|(name, value)| (name == "_limit").then(|| value.to_string()));
assert_eq!(
actual_record_limit.as_deref(),
expected_record_limit,
"expected record limit = {:?} for suggestion limit {:?}; actual = {:?}",
expected_record_limit,
suggestion_limit,
actual_record_limit
);
}
check_limit(None, None);
// 200 suggestions per record, so test with numbers around that
// boundary.
check_limit(Some(0), Some("1"));
check_limit(Some(199), Some("1"));
check_limit(Some(200), Some("1"));
check_limit(Some(201), Some("2"));
check_limit(Some(300), Some("2"));
check_limit(Some(400), Some("2"));
check_limit(Some(401), Some("3"));
}
}

81
third_party/rust/suggest/src/schema.rs vendored
View File

@@ -19,7 +19,7 @@ use sql_support::{
/// [`SuggestConnectionInitializer::upgrade_from`].
/// a. If suggestions should be re-ingested after the migration, call `clear_database()` inside
/// the migration.
pub const VERSION: u32 = 22;
pub const VERSION: u32 = 25;
/// The current Suggest database schema.
pub const SQL: &str = "
@@ -28,6 +28,19 @@ CREATE TABLE meta(
value NOT NULL
) WITHOUT ROWID;
CREATE TABLE rs_cache(
collection TEXT PRIMARY KEY,
data TEXT NOT NULL
) WITHOUT ROWID;
CREATE TABLE ingested_records(
id TEXT,
collection TEXT,
type TEXT NOT NULL,
last_modified INTEGER NOT NULL,
PRIMARY KEY (id, collection)
) WITHOUT ROWID;
CREATE TABLE keywords(
keyword TEXT NOT NULL,
suggestion_id INTEGER NOT NULL,
@@ -93,6 +106,8 @@ CREATE TABLE fakespot_custom_details(
suggestion_id INTEGER PRIMARY KEY,
fakespot_grade TEXT NOT NULL,
product_id TEXT NOT NULL,
keywords TEXT NOT NULL,
product_type TEXT NOT NULL,
rating REAL NOT NULL,
total_reviews INTEGER NOT NULL,
icon_id TEXT,
@@ -101,7 +116,6 @@ CREATE TABLE fakespot_custom_details(
CREATE VIRTUAL TABLE IF NOT EXISTS fakespot_fts USING FTS5(
title,
prefix='4 5 6 7 8 9 10 11',
content='',
contentless_delete=1,
tokenize=\"porter unicode61 remove_diacritics 2 tokenchars '''-'\"
@@ -346,6 +360,69 @@ END;
)?;
Ok(())
}
22 => {
// Drop and re-create the fakespot_fts table to remove the prefix index param
tx.execute_batch(
"
DROP TABLE fakespot_fts;
CREATE VIRTUAL TABLE fakespot_fts USING FTS5(
title,
content='',
contentless_delete=1,
tokenize=\"porter unicode61 remove_diacritics 2 tokenchars '''-'\"
);
",
)?;
Ok(())
}
23 => {
// Drop all suggestions, then recreate the fakespot_custom_details table to add the
// `keywords` and `product_type` fields.
clear_database(tx)?;
tx.execute_batch(
"
DROP TABLE fakespot_custom_details;
CREATE TABLE fakespot_custom_details(
suggestion_id INTEGER PRIMARY KEY,
fakespot_grade TEXT NOT NULL,
product_id TEXT NOT NULL,
keywords TEXT NOT NULL,
product_type TEXT NOT NULL,
rating REAL NOT NULL,
total_reviews INTEGER NOT NULL,
icon_id TEXT,
FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE
);
CREATE TRIGGER fakespot_ai AFTER INSERT ON fakespot_custom_details BEGIN
INSERT INTO fakespot_fts(rowid, title)
SELECT id, title
FROM suggestions
WHERE id = new.suggestion_id;
END;
",
)?;
Ok(())
}
24 => {
// Clear the database so that we re-ingest and populate the ingested_records table.
clear_database(tx)?;
tx.execute_batch(
"
CREATE TABLE rs_cache(
collection TEXT PRIMARY KEY,
data TEXT NOT NULL
) WITHOUT ROWID;
CREATE TABLE ingested_records(
id TEXT,
collection TEXT,
type TEXT NOT NULL,
last_modified INTEGER NOT NULL,
PRIMARY KEY (id, collection)
) WITHOUT ROWID;
",
)?;
Ok(())
}
_ => Err(open_database::Error::IncompatibleVersion(version)),
}
}

724
third_party/rust/suggest/src/store.rs vendored
View File

@@ -4,7 +4,7 @@
*/
use std::{
collections::BTreeSet,
collections::{hash_map::Entry, BTreeSet, HashMap, HashSet},
path::{Path, PathBuf},
sync::Arc,
};
@@ -18,14 +18,16 @@ use serde::de::DeserializeOwned;
use crate::{
config::{SuggestGlobalConfig, SuggestProviderConfig},
db::{ConnectionType, Sqlite3Extension, SuggestDao, SuggestDb},
db::{ConnectionType, IngestedRecord, Sqlite3Extension, SuggestDao, SuggestDb},
error::Error,
metrics::{DownloadTimer, SuggestIngestionMetrics, SuggestQueryMetrics},
provider::SuggestionProvider,
rs::{
Client, Record, RecordRequest, RemoteSettingsClient, SuggestAttachment, SuggestRecord,
Client, Collection, Record, RemoteSettingsClient, SuggestAttachment, SuggestRecord,
SuggestRecordId, SuggestRecordType, DEFAULT_RECORDS_TYPES,
},
Result, SuggestApiResult, Suggestion, SuggestionQuery,
suggestion::AmpSuggestionType,
QueryWithMetricsResult, Result, SuggestApiResult, Suggestion, SuggestionQuery,
};
/// Builder for [SuggestStore]
@@ -177,6 +179,15 @@ impl SuggestStore {
/// Queries the database for suggestions.
#[handle_error(Error)]
pub fn query(&self, query: SuggestionQuery) -> SuggestApiResult<Vec<Suggestion>> {
Ok(self.inner.query(query)?.suggestions)
}
/// Queries the database for suggestions.
#[handle_error(Error)]
pub fn query_with_metrics(
&self,
query: SuggestionQuery,
) -> SuggestApiResult<QueryWithMetricsResult> {
self.inner.query(query)
}
@@ -207,7 +218,10 @@ impl SuggestStore {
/// Ingests new suggestions from Remote Settings.
#[handle_error(Error)]
pub fn ingest(&self, constraints: SuggestIngestionConstraints) -> SuggestApiResult<()> {
pub fn ingest(
&self,
constraints: SuggestIngestionConstraints,
) -> SuggestApiResult<SuggestIngestionMetrics> {
self.inner.ingest(constraints)
}
@@ -231,17 +245,15 @@ impl SuggestStore {
) -> SuggestApiResult<Option<SuggestProviderConfig>> {
self.inner.fetch_provider_config(provider)
}
pub fn force_reingest(&self) {
self.inner.force_reingest()
}
}
/// Constraints limit which suggestions to ingest from Remote Settings.
#[derive(Clone, Default, Debug)]
pub struct SuggestIngestionConstraints {
/// The approximate maximum number of suggestions to ingest. Set to [`None`]
/// for "no limit".
///
/// Because of how suggestions are partitioned in Remote Settings, this is a
/// soft limit, and the store might ingest more than requested.
pub max_suggestions: Option<u64>,
pub providers: Option<Vec<SuggestionProvider>>,
/// Only run ingestion if the table `suggestions` is empty
pub empty_only: bool,
@@ -302,11 +314,45 @@ impl<S> SuggestStoreInner<S> {
.get_or_try_init(|| SuggestStoreDbs::open(&self.data_path, &self.extensions_to_load))
}
fn query(&self, query: SuggestionQuery) -> Result<Vec<Suggestion>> {
if query.keyword.is_empty() || query.providers.is_empty() {
return Ok(Vec::new());
fn query(&self, query: SuggestionQuery) -> Result<QueryWithMetricsResult> {
let mut metrics = SuggestQueryMetrics::default();
let mut suggestions = vec![];
let unique_providers = query.providers.iter().collect::<HashSet<_>>();
let reader = &self.dbs()?.reader;
for provider in unique_providers {
let new_suggestions = metrics.measure_query(provider.to_string(), || {
reader.read(|dao| match provider {
SuggestionProvider::Amp => {
dao.fetch_amp_suggestions(&query, AmpSuggestionType::Desktop)
}
SuggestionProvider::AmpMobile => {
dao.fetch_amp_suggestions(&query, AmpSuggestionType::Mobile)
}
SuggestionProvider::Wikipedia => dao.fetch_wikipedia_suggestions(&query),
SuggestionProvider::Amo => dao.fetch_amo_suggestions(&query),
SuggestionProvider::Pocket => dao.fetch_pocket_suggestions(&query),
SuggestionProvider::Yelp => dao.fetch_yelp_suggestions(&query),
SuggestionProvider::Mdn => dao.fetch_mdn_suggestions(&query),
SuggestionProvider::Weather => dao.fetch_weather_suggestions(&query),
SuggestionProvider::Fakespot => dao.fetch_fakespot_suggestions(&query),
})
})?;
suggestions.extend(new_suggestions);
}
self.dbs()?.reader.read(|dao| dao.fetch_suggestions(&query))
// Note: it's important that this is a stable sort to keep the intra-provider order stable.
// For example, we can return multiple fakespot-suggestions all with `score=0.245`. In
// that case, they must be in the same order that `fetch_fakespot_suggestions` returned
// them in.
suggestions.sort();
if let Some(limit) = query.limit.and_then(|limit| usize::try_from(limit).ok()) {
suggestions.truncate(limit);
}
Ok(QueryWithMetricsResult {
suggestions,
query_times: metrics.times,
})
}
fn dismiss_suggestion(&self, suggestion_url: String) -> Result<()> {
@@ -354,141 +400,210 @@ impl<S> SuggestStoreInner<S> {
.reader
.read(|dao| dao.get_provider_config(provider))
}
// Cause the next ingestion to re-ingest all data
pub fn force_reingest(&self) {
let writer = &self.dbs().unwrap().writer;
writer.write(|dao| dao.force_reingest()).unwrap();
}
}
impl<S> SuggestStoreInner<S>
where
S: Client,
{
pub fn ingest(&self, constraints: SuggestIngestionConstraints) -> Result<()> {
pub fn ingest(
&self,
constraints: SuggestIngestionConstraints,
) -> Result<SuggestIngestionMetrics> {
breadcrumb!("Ingestion starting");
let writer = &self.dbs()?.writer;
let mut metrics = SuggestIngestionMetrics::default();
if constraints.empty_only && !writer.read(|dao| dao.suggestions_table_empty())? {
return Ok(());
return Ok(metrics);
}
// use std::collections::BTreeSet;
// Figure out which record types we're ingesting
let ingest_record_types = if let Some(rt) = &constraints.providers {
rt.iter()
.flat_map(|x| x.records_for_provider())
.map(|x| x.record_type())
// Always ingest these types
.chain([SuggestRecordType::Icon, SuggestRecordType::GlobalConfig])
.collect::<BTreeSet<_>>()
.into_iter()
.collect()
} else {
DEFAULT_RECORDS_TYPES.to_vec()
DEFAULT_RECORDS_TYPES.into_iter().collect()
};
// Handle ingestion inside single write scope
// Group record types by collection
let mut record_types_by_collection = HashMap::<Collection, Vec<SuggestRecordType>>::new();
for record_type in ingest_record_types {
record_types_by_collection
.entry(record_type.collection())
.or_default()
.push(record_type);
}
// Create a single write scope for all DB operations
let mut write_scope = writer.write_scope()?;
for ingest_record_type in ingest_record_types {
breadcrumb!("Ingesting {ingest_record_type}");
write_scope
.write(|dao| self.ingest_records_by_type(ingest_record_type, dao, &constraints))?;
write_scope.err_if_interrupted()?;
// Read the previously ingested records. We use this to calculate what's changed
let ingested_records = write_scope.read(|dao| dao.get_ingested_records())?;
// For each collection, fetch all records
for (collection, record_types) in record_types_by_collection {
breadcrumb!("Ingesting collection {}", collection.name());
let records =
write_scope.write(|dao| self.settings_client.get_records(collection, dao))?;
// For each record type in that collection, calculate the changes and pass them to
// [Self::ingest_records]
for record_type in record_types {
breadcrumb!("Ingesting {record_type}");
metrics.measure_ingest(record_type.to_string(), |download_timer| {
let changes = RecordChanges::new(
records.iter().filter(|r| r.record_type() == record_type),
ingested_records.iter().filter(|i| {
i.record_type == record_type.as_str()
&& i.collection == collection.name()
}),
);
write_scope
.write(|dao| self.ingest_records(dao, collection, changes, download_timer))
})?;
write_scope.err_if_interrupted()?;
}
}
breadcrumb!("Ingestion complete");
Ok(())
Ok(metrics)
}
fn ingest_records_by_type(
fn ingest_records(
&self,
ingest_record_type: SuggestRecordType,
dao: &mut SuggestDao,
constraints: &SuggestIngestionConstraints,
collection: Collection,
changes: RecordChanges<'_>,
download_timer: &mut DownloadTimer,
) -> Result<()> {
let last_ingest_key = ingest_record_type.last_ingest_meta_key();
let request = RecordRequest {
record_type: ingest_record_type.to_string(),
last_modified: dao.get_meta::<u64>(&last_ingest_key)?,
limit: constraints.max_suggestions,
};
let records = self.settings_client.get_records(request)?;
for record in &records {
for record in &changes.new {
log::trace!("Ingesting: {}", record.id.as_str());
self.ingest_record(dao, record, download_timer)?;
}
for record in &changes.updated {
// Drop any data that we previously ingested from this record.
// Suggestions in particular don't have a stable identifier, and
// determining which suggestions in the record actually changed is
// more complicated than dropping and re-ingesting all of them.
dao.delete_record_data(record)?;
log::trace!("Reingesting: {}", record.id.as_str());
dao.delete_record_data(&record.id)?;
self.ingest_record(dao, record, download_timer)?;
}
self.ingest_records(dao, &records)?;
if let Some(max_last_modified) = records.iter().map(|r| r.last_modified).max() {
dao.put_last_ingest_if_newer(&last_ingest_key, max_last_modified)?;
for record in &changes.deleted {
log::trace!("Deleting: {:?}", record.id);
dao.delete_record_data(&record.id)?;
}
dao.update_ingested_records(
collection.name(),
&changes.new,
&changes.updated,
&changes.deleted,
)?;
Ok(())
}
fn ingest_records(&self, dao: &mut SuggestDao, records: &[Record]) -> Result<()> {
for record in records {
if record.deleted {
continue;
}
let record_id = SuggestRecordId::from(&record.id);
let Ok(fields) =
serde_json::from_value(serde_json::Value::Object(record.fields.clone()))
else {
// We don't recognize this record's type, so we don't know how
// to ingest its suggestions. Skip processing this record.
continue;
};
match fields {
SuggestRecord::AmpWikipedia => {
self.ingest_attachment(dao, record, |dao, record_id, suggestions| {
fn ingest_record(
&self,
dao: &mut SuggestDao,
record: &Record,
download_timer: &mut DownloadTimer,
) -> Result<()> {
match &record.payload {
SuggestRecord::AmpWikipedia => {
self.ingest_attachment(
dao,
record,
download_timer,
|dao, record_id, suggestions| {
dao.insert_amp_wikipedia_suggestions(record_id, suggestions)
})?;
}
SuggestRecord::AmpMobile => {
self.ingest_attachment(dao, record, |dao, record_id, suggestions| {
},
)?;
}
SuggestRecord::AmpMobile => {
self.ingest_attachment(
dao,
record,
download_timer,
|dao, record_id, suggestions| {
dao.insert_amp_mobile_suggestions(record_id, suggestions)
})?;
}
SuggestRecord::Icon => {
let (Some(icon_id), Some(attachment)) =
(record_id.as_icon_id(), record.attachment.as_ref())
else {
// An icon record should have an icon ID and an
// attachment. Icons that don't have these are
// malformed, so skip to the next record.
continue;
};
let data = record.require_attachment_data()?;
dao.put_icon(icon_id, data, &attachment.mimetype)?;
}
SuggestRecord::Amo => {
self.ingest_attachment(dao, record, |dao, record_id, suggestions| {
},
)?;
}
SuggestRecord::Icon => {
let (Some(icon_id), Some(attachment)) =
(record.id.as_icon_id(), record.attachment.as_ref())
else {
// An icon record should have an icon ID and an
// attachment. Icons that don't have these are
// malformed, so skip to the next record.
return Ok(());
};
let data = self.settings_client.download_attachment(record)?;
dao.put_icon(icon_id, &data, &attachment.mimetype)?;
}
SuggestRecord::Amo => {
self.ingest_attachment(
dao,
record,
download_timer,
|dao, record_id, suggestions| {
dao.insert_amo_suggestions(record_id, suggestions)
})?;
}
SuggestRecord::Pocket => {
self.ingest_attachment(dao, record, |dao, record_id, suggestions| {
},
)?;
}
SuggestRecord::Pocket => {
self.ingest_attachment(
dao,
record,
download_timer,
|dao, record_id, suggestions| {
dao.insert_pocket_suggestions(record_id, suggestions)
})?;
}
SuggestRecord::Yelp => {
self.ingest_attachment(dao, record, |dao, record_id, suggestions| {
match suggestions.first() {
Some(suggestion) => dao.insert_yelp_suggestions(record_id, suggestion),
None => Ok(()),
}
})?;
}
SuggestRecord::Mdn => {
self.ingest_attachment(dao, record, |dao, record_id, suggestions| {
},
)?;
}
SuggestRecord::Yelp => {
self.ingest_attachment(
dao,
record,
download_timer,
|dao, record_id, suggestions| match suggestions.first() {
Some(suggestion) => dao.insert_yelp_suggestions(record_id, suggestion),
None => Ok(()),
},
)?;
}
SuggestRecord::Mdn => {
self.ingest_attachment(
dao,
record,
download_timer,
|dao, record_id, suggestions| {
dao.insert_mdn_suggestions(record_id, suggestions)
})?;
}
SuggestRecord::Weather(data) => dao.insert_weather_data(&record_id, &data)?,
SuggestRecord::GlobalConfig(config) => {
dao.put_global_config(&SuggestGlobalConfig::from(&config))?
}
SuggestRecord::Fakespot => {
self.ingest_attachment(dao, record, |dao, record_id, suggestions| {
},
)?;
}
SuggestRecord::Weather(data) => dao.insert_weather_data(&record.id, data)?,
SuggestRecord::GlobalConfig(config) => {
dao.put_global_config(&SuggestGlobalConfig::from(config))?
}
SuggestRecord::Fakespot => {
self.ingest_attachment(
dao,
record,
download_timer,
|dao, record_id, suggestions| {
dao.insert_fakespot_suggestions(record_id, suggestions)
})?;
}
},
)?;
}
}
Ok(())
@@ -498,6 +613,7 @@ where
&self,
dao: &mut SuggestDao,
record: &Record,
download_timer: &mut DownloadTimer,
ingestion_handler: impl FnOnce(&mut SuggestDao<'_>, &SuggestRecordId, &[T]) -> Result<()>,
) -> Result<()>
where
@@ -507,13 +623,10 @@ where
return Ok(());
};
let attachment_data = record.require_attachment_data()?;
match serde_json::from_slice::<SuggestAttachment<T>>(attachment_data) {
Ok(attachment) => ingestion_handler(
dao,
&SuggestRecordId::from(&record.id),
attachment.suggestions(),
),
let attachment_data =
download_timer.measure_download(|| self.settings_client.download_attachment(record))?;
match serde_json::from_slice::<SuggestAttachment<T>>(&attachment_data) {
Ok(attachment) => ingestion_handler(dao, &record.id, attachment.suggestions()),
// If the attachment doesn't match our expected schema, just skip it. It's possible
// that we're using an older version. If so, we'll get the data when we re-ingest
// after updating the schema.
@@ -522,6 +635,44 @@ where
}
}
/// Tracks changes in suggest records since the last ingestion
struct RecordChanges<'a> {
new: Vec<&'a Record>,
updated: Vec<&'a Record>,
deleted: Vec<&'a IngestedRecord>,
}
impl<'a> RecordChanges<'a> {
fn new(
current: impl Iterator<Item = &'a Record>,
previously_ingested: impl Iterator<Item = &'a IngestedRecord>,
) -> Self {
let mut ingested_map: HashMap<&str, &IngestedRecord> =
previously_ingested.map(|i| (i.id.as_str(), i)).collect();
// Iterate through current, finding new/updated records.
// Remove existing records from ingested_map.
let mut new = vec![];
let mut updated = vec![];
for r in current {
match ingested_map.entry(r.id.as_str()) {
Entry::Vacant(_) => new.push(r),
Entry::Occupied(e) => {
if e.remove().last_modified != r.last_modified {
updated.push(r);
}
}
}
}
// Anything left in ingested_map is a deleted record
let deleted = ingested_map.into_values().collect();
Self {
new,
deleted,
updated,
}
}
}
#[cfg(feature = "benchmark_api")]
impl<S> SuggestStoreInner<S>
where
@@ -535,28 +686,30 @@ where
self.dbs().unwrap();
}
pub fn force_reingest(&self, ingest_record_type: SuggestRecordType) {
// To force a re-ingestion, we're going to ingest all records then forget the last
// ingestion time.
self.benchmark_ingest_records_by_type(ingest_record_type);
pub fn ingest_records_by_type(&self, ingest_record_type: SuggestRecordType) {
let writer = &self.dbs().unwrap().writer;
writer
.write(|dao| dao.clear_meta(ingest_record_type.last_ingest_meta_key().as_str()))
let mut timer = DownloadTimer::default();
let ingested_records = writer.read(|dao| dao.get_ingested_records()).unwrap();
let records = writer
.write(|dao| {
self.settings_client
.get_records(ingest_record_type.collection(), dao)
})
.unwrap();
}
pub fn benchmark_ingest_records_by_type(&self, ingest_record_type: SuggestRecordType) {
let writer = &self.dbs().unwrap().writer;
let changes = RecordChanges::new(
records
.iter()
.filter(|r| r.record_type() == ingest_record_type),
ingested_records
.iter()
.filter(|i| i.record_type == ingest_record_type.as_str()),
);
writer
.write(|dao| {
dao.clear_meta(ingest_record_type.last_ingest_meta_key().as_str())?;
self.ingest_records_by_type(
ingest_record_type,
dao,
&SuggestIngestionConstraints::all_providers(),
)
self.ingest_records(dao, ingest_record_type.collection(), changes, &mut timer)
})
.unwrap()
.unwrap();
}
pub fn table_row_counts(&self) -> Vec<(String, u32)> {
@@ -642,22 +795,14 @@ mod tests {
}
}
fn replace_client(&mut self, client: MockRemoteSettingsClient) {
self.inner.settings_client = client;
}
fn last_modified_timestamp(&self) -> u64 {
self.inner.settings_client.last_modified_timestamp
fn client_mut(&mut self) -> &mut MockRemoteSettingsClient {
&mut self.inner.settings_client
}
fn read<T>(&self, op: impl FnOnce(&SuggestDao) -> Result<T>) -> Result<T> {
self.inner.dbs().unwrap().reader.read(op)
}
fn write<T>(&self, op: impl FnOnce(&mut SuggestDao) -> Result<T>) -> Result<T> {
self.inner.dbs().unwrap().writer.write(op)
}
fn count_rows(&self, table_name: &str) -> u64 {
let sql = format!("SELECT count(*) FROM {table_name}");
self.read(|dao| Ok(dao.conn.query_one(&sql)?))
@@ -669,12 +814,7 @@ mod tests {
}
fn fetch_suggestions(&self, query: SuggestionQuery) -> Vec<Suggestion> {
self.inner
.dbs()
.unwrap()
.reader
.read(|dao| Ok(dao.fetch_suggestions(&query).unwrap()))
.unwrap()
self.inner.query(query).unwrap().suggestions
}
pub fn fetch_global_config(&self) -> SuggestGlobalConfig {
@@ -696,7 +836,9 @@ mod tests {
fn before_each() {
static ONCE: Once = Once::new();
ONCE.call_once(|| {
env_logger::init();
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("trace"))
.is_test(true)
.init();
});
}
@@ -749,11 +891,11 @@ mod tests {
assert!(!store.read(|dao| dao.suggestions_table_empty())?);
// This ingestion should not run since the DB is no longer empty
store.replace_client(MockRemoteSettingsClient::default().with_record(
store.client_mut().update_record(
"data",
"1234",
json!([los_pollos_amp(), good_place_eats_amp()]),
));
);
store.ingest(SuggestIngestionConstraints {
empty_only: true,
..SuggestIngestionConstraints::all_providers()
@@ -901,7 +1043,7 @@ mod tests {
store.ingest(SuggestIngestionConstraints::all_providers());
// Update the snapshot with new suggestions: Los pollos has a new name and Good place eats
// is now serving Penne
store.replace_client(MockRemoteSettingsClient::default().with_record(
store.client_mut().update_record(
"data",
"1234",
json!([
@@ -914,7 +1056,7 @@ mod tests {
"url": "https://penne.biz",
}))
]),
));
);
store.ingest(SuggestIngestionConstraints::all_providers());
assert!(matches!(
@@ -952,26 +1094,26 @@ mod tests {
// Reingest with updated icon data
// - Los pollos gets new data and a new id
// - Good place eats gets new data only
store.replace_client(
MockRemoteSettingsClient::default()
.with_record(
"data",
"1234",
json!([
los_pollos_amp().merge(json!({"icon": "1000"})),
good_place_eats_amp()
]),
)
.with_icon(MockIcon {
id: "1000",
data: "new-los-pollos-icon",
..los_pollos_icon()
})
.with_icon(MockIcon {
data: "new-good-place-eats-icon",
..good_place_eats_icon()
}),
);
store
.client_mut()
.update_record(
"data",
"1234",
json!([
los_pollos_amp().merge(json!({"icon": "1000"})),
good_place_eats_amp()
]),
)
.delete_icon(los_pollos_icon())
.add_icon(MockIcon {
id: "1000",
data: "new-los-pollos-icon",
..los_pollos_icon()
})
.update_icon(MockIcon {
data: "new-good-place-eats-icon",
..good_place_eats_icon()
});
store.ingest(SuggestIngestionConstraints::all_providers());
assert!(matches!(
@@ -1019,18 +1161,17 @@ mod tests {
// Update the snapshot with new suggestions: update the second, drop the
// third, and add the fourth.
store.replace_client(
MockRemoteSettingsClient::default()
.with_record("amo-suggestions", "data-1", json!([relay_amo()]))
.with_record(
"amo-suggestions",
"data-2",
json!([
dark_mode_amo().merge(json!({"title": "Updated second suggestion"})),
new_tab_override_amo(),
]),
),
);
store
.client_mut()
.update_record("amo-suggestions", "data-1", json!([relay_amo()]))
.update_record(
"amo-suggestions",
"data-2",
json!([
dark_mode_amo().merge(json!({"title": "Updated second suggestion"})),
new_tab_override_amo(),
]),
);
store.ingest(SuggestIngestionConstraints::all_providers());
assert_eq!(
@@ -1053,10 +1194,9 @@ mod tests {
Ok(())
}
/// Tests ingesting tombstones for previously-ingested suggestions and
/// icons.
/// Tests ingestion when previously-ingested suggestions/icons have been deleted.
#[test]
fn ingest_tombstones() -> anyhow::Result<()> {
fn ingest_with_deletions() -> anyhow::Result<()> {
before_each();
let mut store = TestStore::new(
@@ -1075,16 +1215,12 @@ mod tests {
store.fetch_suggestions(SuggestionQuery::amp("la")),
vec![good_place_eats_suggestion("lasagna")],
);
// Re-ingest with:
// - Los pollos replaced with a tombstone
// - Good place eat's icon replaced with a tombstone
store.replace_client(
MockRemoteSettingsClient::default()
.with_tombstone("data", "data-1")
.with_record("data", "data-2", json!([good_place_eats_amp()]))
.with_icon_tombstone(los_pollos_icon())
.with_icon_tombstone(good_place_eats_icon()),
);
// Re-ingest without los-pollos and good place eat's icon. The suggest store should
// recognize that they're missing and delete them.
store
.client_mut()
.delete_record("quicksuggest", "data-1")
.delete_icon(good_place_eats_icon());
store.ingest(SuggestIngestionConstraints::all_providers());
assert_eq!(store.fetch_suggestions(SuggestionQuery::amp("lo")), vec![]);
@@ -1150,7 +1286,6 @@ mod tests {
json!([burnout_pocket(), multimatch_pocket(),]),
)
.with_record("yelp-suggestions", "data-4", json!([ramen_yelp(),]))
.with_record("yeld-suggestions", "data-4", json!([ramen_yelp(),]))
.with_record("mdn-suggestions", "data-5", json!([array_mdn(),]))
.with_icon(good_place_eats_icon())
.with_icon(california_icon())
@@ -1675,10 +1810,6 @@ mod tests {
store.ingest(SuggestIngestionConstraints::all_providers());
store.read(|dao| {
assert_eq!(
dao.get_meta::<u64>(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
Some(store.last_modified_timestamp())
);
assert_eq!(
dao.conn
.query_one::<i64>("SELECT count(*) FROM suggestions")?,
@@ -1700,24 +1831,11 @@ mod tests {
let store = TestStore::new(
MockRemoteSettingsClient::default()
.with_record("data", "data-1", json!([los_pollos_amp()]))
.with_record("yelp", "yelp-1", json!([ramen_yelp()]))
.with_record("yelp-suggestions", "yelp-1", json!([ramen_yelp()]))
.with_icon(los_pollos_icon()),
);
// Write a last ingestion times to test that we overwrite it properly
store.write(|dao| {
// Check that existing data is updated properly.
dao.put_meta(
SuggestRecordType::AmpWikipedia
.last_ingest_meta_key()
.as_str(),
1,
)?;
Ok(())
})?;
let constraints = SuggestIngestionConstraints {
max_suggestions: Some(100),
providers: Some(vec![SuggestionProvider::Amp, SuggestionProvider::Pocket]),
..SuggestIngestionConstraints::all_providers()
};
@@ -1734,57 +1852,6 @@ mod tests {
vec![]
);
store.read(|dao| {
// This should have its last_modified_timestamp updated, since we ingested an amp
// record
assert_eq!(
dao.get_meta::<u64>(
SuggestRecordType::AmpWikipedia
.last_ingest_meta_key()
.as_str()
)?,
Some(store.last_modified_timestamp())
);
// This should have its last_modified_timestamp updated, since we ingested an icon
assert_eq!(
dao.get_meta::<u64>(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
Some(store.last_modified_timestamp())
);
// This should not have its last_modified_timestamp updated, since there were no pocket
// items to ingest
assert_eq!(
dao.get_meta::<u64>(SuggestRecordType::Pocket.last_ingest_meta_key().as_str())?,
None
);
// This should not have its last_modified_timestamp updated, since we did not ask to
// ingest yelp items.
assert_eq!(
dao.get_meta::<u64>(SuggestRecordType::Yelp.last_ingest_meta_key().as_str())?,
None
);
assert_eq!(
dao.get_meta::<u64>(SuggestRecordType::Amo.last_ingest_meta_key().as_str())?,
None
);
assert_eq!(
dao.get_meta::<u64>(SuggestRecordType::Mdn.last_ingest_meta_key().as_str())?,
None
);
assert_eq!(
dao.get_meta::<u64>(SuggestRecordType::AmpMobile.last_ingest_meta_key().as_str())?,
None
);
assert_eq!(
dao.get_meta::<u64>(
SuggestRecordType::GlobalConfig
.last_ingest_meta_key()
.as_str()
)?,
None
);
Ok(())
})?;
Ok(())
}
@@ -2141,15 +2208,20 @@ mod tests {
store.ingest(SuggestIngestionConstraints::all_providers());
assert_eq!(
store.fetch_suggestions(SuggestionQuery::fakespot("globe")),
vec![snowglobe_suggestion()],
vec![snowglobe_suggestion().with_fakespot_product_type_bonus(0.5)],
);
assert_eq!(
store.fetch_suggestions(SuggestionQuery::fakespot("simpsons")),
vec![simpsons_suggestion()],
);
// The snowglobe suggestion should come before the simpsons one, since `snow` is a partial
// match on the product_type field.
assert_eq!(
store.fetch_suggestions(SuggestionQuery::fakespot("snow")),
vec![simpsons_suggestion(), snowglobe_suggestion()],
vec![
snowglobe_suggestion().with_fakespot_product_type_bonus(0.5),
simpsons_suggestion(),
],
);
// Test FTS by using a query where the keywords are separated in the source text
assert_eq!(
@@ -2165,6 +2237,35 @@ mod tests {
Ok(())
}
#[test]
fn fakespot_keywords() -> anyhow::Result<()> {
before_each();
let store = TestStore::new(
MockRemoteSettingsClient::default()
.with_record(
"fakespot-suggestions",
"fakespot-1",
json!([
// Snow normally returns the snowglobe first. Test using the keyword field
// to force the simpsons result first.
snowglobe_fakespot(),
simpsons_fakespot().merge(json!({"keywords": "snow"})),
]),
)
.with_icon(fakespot_amazon_icon()),
);
store.ingest(SuggestIngestionConstraints::all_providers());
assert_eq!(
store.fetch_suggestions(SuggestionQuery::fakespot("snow")),
vec![
simpsons_suggestion().with_fakespot_keyword_bonus(),
snowglobe_suggestion().with_fakespot_product_type_bonus(0.5),
],
);
Ok(())
}
#[test]
fn fakespot_prefix_matching() -> anyhow::Result<()> {
before_each();
@@ -2213,15 +2314,12 @@ mod tests {
// Update the snapshot so that:
// - The Simpsons entry is deleted
// - Snow globes now use sea glass instead of glitter
store.replace_client(
MockRemoteSettingsClient::default()
.with_record(
"fakespot-suggestions",
"fakespot-1",
json!([snowglobe_fakespot()
.merge(json!({"title": "Make Your Own Sea Glass Snow Globes"}))]),
)
.with_icon(fakespot_amazon_icon()),
store.client_mut().update_record(
"fakespot-suggestions",
"fakespot-1",
json!([
snowglobe_fakespot().merge(json!({"title": "Make Your Own Sea Glass Snow Globes"}))
]),
);
store.ingest(SuggestIngestionConstraints::all_providers());
@@ -2244,4 +2342,72 @@ mod tests {
Ok(())
}
/// Test the pathological case where we ingest records with the same id, but from different
/// collections
#[test]
fn same_record_id_different_collections() -> anyhow::Result<()> {
before_each();
let mut store = TestStore::new(
MockRemoteSettingsClient::default()
// This record is in the fakespot-suggest-products collection
.with_record(
"fakespot-suggestions",
"fakespot-1",
json!([snowglobe_fakespot()]),
)
// This record is in the quicksuggest collection, but it has a fakespot record ID
// for some reason.
.with_record("data", "fakespot-1", json![los_pollos_amp()])
.with_icon(los_pollos_icon())
.with_icon(fakespot_amazon_icon()),
);
store.ingest(SuggestIngestionConstraints::all_providers());
assert_eq!(
store.fetch_suggestions(SuggestionQuery::fakespot("globe")),
vec![snowglobe_suggestion().with_fakespot_product_type_bonus(0.5)],
);
assert_eq!(
store.fetch_suggestions(SuggestionQuery::amp("lo")),
vec![los_pollos_suggestion("los")],
);
// Test deleting one of the records
store
.client_mut()
.delete_record("quicksuggest", "fakespot-1")
.delete_icon(los_pollos_icon());
store.ingest(SuggestIngestionConstraints::all_providers());
// FIXME(Bug 1912283): this setup currently deletes both suggestions, since
// `drop_suggestions` only checks against record ID.
//
// assert_eq!(
// store.fetch_suggestions(SuggestionQuery::fakespot("globe")),
// vec![snowglobe_suggestion()],
// );
// assert_eq!(
// store.fetch_suggestions(SuggestionQuery::amp("lo")),
// vec![],
// );
// However, we can test that the ingested records table has the correct entries
let record_keys = store
.read(|dao| dao.get_ingested_records())
.unwrap()
.into_iter()
.map(|r| format!("{}:{}", r.collection, r.id.as_str()))
.collect::<Vec<_>>();
assert_eq!(
record_keys
.iter()
.map(String::as_str)
.collect::<HashSet<_>>(),
HashSet::from([
"quicksuggest:icon-fakespot-amazon",
"fakespot-suggest-products:fakespot-1"
]),
);
Ok(())
}
}

26
third_party/rust/suggest/src/suggest.udl vendored
View File

@@ -116,7 +116,6 @@ dictionary SuggestionQuery {
};
dictionary SuggestIngestionConstraints {
u64? max_suggestions = null;
sequence<SuggestionProvider>? providers = null;
// Only ingest if the table `suggestions` is empty.
//
@@ -128,6 +127,26 @@ dictionary SuggestIngestionConstraints {
boolean empty_only = false;
};
dictionary SuggestIngestionMetrics {
// Samples for the `suggest.ingestion_time` metric
sequence<LabeledTimingSample> ingestion_times;
// Samples for the `suggest.ingestion_download_time` metric
sequence<LabeledTimingSample> download_times;
};
dictionary QueryWithMetricsResult {
sequence<Suggestion> suggestions;
// Samples for the `suggest.query_time` metric
sequence<LabeledTimingSample> query_times;
};
/// A single sample for a labeled timing distribution metric
dictionary LabeledTimingSample {
string label;
// Time in microseconds
u64 value;
};
dictionary SuggestGlobalConfig {
i32 show_less_frequently_cap;
};
@@ -152,6 +171,9 @@ interface SuggestStore {
[Throws=SuggestApiError]
sequence<Suggestion> query(SuggestionQuery query);
[Throws=SuggestApiError]
QueryWithMetricsResult query_with_metrics(SuggestionQuery query);
[Throws=SuggestApiError]
void dismiss_suggestion(string raw_suggestion_url);
@@ -165,7 +187,7 @@ interface SuggestStore {
void interrupt(optional InterruptKind? kind = null);
[Throws=SuggestApiError]
void ingest(SuggestIngestionConstraints constraints);
SuggestIngestionMetrics ingest(SuggestIngestionConstraints constraints);
[Throws=SuggestApiError]
void clear();

26
third_party/rust/suggest/src/suggestion.rs vendored
View File

@@ -106,12 +106,14 @@ impl Ord for Suggestion {
Suggestion::Amp { score, .. }
| Suggestion::Pocket { score, .. }
| Suggestion::Amo { score, .. } => score,
Suggestion::Fakespot { score, .. } => score,
_ => &DEFAULT_SUGGESTION_SCORE,
};
let b_score = match other {
Suggestion::Amp { score, .. }
| Suggestion::Pocket { score, .. }
| Suggestion::Amo { score, .. } => score,
Suggestion::Fakespot { score, .. } => score,
_ => &DEFAULT_SUGGESTION_SCORE,
};
b_score
@@ -175,6 +177,30 @@ impl Suggestion {
}
}
#[cfg(test)]
/// Testing utilitise
impl Suggestion {
pub fn with_fakespot_keyword_bonus(mut self) -> Self {
match &mut self {
Self::Fakespot { score, .. } => {
*score += 0.01;
}
_ => panic!("Not Suggestion::Fakespot"),
}
self
}
pub fn with_fakespot_product_type_bonus(mut self, bonus: f64) -> Self {
match &mut self {
Self::Fakespot { score, .. } => {
*score += 0.001 * bonus;
}
_ => panic!("Not Suggestion::Fakespot"),
}
self
}
}
impl Eq for Suggestion {}
/// Replaces all template parameters in a "raw" sponsored suggestion URL,
/// producing a "cooked" URL with real values.

245
third_party/rust/suggest/src/testing/client.rs vendored
View File

@@ -9,8 +9,9 @@ use serde_json::json;
use serde_json::Value as JsonValue;
use crate::{
rs::{Client, Record, RecordRequest},
testing::JsonExt,
db::SuggestDao,
error::Error,
rs::{Client, Collection, Record, SuggestRecordId, SuggestRecordType},
Result,
};
@@ -19,100 +20,125 @@ use crate::{
/// MockRemoteSettingsClient uses the builder pattern for its API: most methods input `self` and
/// return a modified version of it.
pub struct MockRemoteSettingsClient {
pub records: HashMap<String, Vec<Record>>,
pub records: Vec<Record>,
pub attachments: HashMap<String, Vec<u8>>,
pub last_modified_timestamp: u64,
}
impl Default for MockRemoteSettingsClient {
fn default() -> Self {
Self {
records: HashMap::new(),
records: Vec::new(),
attachments: HashMap::new(),
last_modified_timestamp: 100,
}
}
}
fn record_type_for_str(record_type_str: &str) -> SuggestRecordType {
for record_type in SuggestRecordType::all() {
if record_type.as_str() == record_type_str {
return *record_type;
}
}
panic!("Invalid record type string: {record_type_str}");
}
impl MockRemoteSettingsClient {
// Consuming Builder API, this is best for constructing the initial client
pub fn with_record(mut self, record_type: &str, record_id: &str, items: JsonValue) -> Self {
self.add_record(record_type, record_id, items);
self
}
pub fn with_icon(mut self, icon: MockIcon) -> Self {
self.add_icon(icon);
self
}
pub fn with_record_but_no_attachment(mut self, record_type: &str, record_id: &str) -> Self {
self.add_record_but_no_attachment(record_type, record_id);
self
}
pub fn with_inline_record(
mut self,
record_type: &str,
record_id: &str,
inline_data: JsonValue,
) -> Self {
self.add_inline_record(record_type, record_id, inline_data);
self
}
// Non-Consuming Builder API, this is best for updating an existing client
/// Add a record to the mock data
///
/// A single record typically contains multiple items in the attachment data. Pass all of them
/// as the `items` param.
pub fn with_record(mut self, record_type: &str, record_id: &str, items: JsonValue) -> Self {
pub fn add_record(
&mut self,
record_type: &str,
record_id: &str,
items: JsonValue,
) -> &mut Self {
let location = format!("{record_type}-{record_id}.json");
let records = self.records.entry(record_type.to_string()).or_default();
records.push(Record {
id: record_id.to_string(),
self.records.push(Record {
id: SuggestRecordId::new(record_id.to_string()),
collection: record_type_for_str(record_type).collection(),
last_modified: self.last_modified_timestamp,
deleted: false,
attachment: Some(Attachment {
filename: location.clone(),
mimetype: "application/json".into(),
hash: "".into(),
size: 0,
location,
location: location.clone(),
}),
fields: json!({"type": record_type}).into_map(),
attachment_data: Some(
serde_json::to_vec(&items).expect("error serializing attachment data"),
),
});
self
}
/// Add a tombstone record
///
/// This is used by remote settings to indicated a deleted record
pub fn with_tombstone(mut self, record_type: &str, record_id: &str) -> Self {
let records = self.records.entry(record_type.to_string()).or_default();
records.push(Record {
id: record_id.to_string(),
last_modified: self.last_modified_timestamp,
deleted: true,
attachment: None,
attachment_data: None,
fields: json!({}).into_map(),
payload: serde_json::from_value(json!({"type": record_type})).unwrap(),
});
self.attachments.insert(
location,
serde_json::to_vec(&items).expect("error serializing attachment data"),
);
self
}
/// Add a record for an icon to the mock data
pub fn with_icon(mut self, icon: MockIcon) -> Self {
pub fn add_icon(&mut self, icon: MockIcon) -> &mut Self {
let icon_id = icon.id;
let record_id = format!("icon-{icon_id}");
let location = format!("icon-{icon_id}.png");
let records = self.records.entry("icon".to_string()).or_default();
records.push(Record {
id: record_id.to_string(),
self.records.push(Record {
id: SuggestRecordId::new(record_id.to_string()),
last_modified: self.last_modified_timestamp,
deleted: false,
collection: Collection::Quicksuggest,
attachment: Some(Attachment {
filename: location.clone(),
mimetype: icon.mimetype.into(),
hash: "".into(),
size: 0,
location,
location: location.clone(),
}),
fields: json!({"type": "icon"}).into_map(),
attachment_data: Some(icon.data.as_bytes().to_vec()),
payload: serde_json::from_value(json!({"type": "icon"})).unwrap(),
});
self.attachments
.insert(location, icon.data.as_bytes().to_vec());
self
}
/// Add a tombstone record for an icon
pub fn with_icon_tombstone(self, icon: MockIcon) -> Self {
self.with_tombstone("icon", &format!("icon-{}", icon.id))
}
/// Add a record without attachment data
pub fn with_record_but_no_attachment(mut self, record_type: &str, record_id: &str) -> Self {
let records = self.records.entry(record_type.to_string()).or_default();
records.push(Record {
id: record_id.to_string(),
pub fn add_record_but_no_attachment(
&mut self,
record_type: &str,
record_id: &str,
) -> &mut Self {
self.records.push(Record {
id: SuggestRecordId::new(record_id.to_string()),
collection: record_type_for_str(record_type).collection(),
last_modified: self.last_modified_timestamp,
deleted: false,
attachment: None,
fields: json!({"type": record_type}).into_map(),
attachment_data: None,
payload: serde_json::from_value(json!({"type": record_type})).unwrap(),
});
self
}
@@ -121,27 +147,100 @@ impl MockRemoteSettingsClient {
///
/// Use this for record types like weather where the data it stored in the record itself rather
/// than in an attachment.
pub fn with_inline_record(
mut self,
pub fn add_inline_record(
&mut self,
record_type: &str,
record_id: &str,
inline_data: JsonValue,
) -> Self {
let records = self.records.entry(record_type.to_string()).or_default();
records.push(Record {
id: record_id.to_string(),
) -> &mut Self {
self.records.push(Record {
id: SuggestRecordId::new(record_id.to_string()),
collection: record_type_for_str(record_type).collection(),
last_modified: self.last_modified_timestamp,
deleted: false,
fields: json!({
payload: serde_json::from_value(json!({
"type": record_type,
record_type: inline_data,
})
.into_map(),
}))
.unwrap(),
attachment: None,
attachment_data: None,
});
self
}
// Update API, these use the &mut builder pattern, since they're used with already built
// clients
/// Update a record, storing a new payload and bumping the modified time
pub fn update_record(
&mut self,
record_type: &str,
record_id: &str,
items: JsonValue,
) -> &mut Self {
let record = self
.records
.iter_mut()
.find(|r| r.id.as_str() == record_id)
.unwrap_or_else(|| panic!("update_record: {record_id} not found"));
let attachment_data = self
.attachments
.get_mut(
&record
.attachment
.as_ref()
.expect("update_record: no attachment")
.location,
)
.unwrap_or_else(|| panic!("update_record: attachment not found for {record_id}"));
record.last_modified += 1;
record.payload = serde_json::from_value(json!({"type": record_type})).unwrap();
*attachment_data = serde_json::to_vec(&items).expect("error serializing attachment data");
self
}
/// Update an icon record, storing a new payload and bumping the modified time
pub fn update_icon(&mut self, icon: MockIcon) -> &mut Self {
let icon_id = &icon.id;
let record_id = format!("icon-{icon_id}");
let record = self
.records
.iter_mut()
.find(|r| r.id.as_str() == record_id)
.unwrap_or_else(|| panic!("update_icon: {record_id} not found"));
let attachment_data = self
.attachments
.get_mut(
&record
.attachment
.as_ref()
.expect("update_icon: no attachment")
.location,
)
.unwrap_or_else(|| panic!("update_icon: attachment not found for {icon_id}"));
record.last_modified += 1;
*attachment_data = icon.data.as_bytes().to_vec();
self
}
/// Delete a record and it's attachment
pub fn delete_record(&mut self, collection: &str, record_id: &str) -> &mut Self {
let idx = self
.records
.iter()
.position(|r| r.id.as_str() == record_id && r.collection.name() == collection)
.unwrap_or_else(|| panic!("delete_record: {collection}:{record_id} not found"));
let deleted = self.records.remove(idx);
if let Some(a) = deleted.attachment {
self.attachments.remove(&a.location);
}
self
}
pub fn delete_icon(&mut self, icon: MockIcon) -> &mut Self {
self.delete_record("quicksuggest", &format!("icon-{}", icon.id))
}
}
pub struct MockIcon {
@@ -151,11 +250,23 @@ pub struct MockIcon {
}
impl Client for MockRemoteSettingsClient {
fn get_records(&self, request: RecordRequest) -> Result<Vec<Record>> {
// Note: limit and modified time are ignored
Ok(match self.records.get(&request.record_type) {
Some(records) => records.clone(),
None => vec![],
})
fn get_records(&self, collection: Collection, _db: &mut SuggestDao) -> Result<Vec<Record>> {
Ok(self
.records
.iter()
.filter(|r| collection == r.record_type().collection())
.cloned()
.collect())
}
fn download_attachment(&self, record: &Record) -> Result<Vec<u8>> {
match &record.attachment {
None => Err(Error::MissingAttachment(record.id.to_string())),
Some(a) => Ok(self
.attachments
.get(&a.location)
.expect("Attachment not in hash map")
.clone()),
}
}
}

Some files were not shown because too many files have changed in this diff Show More