Content：缓冲的代价

// Used from generated code to buffer the contents of the Deserializer when
// deserializing untagged enums and internally tagged enums.
//
// Not public API. Use serde-value instead.
//
// Obsoleted by format-specific buffer types (https://github.com/serde-rs/serde/pull/2912).
#[doc(hidden)]
pub enum Content<'de> {
    Bool(bool),

    U8(u8),
    U16(u16),
    U32(u32),
    U64(u64),

    I8(i8),
    I16(i16),
    I32(i32),
    I64(i64),

    F32(f32),
    F64(f64),

    Char(char),
    String(String),
    Str(&'de str),
    ByteBuf(Vec<u8>),
    Bytes(&'de [u8]),

    None,
    Some(Box<Content<'de>>),

    Unit,
    Newtype(Box<Content<'de>>),
    Seq(Vec<Content<'de>>),
    Map(Vec<(Content<'de>, Content<'de>)>),
}

    impl<'de> DeserializeSeed<'de> for ContentVisitor<'de> {
        type Value = Content<'de>;

        fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
        where
            D: Deserializer<'de>,
        {
            deserializer.__deserialize_content_v1(self)
        }
    }

        fn visit_map<V>(self, mut visitor: V) -> Result<Self::Value, V::Error>
        where
            V: MapAccess<'de>,
        {
            let mut vec =
                Vec::<(Content, Content)>::with_capacity(
                    size_hint::cautious::<(Content, Content)>(visitor.size_hint()),
                );
            while let Some(kv) =
                tri!(visitor.next_entry_seed(ContentVisitor::new(), ContentVisitor::new()))
            {
                vec.push(kv);
            }
            Ok(Content::Map(vec))
        }

        fn visit_enum<V>(self, _visitor: V) -> Result<Self::Value, V::Error>
        where
            V: EnumAccess<'de>,
        {
            Err(de::Error::custom(
                "untagged and internally tagged enums do not support enum input",
            ))
        }

use serde::Deserialize;

#[derive(Deserialize, Debug)]
enum E {
  A(u8),
}

#[derive(Deserialize, Debug)]
struct Inner {
  e: E,
}

#[derive(Deserialize, Debug)]
#[serde(tag = "t")]
enum Tagged {
  V(Inner),
}

fn main() {
  let yaml = "t: V\ne: !A 1\n";
  println!("plain  {:?}", serde_yaml::from_str::<Inner>(yaml).map_err(|e| e.to_string()));
  println!("tagged {:?}", serde_yaml::from_str::<Tagged>(yaml).map_err(|e| e.to_string()));
}

plain  Ok(Inner { e: A(1) })
tagged Err("e: untagged and internally tagged enums do not support enum input at line 2 column 4")

    // Not public API.
    #[cfg(all(not(no_serde_derive), any(feature = "std", feature = "alloc")))]
    #[doc(hidden)]
    fn __deserialize_content_v1<V>(self, visitor: V) -> Result<V::Value, Self::Error>
    where
        V: Visitor<'de, Value = crate::private::Content<'de>>,
    {
        self.deserialize_any(visitor)
    }

        fn __deserialize_content_v1<V>(self, visitor: V) -> Result<V::Value, Self::Error>
        where
            V: Visitor<'de, Value = Content<'de>>,
        {
            let _ = visitor;
            Ok(self.content)
        }

  // The hidden hook: log it, then let the wrapped format decide (serde_json keeps the default).
  fn __deserialize_content_v1<V>(self, visitor: V) -> Result<V::Value, D::Error>
  where
    V: Visitor<'de, Value = serde::__private229::de::Content<'de>>,
  {
    log(self.depth, "__deserialize_content_v1");
    self.inner.__deserialize_content_v1(LogV { inner: visitor, depth: self.depth + 1 })
  }

use exp_serde_content::log_de::LogDe;
use serde::Deserialize;

#[derive(Deserialize, Debug)]
struct Inner {
  x: u8,
}

#[derive(Deserialize, Debug)]
#[serde(untagged)]
enum Untagged {
  V(Inner),
}

fn main() {
  let input = r#"{"x":7}"#;
  println!("Inner:");
  let mut de = serde_json::Deserializer::from_str(input);
  println!("=> {:?}", Inner::deserialize(LogDe::new(&mut de)).unwrap());
  println!("Untagged:");
  let mut de = serde_json::Deserializer::from_str(input);
  println!("=> {:?}", Untagged::deserialize(LogDe::new(&mut de)).unwrap());
}

Inner:
deserialize_struct("Inner", ["x"])
  visit_map
    next_key_seed
      deserialize_identifier
        visit_borrowed_str("x")
    -> Some
    next_value_seed
      deserialize_u8
        visit_u64(7)
    next_key_seed
    -> None
=> Inner { x: 7 }
Untagged:
__deserialize_content_v1
  visit_map
    size_hint() = None
    next_entry_seed
      __deserialize_content_v1
        visit_borrowed_str("x")
      __deserialize_content_v1
        visit_u64(7)
    -> Some
    next_entry_seed
    -> None
=> V(Inner { x: 7 })

        fn deserialize_integer<V>(self, visitor: V) -> Result<V::Value, E>
        where
            V: Visitor<'de>,
        {
            match self.content {
                Content::U8(v) => visitor.visit_u8(v),
                Content::U16(v) => visitor.visit_u16(v),
                Content::U32(v) => visitor.visit_u32(v),
                Content::U64(v) => visitor.visit_u64(v),
                Content::I8(v) => visitor.visit_i8(v),
                Content::I16(v) => visitor.visit_i16(v),
                Content::I32(v) => visitor.visit_i32(v),
                Content::I64(v) => visitor.visit_i64(v),
                _ => Err(self.invalid_type(&visitor)),
            }
        }

pub struct Log;

impl<'de> Visitor<'de> for Log {
  type Value = String;
  fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
    f.write_str("anything")
  }
  // visit_bool ... visit_byte_buf: 都是 Ok(format!("{}({:?})", 方法名, v))
  fn visit_some<D: Deserializer<'de>>(self, d: D) -> Result<String, D::Error> {
    Ok(format!("visit_some({})", d.deserialize_any(Log)?))
  }
  // visit_unit、visit_none、visit_newtype_struct、visit_seq、visit_map 同理
}

macro_rules! probe {
  ($($ty:ident => $method:ident),*) => {
    $(
      pub struct $ty(pub String);
      // ... impl Debug：打印 self.0
      impl<'de> de::Deserialize<'de> for $ty {
        fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
          Ok($ty(match d.$method(Log) {
            Ok(s) => s,
            Err(e) => format!("Err({e})"),
          }))
        }
      }
    )*
  };
}

probe! {
  PU8 => deserialize_u8,
  PU128 => deserialize_u128,
  PStr => deserialize_str,
  PString => deserialize_string,
  PBytes => deserialize_bytes,
  POption => deserialize_option,
  PBool => deserialize_bool
  // ...
}

#[derive(Deserialize, Debug)]
struct Fields {
  a: PU8,
  b: PU128,
  big: PU128,
  s: PStr,
  esc: PStr,
  owned: PString,
  bytes: PBytes,
  opt: POption,
  flag: PBool,
}

#[derive(Deserialize, Debug)]
#[serde(tag = "t")]
enum Tagged {
  V(Fields),
}

#[derive(Deserialize, Debug)]
#[serde(untagged)]
enum Untagged {
  V(Fields),
}

const INPUT: &str = r#"{"t":"V","a":7,"b":7,"big":18446744073709551616,
  "s":"hi","esc":"a\nb","owned":"hi","bytes":"hi","opt":7,"flag":"true"}"#;

// show() 逐行打印九个字段
fn main() {
  show("direct (serde_json::Deserializer)", &serde_json::from_str::<Fields>(INPUT).unwrap());
  let Tagged::V(f) = serde_json::from_str::<Tagged>(INPUT).unwrap();
  show("tag = \"t\" (ContentDeserializer)", &f);
  let Untagged::V(f) = serde_json::from_str::<Untagged>(INPUT).unwrap();
  show("untagged (ContentRefDeserializer)", &f);
}

direct (serde_json::Deserializer)
  a     deserialize_u8      -> visit_u64(7)
  b     deserialize_u128    -> visit_u128(7)
  big   deserialize_u128    -> visit_u128(18446744073709551616)
  s     deserialize_str     -> visit_borrowed_str("hi")
  esc   deserialize_str     -> visit_str("a\nb")
  owned deserialize_string  -> visit_borrowed_str("hi")
  bytes deserialize_bytes   -> visit_borrowed_bytes([104, 105])
  opt   deserialize_option  -> visit_some(visit_u64(7))
  flag  deserialize_bool    -> Err(invalid type: string "true", expected anything at line 2 column 71)
tag = "t" (ContentDeserializer)
  a     deserialize_u8      -> visit_u64(7)
  b     deserialize_u128    -> Err(u128 is not supported)
  big   deserialize_u128    -> Err(u128 is not supported)
  s     deserialize_str     -> visit_borrowed_str("hi")
  esc   deserialize_str     -> visit_string("a\nb")
  owned deserialize_string  -> visit_borrowed_str("hi")
  bytes deserialize_bytes   -> visit_borrowed_str("hi")
  opt   deserialize_option  -> visit_some(visit_u64(7))
  flag  deserialize_bool    -> Err(invalid type: string "true", expected anything)
untagged (ContentRefDeserializer)
  a     deserialize_u8      -> visit_u64(7)
  b     deserialize_u128    -> Err(u128 is not supported)
  big   deserialize_u128    -> Err(u128 is not supported)
  s     deserialize_str     -> visit_borrowed_str("hi")
  esc   deserialize_str     -> visit_str("a\nb")
  owned deserialize_string  -> visit_borrowed_str("hi")
  bytes deserialize_bytes   -> visit_borrowed_str("hi")
  opt   deserialize_option  -> visit_some(visit_u64(7))
  flag  deserialize_bool    -> Err(invalid type: string "true", expected anything)

use std::collections::HashMap;

use serde::{Deserialize, Serialize};
use serde::de::DeserializeOwned;

#[derive(Serialize, Deserialize, Debug)]
struct Big {
  n: u128,
}

#[derive(Serialize, Deserialize, Debug)]
struct Keys {
  m: HashMap<u32, bool>,
}

#[derive(Serialize, Deserialize, Debug)]
struct Small {
  x: u8,
}

// The same struct three ways: as is, as an internally tagged variant, as an untagged variant.
macro_rules! wrap {
  ($inner:ident, $tagged:ident, $untagged:ident) => {
    #[derive(Serialize, Deserialize, Debug)]
    #[serde(tag = "t")]
    enum $tagged {
      V($inner),
    }
    #[derive(Serialize, Deserialize, Debug)]
    #[serde(untagged)]
    enum $untagged {
      V($inner),
    }
  };
}
wrap!(Big, BigTagged, BigUntagged);
wrap!(Keys, KeysTagged, KeysUntagged);
wrap!(Small, SmallTagged, SmallUntagged);

fn json<T: DeserializeOwned + std::fmt::Debug>(label: &str, s: &str) {
  match serde_json::from_str::<T>(s) {
    Ok(v) => println!("  {label:<9} Ok({v:?})"),
    Err(e) => println!("  {label:<9} Err({e})"),
  }
}

fn main() {
  let big = r#"{"t":"V","n":7}"#;
  println!("u128 field, input {big}");
  json::<Big>("plain", big);
  json::<BigTagged>("tag", big);
  json::<BigUntagged>("untagged", big);
  // ... Keys 用 {"t":"V","m":{"1":true}}，Small 用 {"t":"V","x":300}，同样三行

  println!("non-self-describing formats, value SmallTagged::V(Small {{ x: 7 }})");
  let v = SmallTagged::V(Small { x: 7 });
  let b = bincode::serialize(&v).unwrap();
  println!("  bincode  bytes {b:?}");
  println!("  bincode  back  {:?}", bincode::deserialize::<SmallTagged>(&b).map_err(|e| e.to_string()));
  let p = postcard::to_allocvec(&v).unwrap();
  println!("  postcard bytes {p:?}");
  println!("  postcard back  {:?}", postcard::from_bytes::<SmallTagged>(&p).map_err(|e| e.to_string()));
  let b = bincode::serialize(&SmallUntagged::V(Small { x: 7 })).unwrap();
  println!("  bincode  untagged back {:?}", bincode::deserialize::<SmallUntagged>(&b).map_err(|e| e.to_string()));
}

u128 field, input {"t":"V","n":7}
  plain     Ok(Big { n: 7 })
  tag       Err(u128 is not supported)
  untagged  Err(data did not match any variant of untagged enum BigUntagged)
HashMap<u32, bool> field, input {"t":"V","m":{"1":true}}
  plain     Ok(Keys { m: {1: true} })
  tag       Err(invalid type: string "1", expected u32)
  untagged  Err(data did not match any variant of untagged enum KeysUntagged)
u8 field, input {"t":"V","x":300}
  plain     Err(invalid value: integer `300`, expected u8 at line 1 column 16)
  tag       Err(invalid value: integer `300`, expected u8)
  untagged  Err(data did not match any variant of untagged enum SmallUntagged)
non-self-describing formats, value SmallTagged::V(Small { x: 7 })
  bincode  bytes [1, 0, 0, 0, 0, 0, 0, 0, 86, 7]
  bincode  back  Err("Bincode does not support the serde::Deserializer::deserialize_any method")
  postcard bytes [1, 86, 7]
  postcard back  Err("This is a feature that PostCard will never implement")
  bincode  untagged back Err("Bincode does not support the serde::Deserializer::deserialize_any method")

use serde::Deserialize;
use serde::de::DeserializeOwned;
use serde_json::value::RawValue;

#[derive(Deserialize, Debug)]
struct Num {
  n: u64,
  f: f64,
}

#[derive(Deserialize, Debug)]
#[serde(tag = "t")]
enum NumTagged {
  V(Num),
}

#[derive(Deserialize, Debug)]
#[serde(untagged)]
enum NumUntagged {
  V(Num),
}

#[derive(Deserialize, Debug)]
struct Val {
  n: serde_json::Value,
}

#[derive(Deserialize, Debug)]
#[serde(tag = "t")]
enum ValTagged {
  V(Val),
}

#[derive(Deserialize, Debug)]
struct Raw {
  r: Box<RawValue>,
}

#[derive(Deserialize, Debug)]
#[serde(tag = "t")]
enum RawTagged {
  V(Raw),
}

// json() 同上
fn main() {
  let num = r#"{"t":"V","n":7,"f":1.5}"#;
  println!("input {num}");
  json::<Num>("plain", num);
  json::<NumTagged>("tag", num);
  json::<NumUntagged>("untagged", num);
  let val = r#"{"t":"V","n":3.14159265358979323846264338327950288}"#;
  println!("input {val}");
  json::<Val>("Value plain", val);
  json::<ValTagged>("Value tag", val);
  let raw = r#"{"t":"V","r":[1, 2]}"#;
  println!("input {raw}");
  json::<Raw>("RawValue plain", raw);
  json::<RawTagged>("RawValue tag", raw);
}

input {"t":"V","n":7,"f":1.5}
  plain          Ok(Num { n: 7, f: 1.5 })
  tag            Err(invalid type: map, expected f64)
  untagged       Err(data did not match any variant of untagged enum NumUntagged)
input {"t":"V","n":3.14159265358979323846264338327950288}
  Value plain    Ok(Val { n: Number(3.14159265358979323846264338327950288) })
  Value tag      Ok(V(Val { n: Number(3.14159265358979323846264338327950288) }))
input {"t":"V","r":[1, 2]}
  RawValue plain Ok(Raw { r: RawValue([1, 2]) })
  RawValue tag   Err(invalid type: newtype struct, expected any valid JSON value)

use std::alloc::{GlobalAlloc, Layout, System};
use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
use std::time::Instant;

use serde::Deserialize;
use serde::de::DeserializeOwned;

// Counts every allocation (and its bytes) made by the process.
struct Counting;
static ALLOCS: AtomicUsize = AtomicUsize::new(0);
static BYTES: AtomicUsize = AtomicUsize::new(0);

unsafe impl GlobalAlloc for Counting {
  unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
    ALLOCS.fetch_add(1, Relaxed);
    BYTES.fetch_add(layout.size(), Relaxed);
    unsafe { System.alloc(layout) }
  }
  unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
    unsafe { System.dealloc(ptr, layout) }
  }
  unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
    ALLOCS.fetch_add(1, Relaxed);
    BYTES.fetch_add(new_size, Relaxed);
    unsafe { System.realloc(ptr, layout, new_size) }
  }
}

#[global_allocator]
static GLOBAL: Counting = Counting;

#[derive(Deserialize, Debug)]
struct Post {
  title: String,
  tags: Vec<String>,
  views: u64,
}

#[derive(Deserialize, Debug)]
#[serde(tag = "t")]
enum Tagged {
  Post(Post),
}

#[derive(Deserialize, Debug)]
#[serde(untagged)]
enum Untagged {
  Post(Post),
}

#[derive(Deserialize, Debug)]
struct Flat {
  #[serde(flatten)]
  post: Post,
}

// Internally tagged around untagged: buffering inside buffering.
#[derive(Deserialize, Debug)]
#[serde(tag = "t")]
enum Outer {
  Post(UntaggedInner),
}

#[derive(Deserialize, Debug)]
#[serde(untagged)]
enum UntaggedInner {
  Post(Post),
}

// Hides the hook: every method, including __deserialize_content_v1, ends in deserialize_any.
struct NoHook<D>(D);

impl<'de, D: serde::Deserializer<'de>> serde::Deserializer<'de> for NoHook<D> {
  type Error = D::Error;
  fn deserialize_any<V: serde::de::Visitor<'de>>(self, v: V) -> Result<V::Value, D::Error> {
    self.0.deserialize_any(v)
  }
  serde::forward_to_deserialize_any! {
    bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string
    bytes byte_buf option unit unit_struct newtype_struct seq tuple
    tuple_struct map struct enum identifier ignored_any
  }
}

#[derive(Debug)]
struct HiddenInner(UntaggedInner);

impl<'de> Deserialize<'de> for HiddenInner {
  fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
    UntaggedInner::deserialize(NoHook(d)).map(HiddenInner)
  }
}

#[derive(Deserialize, Debug)]
#[serde(tag = "t")]
enum OuterNoHook {
  Post(HiddenInner),
}

fn measure<T: DeserializeOwned>(label: &str, input: &str) {
  let a0 = ALLOCS.load(Relaxed);
  let b0 = BYTES.load(Relaxed);
  let v: T = serde_json::from_str(input).unwrap();
  let allocs = ALLOCS.load(Relaxed) - a0;
  let bytes = BYTES.load(Relaxed) - b0;
  drop(v);
  const N: u32 = 200_000;
  let t = Instant::now();
  for _ in 0..N {
    let v: T = serde_json::from_str(std::hint::black_box(input)).unwrap();
    std::hint::black_box(v);
  }
  let ns = t.elapsed().as_nanos() / N as u128;
  println!("{label:<22} allocs={allocs:<3} bytes={bytes:<5} {ns} ns");
}

fn main() {
  println!("size_of::<Content>() = {}", std::mem::size_of::<serde::__private229::de::Content>());
  let plain = r#"{"t":"Post","title":"hello","tags":["a","b","c"],"views":7}"#;
  let escaped = r#"{"t":"Post","title":"he\"llo","tags":["a","b","c"],"views":7}"#;
  for (name, input) in [("no escapes", plain), ("escaped strings", escaped)] {
    println!("{name}: {input}");
    measure::<Post>("plain struct", input);
    measure::<Tagged>("tag = \"t\"", input);
    measure::<Untagged>("untagged", input);
    measure::<Flat>("flatten", input);
    measure::<Outer>("tag wrapping untagged", input);
    measure::<OuterNoHook>("same, hook hidden", input);
  }
}

size_of::<Content>() = 32
no escapes: {"t":"Post","title":"hello","tags":["a","b","c"],"views":7}
plain struct           allocs=5   bytes=104   169 ns
tag = "t"              allocs=7   bytes=464   284 ns
untagged               allocs=7   bytes=464   283 ns
flatten                allocs=7   bytes=464   275 ns
tag wrapping untagged  allocs=7   bytes=464   268 ns
same, hook hidden      allocs=8   bytes=656   311 ns

This demonstrates a backward compatible way that a format such as serde_json could use its own serde_json::Value as its buffer type for untagged enums and flatten, instead of "Content", solving https://github.com/serde-rs/serde/issues/1183. Uses "return position impl trait in traits" (stabilized in Rust 1.75 by https://github.com/rust-lang/rust/pull/115822) and "precise capturing in traits" (stabilized in Rust 1.87 by https://github.com/rust-lang/rust/pull/138128).

escaped strings: {"t":"Post","title":"he\"llo","tags":["a","b","c"],"views":7}
plain struct           allocs=6   bytes=113   186 ns
tag = "t"              allocs=8   bytes=473   304 ns
untagged               allocs=9   bytes=479   323 ns
flatten                allocs=8   bytes=473   294 ns
tag wrapping untagged  allocs=9   bytes=479   303 ns
same, hook hidden      allocs=10  bytes=671   345 ns