分词

pub trait Tokenizer: 'static + Clone + Send + Sync {
    /// The token stream returned by this Tokenizer.
    type TokenStream<'a>: TokenStream;
    /// Creates a token stream for a given `str`.
    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a>;
}

pub trait TokenStream {
    /// Advance to the next token
    ///
    /// Returns false if there are no other tokens.
    fn advance(&mut self) -> bool;

    /// Returns a reference to the current token.
    fn token(&self) -> &Token;

    /// Returns a mutable reference to the current token.
    fn token_mut(&mut self) -> &mut Token;
    // ...
}

pub trait TokenFilter: 'static + Send + Sync {
    /// The Tokenizer type returned by this filter, typically parametrized by the underlying
    /// Tokenizer.
    type Tokenizer<T: Tokenizer>: Tokenizer;
    /// Wraps a Tokenizer and returns a new one.
    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T>;
}

    pub fn filter<F: TokenFilter>(self, token_filter: F) -> TextAnalyzerBuilder<F::Tokenizer<T>> {
        TextAnalyzerBuilder {
            tokenizer: token_filter.transform(self.tokenizer),
        }
    }

    fn predicate(&self, token: &Token) -> bool {
        token.text.len() < self.token_length_limit
    }

    fn default() -> TokenizerManager {
        let manager = TokenizerManager::new();
        manager.register(RAW_TOKENIZER_NAME, RawTokenizer::default());
        manager.register(
            DEFAULT_TOKENIZER_NAME,
            TextAnalyzer::builder(SimpleTokenizer::default())
                .filter(RemoveLongFilter::limit(40))
                .filter(LowerCaser)
                .build(),
        );
        #[cfg(feature = "stemmer")]
        {
            use crate::tokenizer::stemmer::{Language, Stemmer};
            manager.register(
                "en_stem",
                TextAnalyzer::builder(SimpleTokenizer::default())
                    .filter(RemoveLongFilter::limit(40))
                    .filter(LowerCaser) // The stemmer does not lowercase
                    .filter(Stemmer::new(Language::English))
                    .build(),
            );
        }
        manager.register("whitespace", WhitespaceTokenizer::default());
        manager
    }

    fn advance(&mut self) -> bool {
        self.token.text.clear();
        self.token.position = self.token.position.wrapping_add(1);
        while let Some((offset_from, c)) = self.chars.next() {
            if c.is_alphanumeric() {
                let offset_to = self.search_token_end();
                self.token.offset_from = offset_from;
                self.token.offset_to = offset_to;
                self.token.text.push_str(&self.text[offset_from..offset_to]);
                return true;
            }
        }
        false
    }

--- "The Old Man and the Sea" (23 bytes)
     default (6 tokens): the@0[0..3] | old@1[4..7] | man@2[8..11] | and@3[12..15] | the@4[16..19] | sea@5[20..23]
         raw (1 tokens): The Old Man and the Sea@0[0..23]
     en_stem (6 tokens): the@0[0..3] | old@1[4..7] | man@2[8..11] | and@3[12..15] | the@4[16..19] | sea@5[20..23]
  whitespace (6 tokens): The@0[0..3] | Old@1[4..7] | Man@2[8..11] | and@3[12..15] | the@4[16..19] | Sea@5[20..23]
       jieba (11 tokens): The@0[0..3] |  @3[3..4] | Old@4[4..7] |  @7[7..8] | Man@8[8..11] |  @11[11..12] | and@12[12..15] |  @15[15..16] | the@16[16..19] |  @19[19..20] | Sea@20[20..23]
 jieba+lower (11 tokens): the@0[0..3] |  @3[3..4] | old@4[4..7] |  @7[7..8] | man@8[8..11] |  @11[11..12] | and@12[12..15] |  @15[15..16] | the@16[16..19] |  @19[19..20] | sea@20[20..23]
--- "tantivy是Lucene风格的全文检索库，用Rust编写。" (59 bytes)
     default (1 tokens): 用rust编写@1[43..56]
         raw (1 tokens): tantivy是Lucene风格的全文检索库，用Rust编写。@0[0..59]
     en_stem (1 tokens): 用rust编写@1[43..56]
  whitespace (1 tokens): tantivy是Lucene风格的全文检索库，用Rust编写。@0[0..59]
       jieba (14 tokens): tantivy@0[0..7] | 是@7[7..10] | Lucene@8[10..16] | 风格@14[16..22] | 的@16[22..25] | 全文@17[25..31] | 检索@19[31..37] | 全文检索@17[25..37] | 库@21[37..40] | ，@22[40..43] | 用@23[43..46] | Rust@24[46..50] | 编写@28[50..56] | 。@30[56..59]
 jieba+lower (14 tokens): tantivy@0[0..7] | 是@7[7..10] | lucene@8[10..16] | 风格@14[16..22] | 的@16[22..25] | 全文@17[25..31] | 检索@19[31..37] | 全文检索@17[25..37] | 库@21[37..40] | ，@22[40..43] | 用@23[43..46] | rust@24[46..50] | 编写@28[50..56] | 。@30[56..59]

        let per_field_text_analyzers = schema
            .fields()
            .map(|(_, field_entry)| {
                let text_options = match field_entry.field_type() {
                    FieldType::Str(ref text_options) => text_options.get_indexing_options(),
                    FieldType::JsonObject(ref json_object_options) => {
                        json_object_options.get_text_indexing_options()
                    }
                    _ => None,
                };
                let tokenizer_name = text_options
                    .map(|text_index_option| text_index_option.tokenizer())
                    .unwrap_or("default");
                tokenizer_manager.get(tokenizer_name).ok_or_else(|| {
                    TantivyError::SchemaError(format!(
                        "Error getting tokenizer for field: {}",
                        field_entry.name()
                    ))
                })
            })
            .collect::<Result<Vec<_>, _>>()?;

  println!("== A. index time, tokenizer never registered");
  let index = Index::create_in_ram(schema.clone());
  println!("tokenizer_for_field: {:?}", index.tokenizer_for_field(title).err());
  // writer_with_num_threads(1, 15_000_000)，然后 add_document 三次，再 commit

== A. index time, tokenizer never registered
tokenizer_for_field: Some(InvalidArgument("No Tokenizer found for field FieldEntry { name: \"title\", field_type: Str(TextOptions { indexing: Some(TextFieldIndexing { record: WithFreqsAndPositions, fieldnorms: true, tokenizer: \"jieba\" }), stored: true, fast: None, coerce: false }) }"))
writer(): Ok
add_document #0: Ok(0)
add_document #1: Ok(1)
add_document #2: Ok(2)
commit: Err(SchemaError("Error getting tokenizer for field: title"))
== B. index with jieba, reopen without registering
parse_query: Err(UnknownTokenizer { tokenizer: "jieba", field: "title" })
  display: The tokenizer '"jieba"' for the field '"title"' is unknown
parse_query_lenient: errors=[UnknownTokenizer { tokenizer: "jieba", field: "title" }], hits=0
== C. reopen and register a different analyzer under the same name
query "检索" -> TermQuery(Term(field=0, type=Str, "检索")) hits=1
query "全文检索引擎" -> TermQuery(Term(field=0, type=Str, "全文检索引擎")) hits=0
query "rust" -> TermQuery(Term(field=0, type=Str, "rust")) hits=1
with jieba again: TermQuery(Term(field=0, type=Str, "检索")) hits=1
== D. empty TokenizerManager, schema with no text field
add_document: Ok(0)
commit: Err(SchemaError("Error getting tokenizer for field: n"))

        let former_workers_join_handle = std::mem::take(&mut self.workers_join_handle);

        for worker_handle in former_workers_join_handle {
            let indexing_worker_result = worker_handle
                .join()
                .map_err(|e| TantivyError::ErrorInThread(format!("{e:?}")))?;
            indexing_worker_result?;
            self.add_indexing_worker()?;
        }

commit #1: Err(SchemaError("Error getting tokenizer for field: title"))
added 9998
added 9999
[runq: killed after 20s]

$ cargo tree --offline -i tantivy-tokenizer-api
tantivy-tokenizer-api v0.7.0
├── tantivy v0.27.0
│   └── schema-exp v0.1.0 (~/exp-tantivy/schema-exp)
└── tantivy-jieba v0.19.0
    └── schema-exp v0.1.0 (~/exp-tantivy/schema-exp)

error[E0277]: the trait bound `WholeText: tantivy::tokenizer::Tokenizer` is not satisfied
  --> src/main.rs:31:29
   |
31 |   manager.register("whole", WholeText);
   |           --------          ^^^^^^^^^ unsatisfied trait bound
   |           |
   |           required by a bound introduced by this call
   |
...
note: there are multiple different versions of crate `tantivy_tokenizer_api` in the dependency graph
  --> ~/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tantivy-tokenizer-api-0.7.0/src/lib.rs:56:1
   |
56 | pub trait Tokenizer: 'static + Clone + Send + Sync {
   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ this is the expected trait
   |
  ::: fakeapi/src/lib.rs:56:1
   |
56 | pub trait Tokenizer: 'static + Clone + Send + Sync {
   | -------------------------------------------------- this is the found trait
...

["rust"]

--- "一个索引由若干个不可变的段和一个元数据文件组成。" (72 bytes)
     default (0 tokens): 
         raw (1 tokens): 一个索引由若干个不可变的段和一个元数据文件组成。@0[0..72]
     en_stem (0 tokens): 
  whitespace (1 tokens): 一个索引由若干个不可变的段和一个元数据文件组成。@0[0..72]
       jieba (17 tokens): 一个@0[0..6] | 索引@2[6..12] | 由@4[12..15] | 若干@5[15..21] | 若干个@5[15..24] | 不可@8[24..30] | 变@10[30..33] | 的@11[33..36] | 段@12[36..39] | 和@13[39..42] | 一个@14[42..48] | 元@16[48..51] | 数据@17[51..57] | 文件@19[57..63] | 数据文件@17[51..63] | 组成@21[63..69] | 。@23[69..72]