cang-jie(仓颉)

Chinese tokenizer integration for Tantivy, backed by jieba-rs.

Usage

use cangjie::{CangJieTokenizer, TokenizerOption, CANGJIE};
use jieba_rs::Jieba;
use std::sync::Arc;
use tantivy::{
    doc,
    schema::{IndexRecordOption, SchemaBuilder, TextFieldIndexing, TextOptions},
    Index,
};
fn main() -> tantivy::Result<()> {     let mut schema_builder = SchemaBuilder::default();     let text_indexing = TextFieldIndexing::default()         .settokenizer(CANGJIE)         .setindexoption(IndexRecordOption::WithFreqsAndPositions);     let text_options = TextOptions::default()         .setindexingoptions(text_indexing)         .set_stored();     let title = schemabuilder.addtextfield("title", textoptions);     let schema = schema_builder.build();
let index = Index::createinram(schema);     let tokenizer = CangJieTokenizer {         worker: Arc::new(Jieba::new()),         option: TokenizerOption::Default { hmm: false },     };     index.tokenizers().register(CANG_JIE, tokenizer);
let mut index_writer = index.writer(50  1024  1024)?;     indexwriter.adddocument(doc! { title => "南京长江大桥" })?;     index_writer.commit()?;
Ok(()) }

See unicode_split.rs for a complete searchable example.

Maintenance

This crate uses Rust 2024 and requires Rust 1.88 or newer.

Install prek, then run the local checks with:

prek run --all-files