rust-tokenizers-devel-0.22.1-1.fc43

Date	Author	Change
2025-09-28	Alexander F. Lent <lx at xanderlent dot com> - 0.22.1-1	- Initial import of tokenizers 0.22.1 (fixes rhbz#2358553)

usr/
- share/
  - cargo/
    - registry/
      - tokenizers-0.22.1/
        
        .cargo-checksum.json
        
        CHANGELOG.md
        
        Cargo.toml
        
        LICENSE
        
        Makefile
        
        README.md
        
        README.tpl
        
        benches/
        
        bert_benchmark.rs
        
        bpe_benchmark.rs
        
        common/
        
        mod.rs
        
        layout_benchmark.rs
        
        llama3_benchmark.rs
        
        unigram_benchmark.rs
        
        examples/
        
        encode_batch.rs
        
        serialization.rs
        
        src/
        
        decoders/
        
        bpe.rs
        
        byte_fallback.rs
        
        ctc.rs
        
        fuse.rs
        
        mod.rs
        
        sequence.rs
        
        strip.rs
        
        wordpiece.rs
        
        lib.rs
        
        models/
        
        bpe/
        
        mod.rs
        
        model.rs
        
        serialization.rs
        
        trainer.rs
        
        word.rs
        
        mod.rs
        
        unigram/
        
        lattice.rs
        
        mod.rs
        
        model.rs
        
        serialization.rs
        
        trainer.rs
        
        trie.rs
        
        wordlevel/
        
        mod.rs
        
        serialization.rs
        
        trainer.rs
        
        wordpiece/
        
        mod.rs
        
        serialization.rs
        
        trainer.rs
        
        normalizers/
        
        bert.rs
        
        byte_level.rs
        
        mod.rs
        
        precompiled.rs
        
        prepend.rs
        
        replace.rs
        
        strip.rs
        
        unicode.rs
        
        utils.rs
        
        pre_tokenizers/
        
        bert.rs
        
        byte_level.rs
        
        delimiter.rs
        
        digits.rs
        
        fixed_length.rs
        
        metaspace.rs
        
        mod.rs
        
        punctuation.rs
        
        sequence.rs
        
        split.rs
        
        unicode_scripts/
        
        mod.rs
        
        pre_tokenizer.rs
        
        scripts.rs
        
        whitespace.rs
        
        processors/
        
        bert.rs
        
        mod.rs
        
        roberta.rs
        
        sequence.rs
        
        template.rs
        
        tokenizer/
        
        added_vocabulary.rs
        
        encoding.rs
        
        mod.rs
        
        normalizer.rs
        
        pattern.rs
        
        pre_tokenizer.rs
        
        serialization.rs
        
        utils/
        
        cache.rs
        
        fancy.rs
        
        from_pretrained.rs
        
        iter.rs
        
        mod.rs
        
        onig.rs
        
        padding.rs
        
        parallelism.rs
        
        progress.rs
        
        truncation.rs
        
        tests/
        
        added_tokens.rs
        
        common/
        
        mod.rs
        
        documentation.rs
        
        from_pretrained.rs
        
        offsets.rs
        
        serialization.rs
        
        stream.rs
        
        training.rs
        
        unigram.rs

rust-tokenizers-devel-0.22.1-1.fc43 in Fedora 43

Changelog

Dependencies

Provides

Files