sha256:01d974999b85c6a8a3016188cc9070a59c8687bd079a48f0ac2e5adda471a6b5

Found at

Contains Dependency Lockfiles

Diff

Content

sha256:3052155ae21de4df441858eb016c42d23f3cf7d919f7b9db5319fa28a670f66a  tokenizers-0.21.4/.cargo_vcs_info.json
sha256:48555a205c39e1b840ad65203e1359ddea42476b9f515b7340d09bdc52d56242  tokenizers-0.21.4/CHANGELOG.md
sha256:a75525d142a11e8b5d175f2773a608a20ba6cfa8bfa074077a488d277e198125  tokenizers-0.21.4/Cargo.lock
sha256:f0847da8fe95c05f0c11779697cfe9974b797492f4a19ea2d6d274594957bd04  tokenizers-0.21.4/Cargo.toml
sha256:184ce7adb5bb723e2456da8d6003edba96bba8e73739fc0c06e64fccfe5e63aa  tokenizers-0.21.4/Cargo.toml.orig
sha256:c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4  tokenizers-0.21.4/LICENSE
sha256:f530177d873596981db15dd42bb09918ace74eeee25f31c971e8eb44a6b9b66d  tokenizers-0.21.4/Makefile
sha256:83b36dcc3220a9c9e47babc7245afe7b63f2bc4123450c504a6f48b6f3b4b8c8  tokenizers-0.21.4/README.md
sha256:487d57adf5a2cb4729326ce4a0c85e80c965e8951f3b38e6bc3868902519b39f  tokenizers-0.21.4/README.tpl
sha256:08d3a412d392c3f6122832142e3bd77702581a7da92e53cecd3f630dbdd81d7e  tokenizers-0.21.4/benches/bert_benchmark.rs
sha256:d63ceac74b1214bb92d1f6674996eb8b1eee5e602a0e8ee668064915405b2397  tokenizers-0.21.4/benches/bpe_benchmark.rs
sha256:27f44a13413285a55f3014ee248251e356cd7e3de65681b58b87156a8a52bc8c  tokenizers-0.21.4/benches/common/mod.rs
sha256:ed2c22d8e76ddfcd61fe946fcae3d2dba0722ea9c5def11586c5b90f138b1c31  tokenizers-0.21.4/benches/layout_benchmark.rs
sha256:11ab295f6f058dbd12d1c2ad774b47562a17c1510bc985bedef195df37f936bc  tokenizers-0.21.4/benches/llama3_benchmark.rs
sha256:5a420b8399924fae6d30c32d7ded13de8c6ef17bc34ce8114c3cefbde15128cf  tokenizers-0.21.4/benches/unigram_benchmark.rs
sha256:2bd564996ff859e1c996238f38f576657e566688bbd435c8f100e3bb0cdc43af  tokenizers-0.21.4/examples/encode_batch.rs
sha256:8f35712a217fef4bccf538a3bc0c71bb460096ff751169a057a2165bc9ee57e1  tokenizers-0.21.4/examples/serialization.rs
sha256:9c363604a82b2e1439afaa78efe6b11c2d33348525bb934c797a37f2007bf3bb  tokenizers-0.21.4/src/decoders/bpe.rs
sha256:74d0d3546551f6ca126617af450a22436289e3be2a8049f19271f41524318e64  tokenizers-0.21.4/src/decoders/byte_fallback.rs
sha256:5bc7697e3661238e33d1614e95003e24b4814cf9e1dc7f458c1fc814d84c9d27  tokenizers-0.21.4/src/decoders/ctc.rs
sha256:19b7bfca165bdee6ab084ed5429a49afc9d52664c51187a500335a80707850fa  tokenizers-0.21.4/src/decoders/fuse.rs
sha256:9963dedcd9a965ae80888e72a7d83fbe2ee1c4f426b313cf1b7014c384419a08  tokenizers-0.21.4/src/decoders/mod.rs
sha256:6bbaf543e2ac97e658bd657ccdb56b8930218c9381492e712d4c93131b663eca  tokenizers-0.21.4/src/decoders/sequence.rs
sha256:8f552ed13ffd5ec94808f00a62d3e28e9d1b5a811412669f87ffbf4cf857b306  tokenizers-0.21.4/src/decoders/strip.rs
sha256:3765198f8efe1e74f6bae74a679db94afec5f030c30d3c24cd579c976354768e  tokenizers-0.21.4/src/decoders/wordpiece.rs
sha256:27f6f5118f0b253590a4e5a532a457b16047a33aa2062454cc39a47b8043e0e3  tokenizers-0.21.4/src/lib.rs
sha256:bd38f20a24f390ea319148bc494010319cb613948edf42e1acd817b5801e6d8f  tokenizers-0.21.4/src/models/bpe/mod.rs
sha256:f348b70dd712de21f50f244d7fea164788419904cdf52f581dc180f90e31d671  tokenizers-0.21.4/src/models/bpe/model.rs
sha256:123a0c7f42580f508a784e0fa3c3115aa5ae1b2e604dcedde3e083d2cb94a92a  tokenizers-0.21.4/src/models/bpe/serialization.rs
sha256:7238cfee709bc7a37f628454db0c4c0ee3065f5c16e68714212a0eb6f38d3d54  tokenizers-0.21.4/src/models/bpe/trainer.rs
sha256:42a5547426082590675979ed72e03ce59a16f7cf0b57aaac6890d43ce0a79ad0  tokenizers-0.21.4/src/models/bpe/word.rs
sha256:d977f388ea1361abe8336a418a44e7ddf02f066c41f76a1fc63b26b23ebf5883  tokenizers-0.21.4/src/models/mod.rs
sha256:d68eeda555a6ab623599e90d51121d1dcd2ad2829167e0aa6d932872da86f7b7  tokenizers-0.21.4/src/models/unigram/lattice.rs
sha256:cf317d99ceebe2b4326237aa8f8bfdc2293a5dd0a7fc69a6e9e0028b804a000e  tokenizers-0.21.4/src/models/unigram/mod.rs
sha256:38c6ff01261c40114ec3070455ae698bc2e5cd549572fe3657f8d2078b3508b1  tokenizers-0.21.4/src/models/unigram/model.rs
sha256:2358d3985ae5330b2caebdbd5567621572368869dcbb36e3a95696325e941ce3  tokenizers-0.21.4/src/models/unigram/serialization.rs
sha256:cb5a1965aafbe00b8893ae494ca1cc8c74809d26a7047363e65df1f7e57c2140  tokenizers-0.21.4/src/models/unigram/trainer.rs
sha256:44eec12962dcc71e9369056ed0a3de8925b19cd1d9023c2e364491e2d087fbe2  tokenizers-0.21.4/src/models/unigram/trie.rs
sha256:ab105b50463b4eff10a2e367aeaa51a1461feb3a535bace6bf5759df8819d2e3  tokenizers-0.21.4/src/models/wordlevel/mod.rs
sha256:75a367e46105e3bebf8fdc2925e81cb1ac625ae1f700571069214d84a05e067a  tokenizers-0.21.4/src/models/wordlevel/serialization.rs
sha256:a270dc783a3d2e1e8c8409b0d5ec5b62178d25769108a50a0862ad61adb66c2c  tokenizers-0.21.4/src/models/wordlevel/trainer.rs
sha256:77a9eaf60f846a8435ddefcb2d466bbf672e410c7ec2875117b686a242b443de  tokenizers-0.21.4/src/models/wordpiece/mod.rs
sha256:6c3f71f99773a354f0ed29df78f13737bc8593e94af8da461b19a7d584b618e7  tokenizers-0.21.4/src/models/wordpiece/serialization.rs
sha256:7abfaf6d843c15d575c00c6b8564e6908612e98360f858204e80ad38cb8e07d5  tokenizers-0.21.4/src/models/wordpiece/trainer.rs
sha256:a938fdac02447e33393d98edcaad015f58e2095d633521cee85a2b60ccd995bd  tokenizers-0.21.4/src/normalizers/bert.rs
sha256:d39bd4a8dc7677fd017394c614d72ec5f10565f8f680e215d3ef29bd85bbd82d  tokenizers-0.21.4/src/normalizers/byte_level.rs
sha256:93bbbae1b8885989b0163ad4e4141aeb0676928e6754e6a3d010f205f712cdd2  tokenizers-0.21.4/src/normalizers/mod.rs
sha256:d1c837b9b9e61a3255a54f1e98580e54a96765bcb02711a14aef82a80648c26f  tokenizers-0.21.4/src/normalizers/precompiled.rs
sha256:99d7336d9c9ba8baca0f30b435c2a7930332100ec0d58f6221eff94da373fa19  tokenizers-0.21.4/src/normalizers/prepend.rs
sha256:0bc5c139e0f4e05f3ead555b40789327a1c6e765601a8c3b6bdbfec877c875d9  tokenizers-0.21.4/src/normalizers/replace.rs
sha256:3d088593a2fa931261f4cd906440b83c053351a0e0e12cb797e541b98d7e2b06  tokenizers-0.21.4/src/normalizers/strip.rs
sha256:6c2312c8a2ba9461b850147fd63432f59f81e942042995d7c28e097f8198139e  tokenizers-0.21.4/src/normalizers/unicode.rs
sha256:5f598772c1140f0252b5f19fcf2581a803d8d6df097e5709c23838d58d27da2c  tokenizers-0.21.4/src/normalizers/utils.rs
sha256:32dd35c1f815072c6ba32df390389d10e5f4073b480d1c4ade06143ba23b571d  tokenizers-0.21.4/src/pre_tokenizers/bert.rs
sha256:f9b5a747a0ad936f0c49cff8f32bc8f166126704ab02d5af0505ad032fe6d99a  tokenizers-0.21.4/src/pre_tokenizers/byte_level.rs
sha256:7d2d0170eeb5d864742df867958f379ec564bc5af67e835f5793b06bfab4185f  tokenizers-0.21.4/src/pre_tokenizers/delimiter.rs
sha256:1452f56c9fcb0265ee91c5439f860033209e44fe61565b743970d64216b9b41a  tokenizers-0.21.4/src/pre_tokenizers/digits.rs
sha256:d888230bb4003270b85599f4b94478386f0a6c836ce197cd0e50fe469aa11836  tokenizers-0.21.4/src/pre_tokenizers/fixed_length.rs
sha256:0048101657d3d0ab8d6280be30c4ac72d0ee1f00eb2a4ddad9ae45c6f424735b  tokenizers-0.21.4/src/pre_tokenizers/metaspace.rs
sha256:e1face4a2de95b1dfe92472de274d7dc5dbcc6854b1eeeb791a1fb86255aca0d  tokenizers-0.21.4/src/pre_tokenizers/mod.rs
sha256:b0bdf7d5edf658ec5e65d8cd91c3b39c8c2d82e1d3ff9e2ebb959ac21b2cde09  tokenizers-0.21.4/src/pre_tokenizers/punctuation.rs
sha256:ebc4547dd4768e25941add14e9e67122dd10de7695f112031603bc52654b7213  tokenizers-0.21.4/src/pre_tokenizers/sequence.rs
sha256:cf6a6805a0fda58dd732fa7d4361639e27e9b5a87368860907995ba027bd58e1  tokenizers-0.21.4/src/pre_tokenizers/split.rs
sha256:5740e30dba4e6e819309a24144c95d228d9aca437950fc0285b134f238f1788e  tokenizers-0.21.4/src/pre_tokenizers/unicode_scripts/mod.rs
sha256:4ed0d408409ec35bcba141d75a2647d7503bc17d1e6f4641627325c99bd56edb  tokenizers-0.21.4/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
sha256:c614fe1c9a79b1e4cbbb9f3cc3ad130dd6506e12ff74688d611a8eeff47c2167  tokenizers-0.21.4/src/pre_tokenizers/unicode_scripts/scripts.rs
sha256:21bf8000de54d7f09488dfaa04ec8ec6c599decdcb88704494162f54bc65d7c8  tokenizers-0.21.4/src/pre_tokenizers/whitespace.rs
sha256:11954abc42c7d9748e8b690ba40a82bd97cf5741d80cd166a8b46eb0dd4f7f8a  tokenizers-0.21.4/src/processors/bert.rs
sha256:1dfcff4557f06c4306cc353846ff68775cceebd774e507108a5aafc81037eed5  tokenizers-0.21.4/src/processors/mod.rs
sha256:5a00fcfcd37cfd28de62cde9c3f1f21c0045f5be904b8f603db55c603d1357f4  tokenizers-0.21.4/src/processors/roberta.rs
sha256:2d10b84a463f107fde6d532c48fab5a518eac75bc6efc302a60d8f2b1a98a8bf  tokenizers-0.21.4/src/processors/sequence.rs
sha256:f7a120027c11339b2f2cbeb814361469dd21adc0029fe3b4c7c75a68f6f3593d  tokenizers-0.21.4/src/processors/template.rs
sha256:9779a20e0ed2fd8ea0197cbe8ed05c58ba05ce4a11b1464ff8d814afd4b95082  tokenizers-0.21.4/src/tokenizer/added_vocabulary.rs
sha256:9866cf67379cbffb48e912102da5712fb4f84ff10be1269d60805f901cb781c5  tokenizers-0.21.4/src/tokenizer/encoding.rs
sha256:6e52b6af481fd47bd52ad37ceff56bf446eb15e89ec4a8b5316e3fc53b50c2ac  tokenizers-0.21.4/src/tokenizer/mod.rs
sha256:490d2b81fd26a14bb711c37911e960b5edda6f1dd45739da7141a98cbcdc3341  tokenizers-0.21.4/src/tokenizer/normalizer.rs
sha256:ba84d97619bdc8ab3eb7c56bf997493988a498d0840f78945805a93ddbaaf3cb  tokenizers-0.21.4/src/tokenizer/pattern.rs
sha256:2050974242070d86f2492277d4a37892a25afc9d3d60215e6fecfac6516440e7  tokenizers-0.21.4/src/tokenizer/pre_tokenizer.rs
sha256:c8d367a6921c59c48404b344396c5fa6c3a76cf0a6e1b4cf1513e5cef92c2295  tokenizers-0.21.4/src/tokenizer/serialization.rs
sha256:e865470393ae5c651cb1fbc7ea59c444c130ddf924d8c8e6c241a0da3707b602  tokenizers-0.21.4/src/utils/cache.rs
sha256:534ed9e80f94b5355cda3692a5ef9ad15296ec5e8f895dd854ee040d45669382  tokenizers-0.21.4/src/utils/fancy.rs
sha256:6d00a9c45584f0bcd64d0d01110d48444191f5450eab2731280f77ae36500757  tokenizers-0.21.4/src/utils/from_pretrained.rs
sha256:004540ac0fc434f6ec25c1cf8e6818998550da936a728aa5a301458c6c66a540  tokenizers-0.21.4/src/utils/iter.rs
sha256:937da1c859adf74c72c320c51404c05dcaba0a991120cc33491accfb3198f9ef  tokenizers-0.21.4/src/utils/mod.rs
sha256:f6f58fb3fcf2dfbee2e8edc9b763247443fa102aea185481ff591f7d52c5256e  tokenizers-0.21.4/src/utils/onig.rs
sha256:1015ed9beb2adc62e1c1bbab847a1f2fe379acf89a6f2bd9a3e81b92c30d4c69  tokenizers-0.21.4/src/utils/padding.rs
sha256:f207764fa62d362885fd04ac5e14e9e32645a976e57e72e7663e64eaac15d531  tokenizers-0.21.4/src/utils/parallelism.rs
sha256:2bae922cafe5167d36cc6cd973edf4fcf6d0224be4d27fa47b9285dcca447ca8  tokenizers-0.21.4/src/utils/progress.rs
sha256:395080b77eebaaa51549231064830829773e965269a7441f106d0c9e6cc92608  tokenizers-0.21.4/src/utils/truncation.rs
sha256:493c7097ed9a51ac4e6837ff729e71cc2ccd1bba1623e882147ca9447449bff6  tokenizers-0.21.4/tests/added_tokens.rs
sha256:75e506cb6cabd1a07989f086d8b573201a010318057fbb0ba287ea5e2b2d0686  tokenizers-0.21.4/tests/common/mod.rs
sha256:993c04dd7f9e8e1e31c4f52ff42b0ae21cceeed2073caa97b02f736eb91df922  tokenizers-0.21.4/tests/documentation.rs
sha256:c44fc0afd1f9ec6c9ab30218b84866175fee2ae8e4315603fec0af9d1ae0f813  tokenizers-0.21.4/tests/from_pretrained.rs
sha256:f0f790bbe60a47112b4c250991c5769f10fa0ad3bf40fb200922b59caf841933  tokenizers-0.21.4/tests/offsets.rs
sha256:f6cedde9ff05d6b24917efdb7a52b75a31f2d400b52dacf4513894bc479b3857  tokenizers-0.21.4/tests/serialization.rs
sha256:e2ffa9c737605d111180e56de32f8e36ee6726a1e1c7419351c18faf87dc625d  tokenizers-0.21.4/tests/stream.rs
sha256:f0433b135226479693729e4ebe15d91767545b17a5ecefe87717afc42e0dcb19  tokenizers-0.21.4/tests/training.rs
sha256:73192b32a0a6d53a7b22e50e74f48e9b85c1295cd956e0b6def51b5a1f71be4f  tokenizers-0.21.4/tests/unigram.rs

Return to home page.