sha256:fa82b46efc87eaf03bb3be88fbc0c6cfce1eb9fa2805875fd34a051da4ce8f9b
Alias: sha256(gz(tar))
The hash you've requested:
sha256:65dd1f079404161321ece0d1af4f678e5846e3e6056f004600bde776d489bddc
is considered an alias for sha256(tar).
Visit the canonical page.
Build input of
- Gentoo: sci-ml/tokenizers 0.21.1 (https://github.com/huggingface/tokenizers/archive/refs/tags/v0.21.1.tar.gz) blake2b:e076429795d9e34a27c88b7d030e866198819288c5e93556369faaf424140d1f37c6f6bd07672915e60c71ae24aedd135db2ac135d26757a5edc8549821f1564
- Gentoo: sci-ml/tokenizers 0.21.1-r1 (https://github.com/huggingface/tokenizers/archive/refs/tags/v0.21.1.tar.gz) blake2b:e076429795d9e34a27c88b7d030e866198819288c5e93556369faaf424140d1f37c6f6bd07672915e60c71ae24aedd135db2ac135d26757a5edc8549821f1564
- openSUSE: python-tokenizers 0.21.1-1.1 (tokenizers-0.21.1.tar.gz) sha256:65dd1f079404161321ece0d1af4f678e5846e3e6056f004600bde776d489bddc
Contains Dependency Lockfiles
-
sha256:2f4f6ff53bd5c197bb82c3549bd9d3968eb7e9758fd271d213cbf21a49aba531tokenizers-0.21.1/bindings/node/yarn.lock -
sha256:171cdcb7fed12b5f43a4bfb5f418141f1ae15fcfe8d01b9285482f4bd2b3a746tokenizers-0.21.1/bindings/python/Cargo.lock -
sha256:b3a6eb06a5d73eec232ef54a329bb448bfefea5adff6013de7eee94353f41ef9tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/package-lock.json
Diff
Content
tokenizers-0.21.1/
tokenizers-0.21.1/.github/
tokenizers-0.21.1/.github/conda/
sha256:a6591b816fed5e1831c613814bf70c7180e0f015057bad2a5ab6f8d50d6e0c22 tokenizers-0.21.1/.github/conda/bld.bat
sha256:27c731a659d61a4fce9877fea566686a01c3b5bc804f73b9949c2dd3d534b2f6 tokenizers-0.21.1/.github/conda/build.sh
sha256:45f202ad857770620e4e919ebebee205711b49975812ad28f11decc984fdfe7e tokenizers-0.21.1/.github/conda/meta.yaml
sha256:9f92c3d0c0082e9409d62ddf4a7c4612acf8692dcf1b4bed33fdffbe43f153fe tokenizers-0.21.1/.github/stale.yml
tokenizers-0.21.1/.github/workflows/
sha256:b87a5bd6f64976191627e96d0740bbba97f4e402464ae1d223a246ecc4e4d98f tokenizers-0.21.1/.github/workflows/CI.yml
sha256:c3324b307eb3eb71f160357c07ef49f5a86b9a1c6f16672a0946c0195d29c96b tokenizers-0.21.1/.github/workflows/build_documentation.yml
sha256:4e6c12654c7e95e4e51652e096fad2e6bccd3ec5193543ea9b4c8eada7fe80e0 tokenizers-0.21.1/.github/workflows/build_pr_documentation.yml
sha256:57ae5e4c90a9d327f25a9ad148fe7ace589865f1f8d9565d78460facda3f748c tokenizers-0.21.1/.github/workflows/delete_doc_comment.yml
sha256:db118b73b00dc7ccd3aaea04e90b31b6f2bca549160604697182b297d653bb50 tokenizers-0.21.1/.github/workflows/delete_doc_comment_trigger.yml
sha256:d530957189f5d2a22f2ddd659800c921c30c384e93f22d9c11f9c5c4f218d0e1 tokenizers-0.21.1/.github/workflows/docs-check.yml
sha256:24ff5b5d2305bb9a1873e7323ac6b766e83da8bda4ec68e59cb77f7f4c65251c tokenizers-0.21.1/.github/workflows/node-release.yml
sha256:91d4604a2f722d440580e030003e7225ad67e31a7444deff2f22330a333121a2 tokenizers-0.21.1/.github/workflows/node.yml
sha256:0252236cc5193e507abc8a6fa9edfec5622b3d6c88bdd9f516a1c2d4fbed9a92 tokenizers-0.21.1/.github/workflows/python-release-conda.yml
sha256:3ebf7bd8858b1395e2fc0f6ba77d9c02f486643645735971972049fea486ea2b tokenizers-0.21.1/.github/workflows/python-release.yml
sha256:6d19b011023439f862423caad0f50a613205c9a51ab6f04bd9c5e14b087a72de tokenizers-0.21.1/.github/workflows/python.yml
sha256:e331d54ecc193759b5f41c391b8da0570cb70bf2b5704ec16377c30507f70f07 tokenizers-0.21.1/.github/workflows/rust-release.yml
sha256:c1b25b601b542fefe7e54b035f0cc784b9239aa12b48e89ae32e1a10af918e65 tokenizers-0.21.1/.github/workflows/rust.yml
sha256:819e3a9ecb1c3df334a25a296165e686f5f246e237d133c0d1f2fc12e9b6634c tokenizers-0.21.1/.github/workflows/stale.yml
sha256:55c27a5b69209c6920f07d627fcd4dd0950b17c780fb8f89a479474493d9fe8a tokenizers-0.21.1/.github/workflows/trufflehog.yml
sha256:53ff3a72fd52c75b89371910748b4bc05e3718ab640b9751bb7df7cd8d97231f tokenizers-0.21.1/.github/workflows/upload_pr_documentation.yml
sha256:91da3b625c669f733040eaba80a5a8067890e64714ef6089d36a261b532cf7d0 tokenizers-0.21.1/.gitignore
sha256:4d377e818cff7934516a38936b80230a34153c0bbb42dff20f64e84bfad19826 tokenizers-0.21.1/CITATION.cff
sha256:c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4 tokenizers-0.21.1/LICENSE
sha256:90a86d771169e124acf07d804cfc12aaabd7de01afc98c3a61da9a22cdcb6aa5 tokenizers-0.21.1/README.md
sha256:c98eaf43f6046659e2b0e33b24d7ad8041e38be1956925df8d9794bbd1c9b8d5 tokenizers-0.21.1/RELEASE.md
tokenizers-0.21.1/bindings/
tokenizers-0.21.1/bindings/node/
tokenizers-0.21.1/bindings/node/.cargo/
sha256:95eccdbcb0b22f08a34041c422ff5821bd783f9392d8bd1136305239fc1f7287 tokenizers-0.21.1/bindings/node/.cargo/config.toml
sha256:30adc741ffc18728591498ec0d123497e6262c963bf75a91177a0056c50ae9e1 tokenizers-0.21.1/bindings/node/.editorconfig
sha256:f90a518eaf74b6a9c515d44ac0159cf87e3ebc1913792ec0dc38334b6d626eec tokenizers-0.21.1/bindings/node/.eslintrc.yml
sha256:d005ffc2cefc69de8caf1394a6a074754a8bb808fb5ec1d92fc223db3d96bbed tokenizers-0.21.1/bindings/node/.gitattributes
sha256:27071e79228833618f823a63dcaf2d0bca21a2db81ece8edf7422cd9bb992274 tokenizers-0.21.1/bindings/node/.gitignore
sha256:c7206e1a30314ad09058859077f48794851e3079811b7c970a15275b39645a38 tokenizers-0.21.1/bindings/node/.prettierignore
sha256:d32b93a2df8144b2d8e82ac0196e5fa1f162591d00434ff884f1ded5cc50d331 tokenizers-0.21.1/bindings/node/.taplo.toml
tokenizers-0.21.1/bindings/node/.yarn/
tokenizers-0.21.1/bindings/node/.yarn/releases/
sha256:64c0b63e5fa0e21f354b5eff7c9a1b486f32046bfc3283670569e3c6a9cad102 tokenizers-0.21.1/bindings/node/.yarn/releases/yarn-3.5.1.cjs
sha256:bda4132acb74d9f289cf423e33ab81782f03377641413b7626444fa759f52b1d tokenizers-0.21.1/bindings/node/.yarnrc.yml
sha256:bae288b83707bb716e4f636d50f724cf8f91c26530d79a595d1c39616561587d tokenizers-0.21.1/bindings/node/Cargo.toml
sha256:02965ed67e29f871a53f7a7afa593a0ddc8020f86ccce0faaf1aa59215b5269e tokenizers-0.21.1/bindings/node/LICENSE
sha256:e9398e4e77c114500ecb33b03abc0ff98ecd973ee770ad481549232beef59a06 tokenizers-0.21.1/bindings/node/Makefile
sha256:61377fafc00be92c730d6ca920c145de5b1910ddd9a98419309376699da742b2 tokenizers-0.21.1/bindings/node/README.md
sha256:71b61de74caafb5f27db3d24d197b64282f49f807b969e140ca8ce181e5464fc tokenizers-0.21.1/bindings/node/build.rs
tokenizers-0.21.1/bindings/node/examples/
tokenizers-0.21.1/bindings/node/examples/documentation/
sha256:32099423a4192e4ae956656314cfd9c5fefc4b0f0c0783dcd5f5a5206a7a398c tokenizers-0.21.1/bindings/node/examples/documentation/pipeline.test.ts
sha256:f1e06f10edd4dc2144769b5f28ad0f24cb5f08fd26e1e20949ce0611ea012cd9 tokenizers-0.21.1/bindings/node/examples/documentation/quicktour.test.ts
sha256:e928ddd08356b8813a25919156f953fe20584eb49caa5625b1ce0d6d56102b34 tokenizers-0.21.1/bindings/node/index.d.ts
sha256:87cfc88607e694f3eac84d5af93a56dde83efb16567034d4db021a8ef7d591d3 tokenizers-0.21.1/bindings/node/index.js
sha256:fdbdbd048885ea46a8ce806675fb47bb9a04975d55132d7080a458a78a6a2b94 tokenizers-0.21.1/bindings/node/jest.config.js
tokenizers-0.21.1/bindings/node/lib/
tokenizers-0.21.1/bindings/node/lib/bindings/
tokenizers-0.21.1/bindings/node/lib/bindings/__mocks__/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.21.1/bindings/node/lib/bindings/__mocks__/merges.txt
sha256:ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356 tokenizers-0.21.1/bindings/node/lib/bindings/__mocks__/vocab.json
sha256:e2c1d7f033886700bc4d6718e92a3b00547268a5c083ce91eeca79f0771cf26b tokenizers-0.21.1/bindings/node/lib/bindings/__mocks__/vocab.txt
sha256:bae90354fdc246dc1819d8e813f01f7a0b37b8cbc278de5f48c320805819ec25 tokenizers-0.21.1/bindings/node/lib/bindings/decoders.test.ts
sha256:930510ac93ffd04dc780138b29c30f63709a71071aa8b194ce518ad8ed108c1b tokenizers-0.21.1/bindings/node/lib/bindings/encoding.test.ts
sha256:ea23c29304840ecccd24f78a117db88a1380b35d6d3a811626d83e597bdd471c tokenizers-0.21.1/bindings/node/lib/bindings/models.test.ts
sha256:3201d9ca7d7c42d9934f9ae3e3331f9c0488f0adc713f6ff6ec64251c4ab6178 tokenizers-0.21.1/bindings/node/lib/bindings/normalizers.test.ts
sha256:29db4dbd89bd55c88c4a43b6ae24b2610bc05858fc6a55f33cf1c973b72c2198 tokenizers-0.21.1/bindings/node/lib/bindings/post-processors.test.ts
sha256:529ecc9b4adcc0d28ef1b52445898086ed0e5e70bea9d49862820e691b2b6215 tokenizers-0.21.1/bindings/node/lib/bindings/pre-tokenizers.test.ts
sha256:3fd18a74349b98d8bc9392e7a10b3e9ea0335c2b58ee8afe1f97b58a85df1a1a tokenizers-0.21.1/bindings/node/lib/bindings/tokenizer.test.ts
sha256:68c1660db395387412cd5ec38cfd6e01c69f0da362e477acbdbcee561e624c46 tokenizers-0.21.1/bindings/node/lib/bindings/utils.test.ts
tokenizers-0.21.1/bindings/node/npm/
tokenizers-0.21.1/bindings/node/npm/android-arm-eabi/
sha256:05a1d365a6029297f0e0b1209776c25b42cdf7a51a52e8c76b85b542c8c7ab20 tokenizers-0.21.1/bindings/node/npm/android-arm-eabi/README.md
sha256:90a397203ba933dc527dff748946c38d7418ca9cfd62035a20d4e36458e06dec tokenizers-0.21.1/bindings/node/npm/android-arm-eabi/package.json
tokenizers-0.21.1/bindings/node/npm/android-arm64/
sha256:7c0cd0416e0548f05948cf6f4c4d950d966d2107aa3c62e0c4fa47e2ff077240 tokenizers-0.21.1/bindings/node/npm/android-arm64/README.md
sha256:43fed355ff5375b82e34895553f69e3d7a55a6a3173d28038909c5db0d662a73 tokenizers-0.21.1/bindings/node/npm/android-arm64/package.json
tokenizers-0.21.1/bindings/node/npm/darwin-arm64/
sha256:ee3950cafecfb51853d3762ae9369813f4cd397ff0477900ae5623fc0e33540c tokenizers-0.21.1/bindings/node/npm/darwin-arm64/README.md
sha256:a21b46740ea8236cc37ca25a65e5354609bdcf2cdeabfa24461f61f700f470db tokenizers-0.21.1/bindings/node/npm/darwin-arm64/package.json
tokenizers-0.21.1/bindings/node/npm/darwin-x64/
sha256:23b156cb161d34e087477df0a946cfedf47dd665a8cb77bb637a2ea8dfdf80ff tokenizers-0.21.1/bindings/node/npm/darwin-x64/README.md
sha256:aad332092b741c591c39e3898e8f24ccfebb41fc7c1995a2114e2ab84700488d tokenizers-0.21.1/bindings/node/npm/darwin-x64/package.json
tokenizers-0.21.1/bindings/node/npm/freebsd-x64/
sha256:c21c445c4f03fb72275dea0459b584b3d1b107321af13a80b018aacfe89f57e6 tokenizers-0.21.1/bindings/node/npm/freebsd-x64/README.md
sha256:e838711b46934e6b1bfa6477ae6b0d549f07c3f70dcb0b59fcd735ec4e38e22f tokenizers-0.21.1/bindings/node/npm/freebsd-x64/package.json
tokenizers-0.21.1/bindings/node/npm/linux-arm-gnueabihf/
sha256:c485213dbc76af8e6f54bd83be63777f719060e67c87921d426a229ac9363424 tokenizers-0.21.1/bindings/node/npm/linux-arm-gnueabihf/README.md
sha256:0c826d635835adcbc4bb7dde99cac6e9fac6954247c60958b77a63a3a4fb6b6f tokenizers-0.21.1/bindings/node/npm/linux-arm-gnueabihf/package.json
tokenizers-0.21.1/bindings/node/npm/linux-arm64-gnu/
sha256:fca73a7053f15219b1cd717dfe93dad41f1ce8ed0cbe77f06e0bae890be2bcf2 tokenizers-0.21.1/bindings/node/npm/linux-arm64-gnu/README.md
sha256:5de0ee27d1daa0c4d28eba57d66e891a51dc27d65c0cb1843ed47de10ca8399e tokenizers-0.21.1/bindings/node/npm/linux-arm64-gnu/package.json
tokenizers-0.21.1/bindings/node/npm/linux-arm64-musl/
sha256:a42323205e03c06ab50640eb6616e8e29de7f0e1a24bc87a85b5adc39bbc743f tokenizers-0.21.1/bindings/node/npm/linux-arm64-musl/README.md
sha256:ad55cfef52a07822aad94d364c895ee7a9ddaf51be38681ac2b62f88a5beef52 tokenizers-0.21.1/bindings/node/npm/linux-arm64-musl/package.json
tokenizers-0.21.1/bindings/node/npm/linux-x64-gnu/
sha256:b8f3cdc50472110fee2373517b0ef96ce803641d4c74b95f82ffc3f85b449841 tokenizers-0.21.1/bindings/node/npm/linux-x64-gnu/README.md
sha256:9f936b7429d60f3dab9319422da2216b31ba0330311284668e2161c6cac0f22d tokenizers-0.21.1/bindings/node/npm/linux-x64-gnu/package.json
tokenizers-0.21.1/bindings/node/npm/linux-x64-musl/
sha256:9b4cc26fdf788f433f1d7dcbd9aed1979b655418cc8b9810120eb01651f7f18d tokenizers-0.21.1/bindings/node/npm/linux-x64-musl/README.md
sha256:4260f5c07c5cf19f2e9f44500f17162c9c77955582b0887d6a26b143ea5acba2 tokenizers-0.21.1/bindings/node/npm/linux-x64-musl/package.json
tokenizers-0.21.1/bindings/node/npm/win32-arm64-msvc/
sha256:cf057a5481c42a26b03fffa4d33b6d196a10a81223992f1a17271b27290d06a4 tokenizers-0.21.1/bindings/node/npm/win32-arm64-msvc/README.md
sha256:6c35853dc168d4204bdd98c2b1709be3f0fc6b1d898b2c5fdad106fa2c28841b tokenizers-0.21.1/bindings/node/npm/win32-arm64-msvc/package.json
tokenizers-0.21.1/bindings/node/npm/win32-ia32-msvc/
sha256:df2c8f0c9d538872f1c10995666f6a9a6b13b6af4eb202efb76b90c24bc980b5 tokenizers-0.21.1/bindings/node/npm/win32-ia32-msvc/README.md
sha256:366da5b97cdde4fe1f9ba87346cafa178042fc528a9d41726dcea4863f7e55f4 tokenizers-0.21.1/bindings/node/npm/win32-ia32-msvc/package.json
tokenizers-0.21.1/bindings/node/npm/win32-x64-msvc/
sha256:08b6de0abc7178dd64cc5054624a04545d0ac3e1019822452809867fa56b45df tokenizers-0.21.1/bindings/node/npm/win32-x64-msvc/README.md
sha256:6ba80cc97010f6c17faa664569295b97add7252b504ad46a99818e31fd89161c tokenizers-0.21.1/bindings/node/npm/win32-x64-msvc/package.json
sha256:9cf3cca1abebba89176c5fd7a7e6e13ef75706b3db6395081b08119abea31031 tokenizers-0.21.1/bindings/node/package.json
sha256:d53b7e5653f4a60e4f6558b9d7704045ec0ff0687a768ff3163a67acab4c9f6d tokenizers-0.21.1/bindings/node/rustfmt.toml
tokenizers-0.21.1/bindings/node/src/
sha256:0c288ea5db6f2fcf4f9fb37e46472b5d27ace113310440b9681ad745abe8462e tokenizers-0.21.1/bindings/node/src/arc_rwlock_serde.rs
sha256:478285b75a75c90832a4461315cc4cd8c77a8989db74f81f2d0e8f0df6cbe5f6 tokenizers-0.21.1/bindings/node/src/decoders.rs
sha256:03cbe893bb55b4609205d94627e77e60b54dd7ef751a9291efac35265bbc2018 tokenizers-0.21.1/bindings/node/src/encoding.rs
sha256:23e01dcea44b1fc841dc18ad525f6fd5c5e4a94843c1368e802de32bb06e90ad tokenizers-0.21.1/bindings/node/src/lib.rs
sha256:2dc0b75569ede4063f503211cde250abab990460afb495abc79393392f655def tokenizers-0.21.1/bindings/node/src/models.rs
sha256:f7dc43ad7c794ffe23fc29ef4dfa6e3df50fe5f1145c23d7c3abac332a78ba57 tokenizers-0.21.1/bindings/node/src/normalizers.rs
sha256:d54f97b5a1dbe3d539185c1384c158ac300f4dba77a9ee0fe8891bfad9e7c9f2 tokenizers-0.21.1/bindings/node/src/pre_tokenizers.rs
sha256:27c5d78c318c9f15ca440715fc5297b79655fb0b787fc1e1eb9cc5a5dca7ccb3 tokenizers-0.21.1/bindings/node/src/processors.rs
tokenizers-0.21.1/bindings/node/src/tasks/
sha256:97302dbce5e72f37cf4a74de57e17084a6ef98f41a9cc832348842d8ae9e4de1 tokenizers-0.21.1/bindings/node/src/tasks/mod.rs
sha256:7cb04541da8f4b29346caacf41dcfe873b6f9ffab4f4756f14710d3f06025f05 tokenizers-0.21.1/bindings/node/src/tasks/models.rs
sha256:06de8bd57a41b18dce7b169f3798d6c3dadcf361a9006cfcfb792cdbdf9ee4c1 tokenizers-0.21.1/bindings/node/src/tasks/tokenizer.rs
sha256:b50650707bca5a89b6037d0e9f1a074d708a084b2c701f3735fb102874fe17a6 tokenizers-0.21.1/bindings/node/src/tokenizer.rs
sha256:67a3f03a137bf83c2351f736c68ab5bce13016d7f76f838a1b3461c0c0ef061f tokenizers-0.21.1/bindings/node/src/trainers.rs
sha256:7c7ab6fcdd34f3fc94a6652c1af166e03b886a1b7a7b40366011efd04efdec5f tokenizers-0.21.1/bindings/node/src/utils.rs
sha256:3be08c632028bf9f54b8aa711469f6207127e2bcae5cf88b4b9da18d788a3db4 tokenizers-0.21.1/bindings/node/tsconfig.json
sha256:110ee41ab1de180ecc708c8094a815b5023e62a309d9417ebd2d6cf014298896 tokenizers-0.21.1/bindings/node/types.ts
sha256:2f4f6ff53bd5c197bb82c3549bd9d3968eb7e9758fd271d213cbf21a49aba531 tokenizers-0.21.1/bindings/node/yarn.lock
tokenizers-0.21.1/bindings/python/
tokenizers-0.21.1/bindings/python/.cargo/
sha256:0f8f7159b2550efdd76855ec4620c6fd21d85648caa9eb71f41aad19404ee92b tokenizers-0.21.1/bindings/python/.cargo/config.toml
sha256:6667b2d1aab6a00caa5aee5af8ad9f1465e567abf1c209d15727d57b3e8f6e5f tokenizers-0.21.1/bindings/python/.gitignore
sha256:c0778b9c5f36c46141cb570ba45615150476d130fa00a128e1f0331417b361fc tokenizers-0.21.1/bindings/python/CHANGELOG.md
sha256:171cdcb7fed12b5f43a4bfb5f418141f1ae15fcfe8d01b9285482f4bd2b3a746 tokenizers-0.21.1/bindings/python/Cargo.lock
sha256:334a4140d64aa760f871052369785a116e26935149c78d93b37c18ff90eca9a6 tokenizers-0.21.1/bindings/python/Cargo.toml
sha256:b5bba29c1d37582948f7837cd44ca176452f386a8c4d41f2fccd95ab17859abf tokenizers-0.21.1/bindings/python/MANIFEST.in
sha256:c5d3de0aadf6f7933909e43c9af4bfc905d1b3c64e4e6e70b04919be11e1e052 tokenizers-0.21.1/bindings/python/Makefile
sha256:b228767941e834617aa3a7c02f278edf433ebc34adb9236c9d7f160eab883971 tokenizers-0.21.1/bindings/python/README.md
tokenizers-0.21.1/bindings/python/benches/
sha256:3c1718bb820865e98d14ca25d0e9a95bfab1b2296f2226a489ed8bc272c283c8 tokenizers-0.21.1/bindings/python/benches/test_tiktoken.py
sha256:d5fec928f9fee467cbac5b91750c56ce65a646784a8395225dd6b0016b910845 tokenizers-0.21.1/bindings/python/conftest.py
tokenizers-0.21.1/bindings/python/examples/
sha256:46047f44661bbd2454a424a3723f27c01c7b928130229107fb35e089685b5a56 tokenizers-0.21.1/bindings/python/examples/custom_components.py
sha256:7259e3f692c90639d339931d594eeabb878b8cc8e031e960430946f1dcc2fca5 tokenizers-0.21.1/bindings/python/examples/example.py
sha256:58ebba107e08b4c45703c9ba9aad14d143142c8592949e4903ae8e06c830421e tokenizers-0.21.1/bindings/python/examples/train_bert_wordpiece.py
sha256:ec2d5897b2646b8017acbbfa404c212e0142dae05d7df0cf4a3c43e4d0b68d26 tokenizers-0.21.1/bindings/python/examples/train_bytelevel_bpe.py
sha256:31ec2b613b7114b20dc83c198130cfea07e3cfcb6d7e98527cc9dcb1a343ef87 tokenizers-0.21.1/bindings/python/examples/train_with_datasets.py
sha256:26f2b0631f6168d440caba51c5b3c0f037a9e3f2b9cea167c928c8b60474cead tokenizers-0.21.1/bindings/python/examples/using_the_visualizer.ipynb
tokenizers-0.21.1/bindings/python/py_src/
tokenizers-0.21.1/bindings/python/py_src/tokenizers/
sha256:644e596a052fa1b05272b1c141d1286e1c78c2a3346ecabd17d68b62404d8d84 tokenizers-0.21.1/bindings/python/py_src/tokenizers/__init__.py
sha256:8f0df85995da62ef0d041276fdcca97cea8963123b09728fce5bde8ac5f0e174 tokenizers-0.21.1/bindings/python/py_src/tokenizers/__init__.pyi
tokenizers-0.21.1/bindings/python/py_src/tokenizers/decoders/
sha256:85fc0ce821540ef94c1862f8fb1b1a698cfcd4af4fe6b408e592f950758af18e tokenizers-0.21.1/bindings/python/py_src/tokenizers/decoders/__init__.py
sha256:ab294c8a73ca9f6fcd00c0ba073717a85ca776889c2f0b37e10e8e2c7d60729e tokenizers-0.21.1/bindings/python/py_src/tokenizers/decoders/__init__.pyi
tokenizers-0.21.1/bindings/python/py_src/tokenizers/implementations/
sha256:57302ca65688a3bae5e0014ef0c8aeee283b31f663be89f055b959c34d7347a3 tokenizers-0.21.1/bindings/python/py_src/tokenizers/implementations/__init__.py
sha256:d9315984bba968988c0d8189b94366c5826ff9c9d1f1b0c799b333698a4544eb tokenizers-0.21.1/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
sha256:b0a0ae9b414a3d875280914937c2c37ab541a130d14aacaa49dadc9be96f42a2 tokenizers-0.21.1/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
sha256:380fe3cb2dc44266136ba8677fe10ac0b385bb2ae8a8560e273db9cac3360549 tokenizers-0.21.1/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
sha256:436644016d313101c5ed8094b66a65c1ac5b53e2743f634ae0f24633151bfbf7 tokenizers-0.21.1/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
sha256:2f0ae87e8a219d47cc13694ada5418a3240884ff3844fd022251d16a3d21c8db tokenizers-0.21.1/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
sha256:4988955cbf19b6a2d72a9baac27c26adfc601a8b6ef320243aeedd2f3b445c8a tokenizers-0.21.1/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
tokenizers-0.21.1/bindings/python/py_src/tokenizers/models/
sha256:7896781d3010669c672882cdca559a4c5ab15f2f816ba38ab3058de3b7de795f tokenizers-0.21.1/bindings/python/py_src/tokenizers/models/__init__.py
sha256:7253d3c22ca3cfb16555d12ec28ff759afd3990adb6615b44869a737295ea676 tokenizers-0.21.1/bindings/python/py_src/tokenizers/models/__init__.pyi
tokenizers-0.21.1/bindings/python/py_src/tokenizers/normalizers/
sha256:ff4eb0e1ca9122dbde12021d75868b3127209123a4200308cd809eb1be400385 tokenizers-0.21.1/bindings/python/py_src/tokenizers/normalizers/__init__.py
sha256:95216a0dbfe53d905f471106f7d11c144694d4796722121052eef3648c8fe006 tokenizers-0.21.1/bindings/python/py_src/tokenizers/normalizers/__init__.pyi
tokenizers-0.21.1/bindings/python/py_src/tokenizers/pre_tokenizers/
sha256:c1de8a61003f46c19240af87786f68a53461bf8b6d4919329e8d9d93a6b3f8f3 tokenizers-0.21.1/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py
sha256:93d2677b3fade30c3f3f28fc3a455ea3bf2ff539ed70d9faae2f64c01e7dd0a1 tokenizers-0.21.1/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
tokenizers-0.21.1/bindings/python/py_src/tokenizers/processors/
sha256:c4cd8310ac0ab4722e987b2ccccf00324abe0256aabc16455d680b53c48d84e6 tokenizers-0.21.1/bindings/python/py_src/tokenizers/processors/__init__.py
sha256:871efaed963c4878716ff8625cf4719be7ff29ca11e170f1eef7cad9cd2547e4 tokenizers-0.21.1/bindings/python/py_src/tokenizers/processors/__init__.pyi
tokenizers-0.21.1/bindings/python/py_src/tokenizers/tools/
sha256:c46f1c681f4e1c2f1c6c1d354b9bd8575e0767184ee9e59b2de86c6fbd29a62a tokenizers-0.21.1/bindings/python/py_src/tokenizers/tools/__init__.py
sha256:cc0c9dab5a06583f10125978f9ec8bf0b970d01d6cb72fe1a48137b58c4bd369 tokenizers-0.21.1/bindings/python/py_src/tokenizers/tools/visualizer-styles.css
sha256:d16f74b38426f0d77a3fe9e9a905fe6c230b4027c010f936aa08fe2bcafb38c7 tokenizers-0.21.1/bindings/python/py_src/tokenizers/tools/visualizer.py
tokenizers-0.21.1/bindings/python/py_src/tokenizers/trainers/
sha256:513bb6d8019ca7be88be95b8e712d16c95844f4e0dc4f5ba35f09bd98633d043 tokenizers-0.21.1/bindings/python/py_src/tokenizers/trainers/__init__.py
sha256:dd3c052adb3899eef341f5517124e6b576223f85f07118df018b70aa8655b684 tokenizers-0.21.1/bindings/python/py_src/tokenizers/trainers/__init__.pyi
sha256:42fda9ff6b16d260cfce9b6d2928da7f325954e803770a50b80520899c2ff4d2 tokenizers-0.21.1/bindings/python/pyproject.toml
sha256:2b92ea252be0fbc26f70317cdaa7b6411ea634b50d55338cd8c495e4dbf25d1d tokenizers-0.21.1/bindings/python/rust-toolchain
tokenizers-0.21.1/bindings/python/scripts/
sha256:4175726b76ac3ea2ce39bf4661f5e26e9d14e9dcefea5c6e4afc91f01710bb0e tokenizers-0.21.1/bindings/python/scripts/convert.py
sha256:68c16a7e0e00b540cc8a74c49823fd1a0b0f5de7b595f3426ba90d4bafd7d9ad tokenizers-0.21.1/bindings/python/scripts/sentencepiece_extractor.py
sha256:85726a5fe68222c61ddaf1a4dc4bfa59bfdfd9a5da6e1fe5dd76064399a92112 tokenizers-0.21.1/bindings/python/scripts/spm_parity_check.py
sha256:32e7d02fa00cb911b747f419942188de07ebd841aa89bde7ce9c256173ffd696 tokenizers-0.21.1/bindings/python/setup.cfg
tokenizers-0.21.1/bindings/python/src/
sha256:444095038c5b4d72f49c90223e5aa3ab1b06b326a936d8f4253c060a16616f6b tokenizers-0.21.1/bindings/python/src/decoders.rs
sha256:33d9496d661107afa25949fd716e9f1a711cad0c0c74684821db447c875392ff tokenizers-0.21.1/bindings/python/src/encoding.rs
sha256:dceb6771db969593a351d8ea296436ee2cd6e4829c5dd016517cf94545e074d8 tokenizers-0.21.1/bindings/python/src/error.rs
sha256:bc60a54af8e1a923c95c28ef0e28477c90c212511538fbc047db7be5bf3c6afa tokenizers-0.21.1/bindings/python/src/lib.rs
sha256:28f6250ee18f87e6a616524ac8bbd486bd9a5ef465b507d8aaedb2a0889cddbd tokenizers-0.21.1/bindings/python/src/models.rs
sha256:0ed9632b32608f1d06dc6b0c5060c0ff147c03b0c338c57a40e1789a2fd01cc6 tokenizers-0.21.1/bindings/python/src/normalizers.rs
sha256:bbad20adf6ddf66f4d58940114f267e8ea2ca7b41a27dd5dd112fb542665a33c tokenizers-0.21.1/bindings/python/src/pre_tokenizers.rs
sha256:a475324b0d59726ce71dc5cd122f982db56125eac7dee7d63fde489286f5c08e tokenizers-0.21.1/bindings/python/src/processors.rs
sha256:9e49b34d599baa0135cb2f9ba0e380fff813eeb5bf96b29976dac5f7fb68f1a6 tokenizers-0.21.1/bindings/python/src/token.rs
sha256:18e392e1ea718587c0d631b299f7ab36243361371caa9f588d839506e1c74a98 tokenizers-0.21.1/bindings/python/src/tokenizer.rs
sha256:9b2fbe9e092d68c79cfb8f182fccdf37b314613540e7725035c1eb1dffed774e tokenizers-0.21.1/bindings/python/src/trainers.rs
tokenizers-0.21.1/bindings/python/src/utils/
sha256:5652b0834efe55de07e890c52b6e6b12d470f5e2493b605ca8fb608b4623c0c5 tokenizers-0.21.1/bindings/python/src/utils/iterators.rs
sha256:f10237229107d96fc7af97cf93b4528b68960c793ba256c6b2987c2390c7fa76 tokenizers-0.21.1/bindings/python/src/utils/mod.rs
sha256:9d276346d0437a476ab795f305a7351ad70a6d8fc3199800c986adb35bdff0ab tokenizers-0.21.1/bindings/python/src/utils/normalization.rs
sha256:68ef27ac310c4132b344a7fa63db9c35f7fee23a623d98a11bf5d42769d6bf71 tokenizers-0.21.1/bindings/python/src/utils/pretokenization.rs
sha256:9d98c86f47aad177f85b1ebe2dec22443ae45b0972c8ee489d40842e7e011ab8 tokenizers-0.21.1/bindings/python/src/utils/regex.rs
sha256:9b82e9bffda22b7f5e413473a2c4a657dc69f7d88492315c7e5f3955f3bbe9e1 tokenizers-0.21.1/bindings/python/src/utils/serde_pyo3.rs
sha256:d5009041324b13576061b14a0f5cf934c7aec62d21826e4a041110e0a2ea4bf5 tokenizers-0.21.1/bindings/python/stub.py
sha256:70e03c19780c5e7c1f31580930d36f0aa4a64e5fc8e601dd50b74b301729b3ee tokenizers-0.21.1/bindings/python/test.txt
tokenizers-0.21.1/bindings/python/tests/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.21.1/bindings/python/tests/__init__.py
tokenizers-0.21.1/bindings/python/tests/bindings/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.21.1/bindings/python/tests/bindings/__init__.py
sha256:7493e7592df728bb99d33f3074b327a944d4bca2a76210112e014f093beebc75 tokenizers-0.21.1/bindings/python/tests/bindings/test_decoders.py
sha256:5f4b218bbfdaa27751080e4451b8fd4d31fda7c41d661da90c604dbd00339b47 tokenizers-0.21.1/bindings/python/tests/bindings/test_encoding.py
sha256:a816aa009317234855304ed576b804a15035d2efa39e944497212f78adc4d17d tokenizers-0.21.1/bindings/python/tests/bindings/test_models.py
sha256:18e2d28d3853211c4999702fec14cb21ed48695c5b87c92d5b9f80d845e4b059 tokenizers-0.21.1/bindings/python/tests/bindings/test_normalizers.py
sha256:553ac2d6f4333b0faa159c214c9ace503ad3a6d527b204d34ed7011143b00452 tokenizers-0.21.1/bindings/python/tests/bindings/test_pre_tokenizers.py
sha256:ed828e36350ed49c27c4d023371f0ee97defc510d08f0919fcc616a0bb21cb23 tokenizers-0.21.1/bindings/python/tests/bindings/test_processors.py
sha256:3b13d916293d61800d3b095db0b780bd9814e8351b659dc1bfbb6bad81481e61 tokenizers-0.21.1/bindings/python/tests/bindings/test_tokenizer.py
sha256:78df24cd553f4ef8e04fedbf0f78eb6d131ec2bb30b2a9d032c540ae76ce1117 tokenizers-0.21.1/bindings/python/tests/bindings/test_trainers.py
tokenizers-0.21.1/bindings/python/tests/documentation/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.21.1/bindings/python/tests/documentation/__init__.py
sha256:3daf9785f1f14c8f0deccd58dd62fa91f735e54ce2677181d4efbb278dbee4cb tokenizers-0.21.1/bindings/python/tests/documentation/test_pipeline.py
sha256:897010ff492659bedccb321c1eb65006e5ea5e789f0973e34e012f02ae967690 tokenizers-0.21.1/bindings/python/tests/documentation/test_quicktour.py
sha256:ada99dd658322f83f5beee7475a65f686afc234c277d5f09812280b8945efe2b tokenizers-0.21.1/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py
tokenizers-0.21.1/bindings/python/tests/implementations/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.21.1/bindings/python/tests/implementations/__init__.py
sha256:43a84bea7130ba6b1773f3aba409ffd395e4a3b632c0ebede89d96a8a30c75b2 tokenizers-0.21.1/bindings/python/tests/implementations/test_base_tokenizer.py
sha256:51cab1df7f9f4693b143f670b2aa1b3c5fc01dc609b18f1127f9e8d07743794a tokenizers-0.21.1/bindings/python/tests/implementations/test_bert_wordpiece.py
sha256:24683c9bab55fe699f4887b3fbfe5b4990b3ad815fa90ef256380822cd60f237 tokenizers-0.21.1/bindings/python/tests/implementations/test_byte_level_bpe.py
sha256:f6cb83befa8ee6d7ea2cfbb105d0fdbb0fd8fd5cba045b4f1a5f523eddeb193d tokenizers-0.21.1/bindings/python/tests/implementations/test_char_bpe.py
sha256:9f54e6ed6d8856c4ec0a3af33936e26f8e3a402b141d2e69397fd02c27564b0c tokenizers-0.21.1/bindings/python/tests/implementations/test_sentencepiece.py
sha256:9f26a1151cd0440d9f4a6f523c69187c2e6157783410ac6f3b5321ba884fd756 tokenizers-0.21.1/bindings/python/tests/test_serialization.py
sha256:b4e4576d427cf2c8b7b634c402ed1c8f45cbb030d1a4e30841502ec69f369b0d tokenizers-0.21.1/bindings/python/tests/utils.py
tokenizers-0.21.1/docs/
sha256:64da2857b2b8b721109ede3b1b8c1469281129af0c45c2213b17916c1c1efd73 tokenizers-0.21.1/docs/Makefile
sha256:9a5efb500e56a297d44431026a0911a85440854ad8be8d80a955eb76f2be361b tokenizers-0.21.1/docs/README.md
tokenizers-0.21.1/docs/source-doc-builder/
sha256:8c9e3c6b89eae2b85b98ee7fd27341be9495d601d0778b8db0bb06c4f9a413bc tokenizers-0.21.1/docs/source-doc-builder/_toctree.yml
tokenizers-0.21.1/docs/source-doc-builder/api/
sha256:6c80b2c6fafea22cc8210e8c38fc8a22934e532f8b5d7057ccfb9da03794c0c5 tokenizers-0.21.1/docs/source-doc-builder/api/added-tokens.mdx
sha256:aa600d7ecf65f56ad2c6fb115950c5e421d360e09a354e0722bbb59fbb92d504 tokenizers-0.21.1/docs/source-doc-builder/api/decoders.mdx
sha256:af65169fdb9a777f17c7ac74dc47af1308882c669ba14309bdb13151eb534ab9 tokenizers-0.21.1/docs/source-doc-builder/api/encode-inputs.mdx
sha256:7ab4617e68210c301a5cec7652fe499351b52b22775c14e19d51ae09f6e9852d tokenizers-0.21.1/docs/source-doc-builder/api/encoding.mdx
sha256:c26868132cce48ddfcdbc287e7ec54af20e79fc54942211eb1e54de329a2a224 tokenizers-0.21.1/docs/source-doc-builder/api/input-sequences.mdx
sha256:0fa43188b0569e2452ce4a2cdad61c4d966653f1a5b016e6e34dee691360c74e tokenizers-0.21.1/docs/source-doc-builder/api/models.mdx
sha256:5b8e0dfb7ac14288d96825dfd0cb60ec0265ae42dea19f59a30b209a210e72ff tokenizers-0.21.1/docs/source-doc-builder/api/normalizers.mdx
sha256:af1dcc0d43802ed5aac6fdc271c99312b0741685712738e11884240b2df17274 tokenizers-0.21.1/docs/source-doc-builder/api/post-processors.mdx
sha256:7f6172e90e8768ab13eff60a68486c3a67ec0b18ab61029f263f71e678f881ad tokenizers-0.21.1/docs/source-doc-builder/api/pre-tokenizers.mdx
sha256:33c9d09bcc5dd3f5775f8521ad5e0665f357873c389615aa260bea4ed7eafb39 tokenizers-0.21.1/docs/source-doc-builder/api/tokenizer.mdx
sha256:743339cf86db82446d01a4afbbe24b86c1fd42998c76948bb7f7fa5792208829 tokenizers-0.21.1/docs/source-doc-builder/api/trainers.mdx
sha256:3b08ec9077407db0076bf25b9efc271473b3917264f77348ee40dc7dca4b0ec3 tokenizers-0.21.1/docs/source-doc-builder/api/visualizer.mdx
sha256:0bf776a64f56d021a701d6981c529b0229610a71279bcd500ab77ae31b7998c3 tokenizers-0.21.1/docs/source-doc-builder/components.mdx
sha256:bf6a3a26a19f0d90bdc24fea638c8b4f12956ac23718e20a425ad20fdf9f537c tokenizers-0.21.1/docs/source-doc-builder/index.mdx
sha256:64c8a8225697260de7c364e69cb33cee0e374281dc0bdee323d0b3cc8ea32f19 tokenizers-0.21.1/docs/source-doc-builder/installation.mdx
sha256:25f1b73c5f1b7c3b3a0f978377af044ac46ba77172978e7ebacaac2c71b07b4d tokenizers-0.21.1/docs/source-doc-builder/pipeline.mdx
sha256:c719c152095b152e9dd8100f85f0ac8b9b62e43a5d773d4ef68c82f8fd099010 tokenizers-0.21.1/docs/source-doc-builder/quicktour.mdx
sha256:1bd0aeccbcc56d57ad00a272f46773d262f6ce24827cf02227e00354a1779eb4 tokenizers-0.21.1/docs/source-doc-builder/training_from_memory.mdx
tokenizers-0.21.1/docs/source/
tokenizers-0.21.1/docs/source/_ext/
sha256:18d280dbaa0a211a5745e18b271ca6f20b5133a1dbb59ef412cc24c032301939 tokenizers-0.21.1/docs/source/_ext/entities.py
sha256:f6b1fd1f777b23885ed350bd0dea306b7ff6bfabcaae067f56f7ebb7847ef895 tokenizers-0.21.1/docs/source/_ext/rust_doc.py
sha256:d1079093942989915b32985c10183b83445e305370c1aeba026b0bcdeb8644fd tokenizers-0.21.1/docs/source/_ext/toctree_tags.py
tokenizers-0.21.1/docs/source/_static/
tokenizers-0.21.1/docs/source/_static/css/
sha256:d68e6fef383ffcee718a5728690415a204fddda208cd49a2df4931257bb3c476 tokenizers-0.21.1/docs/source/_static/css/Calibre-Light.ttf
sha256:2cc588a746277a0e8a4bc64b95b63aed4ce60061f43cef06d44899b9d0395eec tokenizers-0.21.1/docs/source/_static/css/Calibre-Medium.otf
sha256:8f3af66227ee366466149c568be461f579bac53e411a39acc33583c7b310dfef tokenizers-0.21.1/docs/source/_static/css/Calibre-Regular.otf
sha256:83148cb95241197bcbb93352458548c630dfb707e822f07d2f0f03a3f61b0870 tokenizers-0.21.1/docs/source/_static/css/Calibre-Thin.otf
sha256:a6826e671da01a2fa5a0409e4f9f2edbcf1d3059a988548dd439774906ec448b tokenizers-0.21.1/docs/source/_static/css/code-snippets.css
sha256:c712f6e603226ead1b3a08733599979d1fbb7c063cd14223b65ed9faed78cad7 tokenizers-0.21.1/docs/source/_static/css/huggingface.css
tokenizers-0.21.1/docs/source/_static/js/
sha256:82a457d4b85c2d1a9c8af4de785a98acba9bce5e0e5c27cba823c72cef3d5acd tokenizers-0.21.1/docs/source/_static/js/custom.js
tokenizers-0.21.1/docs/source/api/
sha256:ff2cc6e66aba30b79ec0aadd3e2b214095d47ef1f7c1d7dac701cfc056560f24 tokenizers-0.21.1/docs/source/api/node.inc
sha256:591900f42ad20d8cd396a5fb77cc440be3a60e698b61ae3babc818e662e0c68a tokenizers-0.21.1/docs/source/api/python.inc
sha256:c21848d284baef80f8c8ecfc057648c509e91b400fde506304da004e2f22cd34 tokenizers-0.21.1/docs/source/api/reference.rst
sha256:8253c97c3382c969f82ac09ce2854f40cbe530f97ac03d2404abfe5141e31dc3 tokenizers-0.21.1/docs/source/api/rust.inc
sha256:c5758d9771c9693992880e6e901c5dc506a7d146a3dac4cf53ee2e28c6d08106 tokenizers-0.21.1/docs/source/components.rst
sha256:93c5dafd101322d8f98c26943744735f99b5f67d93e809dd721b4ebf90371310 tokenizers-0.21.1/docs/source/conf.py
sha256:eb7dfb878646f949ce8e2eae062af4768a79e9afcb200fa124f50201570059e8 tokenizers-0.21.1/docs/source/entities.inc
sha256:3a6453b0107cf0421a04d046a09b736cc248a4ffa847e9e3a7325854455606f7 tokenizers-0.21.1/docs/source/index.rst
tokenizers-0.21.1/docs/source/installation/
sha256:7d7d111f50cec5ea6f39b7b35db11688e4cc0c8acba096e27ff1d053506244d5 tokenizers-0.21.1/docs/source/installation/main.rst
sha256:6590d59131ae3dc72cf0e88106b9a75fa4fbf4ddf4cc713c19536832e13bbaeb tokenizers-0.21.1/docs/source/installation/node.inc
sha256:41675907e4e9a4681a44cedc5e6aedd4e6360665f89941d665d9e13ab56296ac tokenizers-0.21.1/docs/source/installation/python.inc
sha256:858c8ab604d54821c8f1b43f9d18638b5c8b703fb098058f3e329f4828e3910a tokenizers-0.21.1/docs/source/installation/rust.inc
sha256:670f6cde0c8918f722e0c8128b1fda6ce625635bbb5630680666d6cad2a0779a tokenizers-0.21.1/docs/source/pipeline.rst
sha256:78452aac2be736ee2313929e49b8c342dd1182bc8ad5b75e6d6a1998e1353e08 tokenizers-0.21.1/docs/source/quicktour.rst
tokenizers-0.21.1/docs/source/tutorials/
tokenizers-0.21.1/docs/source/tutorials/python/
sha256:c78ea0da873d0899476945f9a00091bffbc2fc2a6036d5df2a706abf9ba26d4e tokenizers-0.21.1/docs/source/tutorials/python/training_from_memory.rst
tokenizers-0.21.1/tokenizers/
sha256:48555a205c39e1b840ad65203e1359ddea42476b9f515b7340d09bdc52d56242 tokenizers-0.21.1/tokenizers/CHANGELOG.md
sha256:eb5ca752ad3ed9b8f77061ff0ad43ff5206e1cd08cda3f889e5a8a3e303cb861 tokenizers-0.21.1/tokenizers/Cargo.toml
tokenizers-0.21.1/tokenizers/LICENSE -> ../LICENSE
sha256:0c3217a20a94e1161932a20147a3db4d3063112d6f25840b6aee3d46584f1093 tokenizers-0.21.1/tokenizers/Makefile
sha256:d0da41571af903fe9f23d5da9c569f941a0a40d4ea88bbc03dd4ac139c5a3e7c tokenizers-0.21.1/tokenizers/README.md
sha256:487d57adf5a2cb4729326ce4a0c85e80c965e8951f3b38e6bc3868902519b39f tokenizers-0.21.1/tokenizers/README.tpl
tokenizers-0.21.1/tokenizers/benches/
sha256:62e3df468e2f41c8f3c555164553ac72c2a5e7bc0f530f39d16f833b8ebca2db tokenizers-0.21.1/tokenizers/benches/bert_benchmark.rs
sha256:684ea83441a4bf1249e89d47c1966cc90f8acdea02afef8733a9c6cb8250e873 tokenizers-0.21.1/tokenizers/benches/bpe_benchmark.rs
tokenizers-0.21.1/tokenizers/benches/common/
sha256:2fdeba251f5d18b991b868a5902d8aa6f5eba8837fd62debb3422351c83ac184 tokenizers-0.21.1/tokenizers/benches/common/mod.rs
sha256:bc71182748a4b70f26cf45e4f819adff1bd48654af6f0253d4c70ecd1bef2a6a tokenizers-0.21.1/tokenizers/benches/layout_benchmark.rs
sha256:c137b80a7d5f1f9f08f7b5ee427089e70a06dffd02c18e40e283530137a2a49e tokenizers-0.21.1/tokenizers/benches/llama3.rs
sha256:1c3b8edfc263f509453742306aa839eaa6cc53b30e0fbe2f0260c7e5a4fbf39c tokenizers-0.21.1/tokenizers/benches/unigram_benchmark.rs
tokenizers-0.21.1/tokenizers/examples/
sha256:2bd564996ff859e1c996238f38f576657e566688bbd435c8f100e3bb0cdc43af tokenizers-0.21.1/tokenizers/examples/encode_batch.rs
sha256:8f35712a217fef4bccf538a3bc0c71bb460096ff751169a057a2165bc9ee57e1 tokenizers-0.21.1/tokenizers/examples/serialization.rs
tokenizers-0.21.1/tokenizers/examples/unstable_wasm/
sha256:f5ba0dc3ff1ad2d818ccbe695f554c31822aea61713639e7d5b76d9a98f2ecf5 tokenizers-0.21.1/tokenizers/examples/unstable_wasm/.gitignore
sha256:c0a32c90ef7eda62d0dcadee83f91759d1251297c715555a39586d59df98ac1b tokenizers-0.21.1/tokenizers/examples/unstable_wasm/Cargo.toml
sha256:b83a774dea05794aa3996310404dad4806617358488558a6c96f5d5922269b7b tokenizers-0.21.1/tokenizers/examples/unstable_wasm/README.md
tokenizers-0.21.1/tokenizers/examples/unstable_wasm/src/
sha256:700108192809e07d83d25abd3c5465d0d8c268f9283428c6c38be3d68876fc94 tokenizers-0.21.1/tokenizers/examples/unstable_wasm/src/lib.rs
sha256:4202fe474a0ae76174696d04bc001ae1f838071e1b14ebe9a4913e9efdfade36 tokenizers-0.21.1/tokenizers/examples/unstable_wasm/src/utils.rs
tokenizers-0.21.1/tokenizers/examples/unstable_wasm/tests/
sha256:181970ff03a4a3115a9ead0e90015f0aefef46089c4fa6cd4d113c5d13e64e09 tokenizers-0.21.1/tokenizers/examples/unstable_wasm/tests/web.rs
tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/
tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/.bin/
sha256:2bf3871139ef28ae0d47e9b89ac0bb4b67ed5c794fca8d6092eaa37d9c9230e3 tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/.bin/create-wasm-app.js
sha256:c8015948b6eaefde5eaa3862f2fe5ce0245fcde4d6e66f7441dcdcbbed46e652 tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/.gitignore
sha256:06fc329654011dddca329034f2a0f57650b9e0fd50930509f411d43ad05b12ea tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/.travis.yml
sha256:8173d5c29b4f956d532781d2b86e4e30f83e6b7878dce18c919451d6ba707c90 tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/LICENSE-APACHE
sha256:ef36e00d5e8dd8c6260a46855a1a578ff546916c7624119a60832e5ce5c510e4 tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/LICENSE-MIT
sha256:06d60699af573a691b095bded90a83e5d2464e98c56e36d0d6bd1608c4d45a8f tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/README.md
sha256:5b38ab61e867664a13a5d8af50be6904726da873186752497d54bba0ff857c6a tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/bootstrap.js
sha256:13fb2506ed4b9c6152dca7cabe2ec8d8e6052f2a27642e22d81ae58f75acc98a tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/index.html
sha256:4e7d9c000d676dcaecf56808298b3d3c5efe50a3a386326b5fb96bf7f439b1ca tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/index.js
sha256:b3a6eb06a5d73eec232ef54a329bb448bfefea5adff6013de7eee94353f41ef9 tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/package-lock.json
sha256:a28345c99b3be9287a042467cd5b55be673d5bd8431b391506589ef2667eba9a tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/package.json
sha256:1be82d86fa4bf762b6744e612ea837ceda33640e71ae481e9ecbddebf9e35fe6 tokenizers-0.21.1/tokenizers/examples/unstable_wasm/www/webpack.config.js
sha256:2b92ea252be0fbc26f70317cdaa7b6411ea634b50d55338cd8c495e4dbf25d1d tokenizers-0.21.1/tokenizers/rust-toolchain
tokenizers-0.21.1/tokenizers/src/
tokenizers-0.21.1/tokenizers/src/decoders/
sha256:9c363604a82b2e1439afaa78efe6b11c2d33348525bb934c797a37f2007bf3bb tokenizers-0.21.1/tokenizers/src/decoders/bpe.rs
sha256:d75629c23e7ebfb03a9d9037e7200f61bf1ed152be99229256717ead91cea254 tokenizers-0.21.1/tokenizers/src/decoders/byte_fallback.rs
sha256:5bc7697e3661238e33d1614e95003e24b4814cf9e1dc7f458c1fc814d84c9d27 tokenizers-0.21.1/tokenizers/src/decoders/ctc.rs
sha256:19b7bfca165bdee6ab084ed5429a49afc9d52664c51187a500335a80707850fa tokenizers-0.21.1/tokenizers/src/decoders/fuse.rs
sha256:9963dedcd9a965ae80888e72a7d83fbe2ee1c4f426b313cf1b7014c384419a08 tokenizers-0.21.1/tokenizers/src/decoders/mod.rs
sha256:6bbaf543e2ac97e658bd657ccdb56b8930218c9381492e712d4c93131b663eca tokenizers-0.21.1/tokenizers/src/decoders/sequence.rs
sha256:8f552ed13ffd5ec94808f00a62d3e28e9d1b5a811412669f87ffbf4cf857b306 tokenizers-0.21.1/tokenizers/src/decoders/strip.rs
sha256:de59951937e6adc45b6f549379c63fcae9d961d624bf3c274cd0bc17f4e9b55f tokenizers-0.21.1/tokenizers/src/decoders/wordpiece.rs
sha256:8eca0d74064331636f1fd03010894985b9311d1ad8efe28f6efa6ebd49702f61 tokenizers-0.21.1/tokenizers/src/lib.rs
tokenizers-0.21.1/tokenizers/src/models/
tokenizers-0.21.1/tokenizers/src/models/bpe/
sha256:bd38f20a24f390ea319148bc494010319cb613948edf42e1acd817b5801e6d8f tokenizers-0.21.1/tokenizers/src/models/bpe/mod.rs
sha256:e6ebe5381306689655a535cc1880af7b23935db355714939288aa9d0c4c1fc51 tokenizers-0.21.1/tokenizers/src/models/bpe/model.rs
sha256:da2d119b8dc1352ceef158ef040403b43b60b412ff6fb9ebf6d647b04f8e7fd4 tokenizers-0.21.1/tokenizers/src/models/bpe/serialization.rs
sha256:e0dbba6681680706677b5d2ae3a8e0fef48456a7a4aaf010c78670b36dd327e6 tokenizers-0.21.1/tokenizers/src/models/bpe/trainer.rs
sha256:eec2df536184c0d084b902563d31f3de77953711af580ad72b2be03190e088b9 tokenizers-0.21.1/tokenizers/src/models/bpe/word.rs
sha256:5338d26194741ec6e0d9147016defac1d1363b68ef0ebc7362c26c3b866146ca tokenizers-0.21.1/tokenizers/src/models/mod.rs
tokenizers-0.21.1/tokenizers/src/models/unigram/
sha256:d0952d809c182b07df13114961dfd068eabd53e82a6e6a5a4fc5fb4a2cf72a88 tokenizers-0.21.1/tokenizers/src/models/unigram/lattice.rs
sha256:cf317d99ceebe2b4326237aa8f8bfdc2293a5dd0a7fc69a6e9e0028b804a000e tokenizers-0.21.1/tokenizers/src/models/unigram/mod.rs
sha256:2248dcc7e7b2c726055d04a362a17a821f530ce1887a100fd4f819c0e11b647e tokenizers-0.21.1/tokenizers/src/models/unigram/model.rs
sha256:2358d3985ae5330b2caebdbd5567621572368869dcbb36e3a95696325e941ce3 tokenizers-0.21.1/tokenizers/src/models/unigram/serialization.rs
sha256:7125823b970c874d2187982dab8a11ec74c85398e9e8c0d13438cbdf813b19b8 tokenizers-0.21.1/tokenizers/src/models/unigram/trainer.rs
sha256:304f7f387d7be43bd01e72b05a950f0a41d95ff51b219f0b123d46b84f70f30e tokenizers-0.21.1/tokenizers/src/models/unigram/trie.rs
tokenizers-0.21.1/tokenizers/src/models/wordlevel/
sha256:bb3382e4eb8087de2dbaf1e24f3b5b95afc8e2105fc9a456ada43cd93e575036 tokenizers-0.21.1/tokenizers/src/models/wordlevel/mod.rs
sha256:1af61e4f303e4e849162f0a50744d592e2a0ef3fcaee202b474ebaaeaeb6b453 tokenizers-0.21.1/tokenizers/src/models/wordlevel/serialization.rs
sha256:11fa23268c8f145a2e18e6a3a7da3519632ce72a9be7624ad950e8fa76fdb528 tokenizers-0.21.1/tokenizers/src/models/wordlevel/trainer.rs
tokenizers-0.21.1/tokenizers/src/models/wordpiece/
sha256:5f83214247ccaac4db9ad86d71cdeb28c37faf73d35f03971c804daf12ebae5d tokenizers-0.21.1/tokenizers/src/models/wordpiece/mod.rs
sha256:188ece9ae1641994bb5145a2ee600fc9b4d616ca3c0b9792b6c740b8348b0569 tokenizers-0.21.1/tokenizers/src/models/wordpiece/serialization.rs
sha256:4aa1ffd863a4be04dce17c8941ea68e3b4b4d257317593aacec3282aef698888 tokenizers-0.21.1/tokenizers/src/models/wordpiece/trainer.rs
tokenizers-0.21.1/tokenizers/src/normalizers/
sha256:a938fdac02447e33393d98edcaad015f58e2095d633521cee85a2b60ccd995bd tokenizers-0.21.1/tokenizers/src/normalizers/bert.rs
sha256:699e75ad10cd48e8416d93cb56cf0cd866b50043119574d5052934ecd808ed22 tokenizers-0.21.1/tokenizers/src/normalizers/byte_level.rs
sha256:145809163e0bd0161093f88e5cc316eea5407f2ecc1480530c5b77f96608ff6b tokenizers-0.21.1/tokenizers/src/normalizers/mod.rs
sha256:69acb2a2be78760c59a39a2c1f7db6469c75708900b53a7f9bbe189ed83aa6da tokenizers-0.21.1/tokenizers/src/normalizers/precompiled.rs
sha256:99d7336d9c9ba8baca0f30b435c2a7930332100ec0d58f6221eff94da373fa19 tokenizers-0.21.1/tokenizers/src/normalizers/prepend.rs
sha256:2f4566ee4f7b108559cc0946a6e4c91866d0bb98de67e61d139cfce8e1045cb0 tokenizers-0.21.1/tokenizers/src/normalizers/replace.rs
sha256:3d088593a2fa931261f4cd906440b83c053351a0e0e12cb797e541b98d7e2b06 tokenizers-0.21.1/tokenizers/src/normalizers/strip.rs
sha256:6c2312c8a2ba9461b850147fd63432f59f81e942042995d7c28e097f8198139e tokenizers-0.21.1/tokenizers/src/normalizers/unicode.rs
sha256:4b09adb7232537923d55352f2b0707f01c14929117d28fc34dd4b13a70049328 tokenizers-0.21.1/tokenizers/src/normalizers/utils.rs
tokenizers-0.21.1/tokenizers/src/pre_tokenizers/
sha256:32dd35c1f815072c6ba32df390389d10e5f4073b480d1c4ade06143ba23b571d tokenizers-0.21.1/tokenizers/src/pre_tokenizers/bert.rs
sha256:5c962ea0b56d70ed464b96b88c4bba7badc1872a3d3b3cf084140e21d5cf8f38 tokenizers-0.21.1/tokenizers/src/pre_tokenizers/byte_level.rs
sha256:7d2d0170eeb5d864742df867958f379ec564bc5af67e835f5793b06bfab4185f tokenizers-0.21.1/tokenizers/src/pre_tokenizers/delimiter.rs
sha256:1452f56c9fcb0265ee91c5439f860033209e44fe61565b743970d64216b9b41a tokenizers-0.21.1/tokenizers/src/pre_tokenizers/digits.rs
sha256:e3fe6724cf38f700c8ae549c06889ae82d889c25b2975f8b6cec636ec81a26e6 tokenizers-0.21.1/tokenizers/src/pre_tokenizers/metaspace.rs
sha256:5f0383e59902675f7c1d7ff446249665ce99414a9d28fad0f613c03143d6f468 tokenizers-0.21.1/tokenizers/src/pre_tokenizers/mod.rs
sha256:cfb82a699365f2bad1fe36be7ec5523276c90d4cfcf86e8e985e0ba0de6fd9e3 tokenizers-0.21.1/tokenizers/src/pre_tokenizers/punctuation.rs
sha256:1742957bb2cbe971fa791dac41b9e5fad5bc9843cb4eaa0e8e604f6cdc75e1df tokenizers-0.21.1/tokenizers/src/pre_tokenizers/sequence.rs
sha256:d12ee03b7a4d8006979906883ed3d993132cf2b9503f31ebfb0b0d376020d220 tokenizers-0.21.1/tokenizers/src/pre_tokenizers/split.rs
tokenizers-0.21.1/tokenizers/src/pre_tokenizers/unicode_scripts/
sha256:5740e30dba4e6e819309a24144c95d228d9aca437950fc0285b134f238f1788e tokenizers-0.21.1/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs
sha256:4ed0d408409ec35bcba141d75a2647d7503bc17d1e6f4641627325c99bd56edb tokenizers-0.21.1/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
sha256:c614fe1c9a79b1e4cbbb9f3cc3ad130dd6506e12ff74688d611a8eeff47c2167 tokenizers-0.21.1/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs
sha256:0d0d8284ab86d9b0f06b0b0f7024d50618f74de2b1c2ffc2877032a587a0753f tokenizers-0.21.1/tokenizers/src/pre_tokenizers/whitespace.rs
tokenizers-0.21.1/tokenizers/src/processors/
sha256:db17c4467ab2cb95db5dbe22d3abd6ca61d0c463537c49f40896a148bcf37ec1 tokenizers-0.21.1/tokenizers/src/processors/bert.rs
sha256:1dfcff4557f06c4306cc353846ff68775cceebd774e507108a5aafc81037eed5 tokenizers-0.21.1/tokenizers/src/processors/mod.rs
sha256:713d3ade0492cfcd9d1aecb269a5cc8aacf321a716b13fe2dfbfc7b023c0a67a tokenizers-0.21.1/tokenizers/src/processors/roberta.rs
sha256:96bfe2c23419a88b270bb3d6edb34d9e61e12b9e50cc8cd8b852a48e81a2c18e tokenizers-0.21.1/tokenizers/src/processors/sequence.rs
sha256:f4a8d2f7a992755ac3981cdaf4b8011a97e93d61a55d6a40d18eaad07be3f553 tokenizers-0.21.1/tokenizers/src/processors/template.rs
tokenizers-0.21.1/tokenizers/src/tokenizer/
sha256:925958a7989906624616218ccd5d63cd685c16cceebaa9da8974a7deea594b31 tokenizers-0.21.1/tokenizers/src/tokenizer/added_vocabulary.rs
sha256:1fcc839c6094a1a8a1e8fbabc5def5ab8823dc034cb886f91e2320b618054e8e tokenizers-0.21.1/tokenizers/src/tokenizer/encoding.rs
sha256:fce0d40f06bc0cffe7ed04fbc7f843f7c79d524977e9b293b467f1ffb978172a tokenizers-0.21.1/tokenizers/src/tokenizer/mod.rs
sha256:711726954209e143aa42392cf1f4b8b3abe8acdacd7fc75c3ce6ebcf93dcd45a tokenizers-0.21.1/tokenizers/src/tokenizer/normalizer.rs
sha256:036af136c7991371dd30571d13875c13c775e007dbd270cb5efdb128aba7d6a4 tokenizers-0.21.1/tokenizers/src/tokenizer/pattern.rs
sha256:2050974242070d86f2492277d4a37892a25afc9d3d60215e6fecfac6516440e7 tokenizers-0.21.1/tokenizers/src/tokenizer/pre_tokenizer.rs
sha256:b5ce8abfe70dbafb5e79961ba16de382b3e208eeb41779245d483f0cbcc2d066 tokenizers-0.21.1/tokenizers/src/tokenizer/serialization.rs
tokenizers-0.21.1/tokenizers/src/utils/
sha256:5f04353418d380ad62bcaf6ce02c806b4c7fc62aeef790ace510922afc61c17e tokenizers-0.21.1/tokenizers/src/utils/cache.rs
sha256:534ed9e80f94b5355cda3692a5ef9ad15296ec5e8f895dd854ee040d45669382 tokenizers-0.21.1/tokenizers/src/utils/fancy.rs
sha256:1ccdae3491dacd9ff8c5f74c0ecda27c52aa3286802c5e7131db07efe49a2264 tokenizers-0.21.1/tokenizers/src/utils/from_pretrained.rs
sha256:004540ac0fc434f6ec25c1cf8e6818998550da936a728aa5a301458c6c66a540 tokenizers-0.21.1/tokenizers/src/utils/iter.rs
sha256:ef57380e0ebd207fb7bc09f90a1aa23db851249d80fdf56d6738e8ac55cb6dbd tokenizers-0.21.1/tokenizers/src/utils/mod.rs
sha256:f6f58fb3fcf2dfbee2e8edc9b763247443fa102aea185481ff591f7d52c5256e tokenizers-0.21.1/tokenizers/src/utils/onig.rs
sha256:17e2b82c50ae7a3c296cda7898519c57911a8887a6ff5e9e57c27d215f5ed9e7 tokenizers-0.21.1/tokenizers/src/utils/padding.rs
sha256:f207764fa62d362885fd04ac5e14e9e32645a976e57e72e7663e64eaac15d531 tokenizers-0.21.1/tokenizers/src/utils/parallelism.rs
sha256:2bae922cafe5167d36cc6cd973edf4fcf6d0224be4d27fa47b9285dcca447ca8 tokenizers-0.21.1/tokenizers/src/utils/progress.rs
sha256:f5fa3a2fc93d5f473074518f1dab811cc6ff0d680e79a7bc0e0b5d204b8673b4 tokenizers-0.21.1/tokenizers/src/utils/truncation.rs
tokenizers-0.21.1/tokenizers/tests/
sha256:493c7097ed9a51ac4e6837ff729e71cc2ccd1bba1623e882147ca9447449bff6 tokenizers-0.21.1/tokenizers/tests/added_tokens.rs
tokenizers-0.21.1/tokenizers/tests/common/
sha256:75e506cb6cabd1a07989f086d8b573201a010318057fbb0ba287ea5e2b2d0686 tokenizers-0.21.1/tokenizers/tests/common/mod.rs
sha256:dd2cfa17574df96ef76b1ed8b1d89d35eb6f5aa28c0350465ef332e377829827 tokenizers-0.21.1/tokenizers/tests/documentation.rs
sha256:c44fc0afd1f9ec6c9ab30218b84866175fee2ae8e4315603fec0af9d1ae0f813 tokenizers-0.21.1/tokenizers/tests/from_pretrained.rs
sha256:f0f790bbe60a47112b4c250991c5769f10fa0ad3bf40fb200922b59caf841933 tokenizers-0.21.1/tokenizers/tests/offsets.rs
sha256:f6cedde9ff05d6b24917efdb7a52b75a31f2d400b52dacf4513894bc479b3857 tokenizers-0.21.1/tokenizers/tests/serialization.rs
sha256:e2ffa9c737605d111180e56de32f8e36ee6726a1e1c7419351c18faf87dc625d tokenizers-0.21.1/tokenizers/tests/stream.rs
sha256:f0433b135226479693729e4ebe15d91767545b17a5ecefe87717afc42e0dcb19 tokenizers-0.21.1/tokenizers/tests/training.rs
sha256:73192b32a0a6d53a7b22e50e74f48e9b85c1295cd956e0b6def51b5a1f71be4f tokenizers-0.21.1/tokenizers/tests/unigram.rs
Return to home page.