sha256:01f9625d465d86f83c96bbffcd9a9e22ddd30a7139ea888f34de69ddd01b0319
Build input of
- Gentoo: sci-libs/tokenizers 0.15.2-r1 (https://github.com/huggingface/tokenizers/archive/refs/tags/v0.15.2.tar.gz) blake2b:38e705cd2eb72ffb7f0ed8ee1be4db75d3b9662c24919f1d588c55a91d1a01fe057a19cac464460cde7971f595b0518e594947b6da2c632e90cc771519890d23
Contains Dependency Lockfiles
-
sha256:544e20c2d79a24710ce0fbca67384e33716fcea9f0f52f17868118a837451397tokenizers-0.15.2/bindings/node/yarn.lock -
sha256:622298acad6fc8337efa439e920eb694f2b5c99fa9a6572cbfd92aeeab0d42c1tokenizers-0.15.2/bindings/python/Cargo.lock -
sha256:7e6e6b8cde0cf7452e4297d16c5ec10c983378b50da2af2eba1452ccd8b0af23tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/package-lock.json
Diff
Content
tokenizers-0.15.2/
tokenizers-0.15.2/.github/
tokenizers-0.15.2/.github/conda/
sha256:a6591b816fed5e1831c613814bf70c7180e0f015057bad2a5ab6f8d50d6e0c22 tokenizers-0.15.2/.github/conda/bld.bat
sha256:27c731a659d61a4fce9877fea566686a01c3b5bc804f73b9949c2dd3d534b2f6 tokenizers-0.15.2/.github/conda/build.sh
sha256:45f202ad857770620e4e919ebebee205711b49975812ad28f11decc984fdfe7e tokenizers-0.15.2/.github/conda/meta.yaml
sha256:9f92c3d0c0082e9409d62ddf4a7c4612acf8692dcf1b4bed33fdffbe43f153fe tokenizers-0.15.2/.github/stale.yml
tokenizers-0.15.2/.github/workflows/
sha256:c3324b307eb3eb71f160357c07ef49f5a86b9a1c6f16672a0946c0195d29c96b tokenizers-0.15.2/.github/workflows/build_documentation.yml
sha256:4e6c12654c7e95e4e51652e096fad2e6bccd3ec5193543ea9b4c8eada7fe80e0 tokenizers-0.15.2/.github/workflows/build_pr_documentation.yml
sha256:57ae5e4c90a9d327f25a9ad148fe7ace589865f1f8d9565d78460facda3f748c tokenizers-0.15.2/.github/workflows/delete_doc_comment.yml
sha256:db118b73b00dc7ccd3aaea04e90b31b6f2bca549160604697182b297d653bb50 tokenizers-0.15.2/.github/workflows/delete_doc_comment_trigger.yml
sha256:1e161fa7106abfe13c9d8b483eb12475d18cfb56e453193299bd64c1de00a2f0 tokenizers-0.15.2/.github/workflows/docs-check.yml
sha256:c4a3c9464e7f2d687242371f8579d2cea8ee05a902842c496960b315c74cc766 tokenizers-0.15.2/.github/workflows/node-release.yml
sha256:14dd18555a667af607860b8402d3b18943e031a67481890b9efe31b2d3943cfe tokenizers-0.15.2/.github/workflows/node.yml
sha256:29c409d1b01b05b78284f22eb56092a65e4c2c88ba80ffbde5893526d8ac6e39 tokenizers-0.15.2/.github/workflows/python-release-conda.yml
sha256:c16fc56ad0d480dd619f670de61c9129096c3166b80c61d13bcb42700718e7ab tokenizers-0.15.2/.github/workflows/python-release.yml
sha256:1503d12a3c56bd890b758d27570cd3f7e9d28d8eb0a8b8d7f10d2f93e10ebac8 tokenizers-0.15.2/.github/workflows/python.yml
sha256:35b5b976278dc768b9d362d7635e6beb4b41b614e516d395d6bafb5663f8afe8 tokenizers-0.15.2/.github/workflows/rust-release.yml
sha256:3eefdc28bd1c713b9e88db4b47769666f60b10ce950d5ce2c4ed26bdbcbf99ff tokenizers-0.15.2/.github/workflows/rust.yml
sha256:8cbc2ee27b9fa769a9889e460e8b9337ac8d0dd67d3e18706a4f098cfccffdef tokenizers-0.15.2/.github/workflows/stale.yml
sha256:53ff3a72fd52c75b89371910748b4bc05e3718ab640b9751bb7df7cd8d97231f tokenizers-0.15.2/.github/workflows/upload_pr_documentation.yml
sha256:91da3b625c669f733040eaba80a5a8067890e64714ef6089d36a261b532cf7d0 tokenizers-0.15.2/.gitignore
sha256:4d377e818cff7934516a38936b80230a34153c0bbb42dff20f64e84bfad19826 tokenizers-0.15.2/CITATION.cff
sha256:c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4 tokenizers-0.15.2/LICENSE
sha256:551d3945bd0f36eaa22a7131b5020c115179eef155362b41b53e10ae3e37b34b tokenizers-0.15.2/README.md
sha256:815650e21cafbce968e60542326a6e4fa5446af70a85b0494f6e2ef4e8a1fc1a tokenizers-0.15.2/RELEASE.md
tokenizers-0.15.2/bindings/
tokenizers-0.15.2/bindings/node/
tokenizers-0.15.2/bindings/node/.cargo/
sha256:95eccdbcb0b22f08a34041c422ff5821bd783f9392d8bd1136305239fc1f7287 tokenizers-0.15.2/bindings/node/.cargo/config.toml
sha256:30adc741ffc18728591498ec0d123497e6262c963bf75a91177a0056c50ae9e1 tokenizers-0.15.2/bindings/node/.editorconfig
sha256:f90a518eaf74b6a9c515d44ac0159cf87e3ebc1913792ec0dc38334b6d626eec tokenizers-0.15.2/bindings/node/.eslintrc.yml
sha256:d005ffc2cefc69de8caf1394a6a074754a8bb808fb5ec1d92fc223db3d96bbed tokenizers-0.15.2/bindings/node/.gitattributes
sha256:27071e79228833618f823a63dcaf2d0bca21a2db81ece8edf7422cd9bb992274 tokenizers-0.15.2/bindings/node/.gitignore
sha256:c7206e1a30314ad09058859077f48794851e3079811b7c970a15275b39645a38 tokenizers-0.15.2/bindings/node/.prettierignore
sha256:d32b93a2df8144b2d8e82ac0196e5fa1f162591d00434ff884f1ded5cc50d331 tokenizers-0.15.2/bindings/node/.taplo.toml
tokenizers-0.15.2/bindings/node/.yarn/
tokenizers-0.15.2/bindings/node/.yarn/releases/
sha256:64c0b63e5fa0e21f354b5eff7c9a1b486f32046bfc3283670569e3c6a9cad102 tokenizers-0.15.2/bindings/node/.yarn/releases/yarn-3.5.1.cjs
sha256:bda4132acb74d9f289cf423e33ab81782f03377641413b7626444fa759f52b1d tokenizers-0.15.2/bindings/node/.yarnrc.yml
sha256:5a1a735257aa3261e9334ce8205feee0bf8369f859b8fb43861321e955ea1184 tokenizers-0.15.2/bindings/node/Cargo.toml
sha256:02965ed67e29f871a53f7a7afa593a0ddc8020f86ccce0faaf1aa59215b5269e tokenizers-0.15.2/bindings/node/LICENSE
sha256:e9398e4e77c114500ecb33b03abc0ff98ecd973ee770ad481549232beef59a06 tokenizers-0.15.2/bindings/node/Makefile
sha256:61377fafc00be92c730d6ca920c145de5b1910ddd9a98419309376699da742b2 tokenizers-0.15.2/bindings/node/README.md
sha256:71b61de74caafb5f27db3d24d197b64282f49f807b969e140ca8ce181e5464fc tokenizers-0.15.2/bindings/node/build.rs
tokenizers-0.15.2/bindings/node/examples/
tokenizers-0.15.2/bindings/node/examples/documentation/
sha256:32099423a4192e4ae956656314cfd9c5fefc4b0f0c0783dcd5f5a5206a7a398c tokenizers-0.15.2/bindings/node/examples/documentation/pipeline.test.ts
sha256:f1e06f10edd4dc2144769b5f28ad0f24cb5f08fd26e1e20949ce0611ea012cd9 tokenizers-0.15.2/bindings/node/examples/documentation/quicktour.test.ts
sha256:434df54ba055a4e28b530e1c85a5a773ba7e2c1ee14f2df9c748536b703b2f27 tokenizers-0.15.2/bindings/node/index.d.ts
sha256:40a046e6014b8fe28985d085dc2a653520e4e5f749d9b46f477bd4cd866a2855 tokenizers-0.15.2/bindings/node/index.js
sha256:fdbdbd048885ea46a8ce806675fb47bb9a04975d55132d7080a458a78a6a2b94 tokenizers-0.15.2/bindings/node/jest.config.js
tokenizers-0.15.2/bindings/node/lib/
tokenizers-0.15.2/bindings/node/lib/bindings/
tokenizers-0.15.2/bindings/node/lib/bindings/__mocks__/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.15.2/bindings/node/lib/bindings/__mocks__/merges.txt
sha256:ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356 tokenizers-0.15.2/bindings/node/lib/bindings/__mocks__/vocab.json
sha256:e2c1d7f033886700bc4d6718e92a3b00547268a5c083ce91eeca79f0771cf26b tokenizers-0.15.2/bindings/node/lib/bindings/__mocks__/vocab.txt
sha256:bae90354fdc246dc1819d8e813f01f7a0b37b8cbc278de5f48c320805819ec25 tokenizers-0.15.2/bindings/node/lib/bindings/decoders.test.ts
sha256:930510ac93ffd04dc780138b29c30f63709a71071aa8b194ce518ad8ed108c1b tokenizers-0.15.2/bindings/node/lib/bindings/encoding.test.ts
sha256:ea23c29304840ecccd24f78a117db88a1380b35d6d3a811626d83e597bdd471c tokenizers-0.15.2/bindings/node/lib/bindings/models.test.ts
sha256:3201d9ca7d7c42d9934f9ae3e3331f9c0488f0adc713f6ff6ec64251c4ab6178 tokenizers-0.15.2/bindings/node/lib/bindings/normalizers.test.ts
sha256:29db4dbd89bd55c88c4a43b6ae24b2610bc05858fc6a55f33cf1c973b72c2198 tokenizers-0.15.2/bindings/node/lib/bindings/post-processors.test.ts
sha256:529ecc9b4adcc0d28ef1b52445898086ed0e5e70bea9d49862820e691b2b6215 tokenizers-0.15.2/bindings/node/lib/bindings/pre-tokenizers.test.ts
sha256:3fd18a74349b98d8bc9392e7a10b3e9ea0335c2b58ee8afe1f97b58a85df1a1a tokenizers-0.15.2/bindings/node/lib/bindings/tokenizer.test.ts
sha256:68c1660db395387412cd5ec38cfd6e01c69f0da362e477acbdbcee561e624c46 tokenizers-0.15.2/bindings/node/lib/bindings/utils.test.ts
tokenizers-0.15.2/bindings/node/npm/
tokenizers-0.15.2/bindings/node/npm/android-arm-eabi/
sha256:05a1d365a6029297f0e0b1209776c25b42cdf7a51a52e8c76b85b542c8c7ab20 tokenizers-0.15.2/bindings/node/npm/android-arm-eabi/README.md
sha256:90a397203ba933dc527dff748946c38d7418ca9cfd62035a20d4e36458e06dec tokenizers-0.15.2/bindings/node/npm/android-arm-eabi/package.json
tokenizers-0.15.2/bindings/node/npm/android-arm64/
sha256:7c0cd0416e0548f05948cf6f4c4d950d966d2107aa3c62e0c4fa47e2ff077240 tokenizers-0.15.2/bindings/node/npm/android-arm64/README.md
sha256:43fed355ff5375b82e34895553f69e3d7a55a6a3173d28038909c5db0d662a73 tokenizers-0.15.2/bindings/node/npm/android-arm64/package.json
tokenizers-0.15.2/bindings/node/npm/darwin-arm64/
sha256:ee3950cafecfb51853d3762ae9369813f4cd397ff0477900ae5623fc0e33540c tokenizers-0.15.2/bindings/node/npm/darwin-arm64/README.md
sha256:a21b46740ea8236cc37ca25a65e5354609bdcf2cdeabfa24461f61f700f470db tokenizers-0.15.2/bindings/node/npm/darwin-arm64/package.json
tokenizers-0.15.2/bindings/node/npm/darwin-x64/
sha256:23b156cb161d34e087477df0a946cfedf47dd665a8cb77bb637a2ea8dfdf80ff tokenizers-0.15.2/bindings/node/npm/darwin-x64/README.md
sha256:aad332092b741c591c39e3898e8f24ccfebb41fc7c1995a2114e2ab84700488d tokenizers-0.15.2/bindings/node/npm/darwin-x64/package.json
tokenizers-0.15.2/bindings/node/npm/freebsd-x64/
sha256:c21c445c4f03fb72275dea0459b584b3d1b107321af13a80b018aacfe89f57e6 tokenizers-0.15.2/bindings/node/npm/freebsd-x64/README.md
sha256:e838711b46934e6b1bfa6477ae6b0d549f07c3f70dcb0b59fcd735ec4e38e22f tokenizers-0.15.2/bindings/node/npm/freebsd-x64/package.json
tokenizers-0.15.2/bindings/node/npm/linux-arm-gnueabihf/
sha256:c485213dbc76af8e6f54bd83be63777f719060e67c87921d426a229ac9363424 tokenizers-0.15.2/bindings/node/npm/linux-arm-gnueabihf/README.md
sha256:0c826d635835adcbc4bb7dde99cac6e9fac6954247c60958b77a63a3a4fb6b6f tokenizers-0.15.2/bindings/node/npm/linux-arm-gnueabihf/package.json
tokenizers-0.15.2/bindings/node/npm/linux-arm64-gnu/
sha256:fca73a7053f15219b1cd717dfe93dad41f1ce8ed0cbe77f06e0bae890be2bcf2 tokenizers-0.15.2/bindings/node/npm/linux-arm64-gnu/README.md
sha256:5de0ee27d1daa0c4d28eba57d66e891a51dc27d65c0cb1843ed47de10ca8399e tokenizers-0.15.2/bindings/node/npm/linux-arm64-gnu/package.json
tokenizers-0.15.2/bindings/node/npm/linux-arm64-musl/
sha256:a42323205e03c06ab50640eb6616e8e29de7f0e1a24bc87a85b5adc39bbc743f tokenizers-0.15.2/bindings/node/npm/linux-arm64-musl/README.md
sha256:ad55cfef52a07822aad94d364c895ee7a9ddaf51be38681ac2b62f88a5beef52 tokenizers-0.15.2/bindings/node/npm/linux-arm64-musl/package.json
tokenizers-0.15.2/bindings/node/npm/linux-x64-gnu/
sha256:b8f3cdc50472110fee2373517b0ef96ce803641d4c74b95f82ffc3f85b449841 tokenizers-0.15.2/bindings/node/npm/linux-x64-gnu/README.md
sha256:9f936b7429d60f3dab9319422da2216b31ba0330311284668e2161c6cac0f22d tokenizers-0.15.2/bindings/node/npm/linux-x64-gnu/package.json
tokenizers-0.15.2/bindings/node/npm/linux-x64-musl/
sha256:9b4cc26fdf788f433f1d7dcbd9aed1979b655418cc8b9810120eb01651f7f18d tokenizers-0.15.2/bindings/node/npm/linux-x64-musl/README.md
sha256:4260f5c07c5cf19f2e9f44500f17162c9c77955582b0887d6a26b143ea5acba2 tokenizers-0.15.2/bindings/node/npm/linux-x64-musl/package.json
tokenizers-0.15.2/bindings/node/npm/win32-arm64-msvc/
sha256:cf057a5481c42a26b03fffa4d33b6d196a10a81223992f1a17271b27290d06a4 tokenizers-0.15.2/bindings/node/npm/win32-arm64-msvc/README.md
sha256:6c35853dc168d4204bdd98c2b1709be3f0fc6b1d898b2c5fdad106fa2c28841b tokenizers-0.15.2/bindings/node/npm/win32-arm64-msvc/package.json
tokenizers-0.15.2/bindings/node/npm/win32-ia32-msvc/
sha256:df2c8f0c9d538872f1c10995666f6a9a6b13b6af4eb202efb76b90c24bc980b5 tokenizers-0.15.2/bindings/node/npm/win32-ia32-msvc/README.md
sha256:366da5b97cdde4fe1f9ba87346cafa178042fc528a9d41726dcea4863f7e55f4 tokenizers-0.15.2/bindings/node/npm/win32-ia32-msvc/package.json
tokenizers-0.15.2/bindings/node/npm/win32-x64-msvc/
sha256:08b6de0abc7178dd64cc5054624a04545d0ac3e1019822452809867fa56b45df tokenizers-0.15.2/bindings/node/npm/win32-x64-msvc/README.md
sha256:6ba80cc97010f6c17faa664569295b97add7252b504ad46a99818e31fd89161c tokenizers-0.15.2/bindings/node/npm/win32-x64-msvc/package.json
sha256:055c5d37894234778633b626446105f77e0c7bf6fcec009705b07014644caf5c tokenizers-0.15.2/bindings/node/package.json
sha256:d53b7e5653f4a60e4f6558b9d7704045ec0ff0687a768ff3163a67acab4c9f6d tokenizers-0.15.2/bindings/node/rustfmt.toml
tokenizers-0.15.2/bindings/node/src/
sha256:0c288ea5db6f2fcf4f9fb37e46472b5d27ace113310440b9681ad745abe8462e tokenizers-0.15.2/bindings/node/src/arc_rwlock_serde.rs
sha256:b266a4b83d38ad6db26845c3bcc933ca8ebe4e75a5a6180955d013b614e6b1a8 tokenizers-0.15.2/bindings/node/src/decoders.rs
sha256:03cbe893bb55b4609205d94627e77e60b54dd7ef751a9291efac35265bbc2018 tokenizers-0.15.2/bindings/node/src/encoding.rs
sha256:23e01dcea44b1fc841dc18ad525f6fd5c5e4a94843c1368e802de32bb06e90ad tokenizers-0.15.2/bindings/node/src/lib.rs
sha256:2dc0b75569ede4063f503211cde250abab990460afb495abc79393392f655def tokenizers-0.15.2/bindings/node/src/models.rs
sha256:f7dc43ad7c794ffe23fc29ef4dfa6e3df50fe5f1145c23d7c3abac332a78ba57 tokenizers-0.15.2/bindings/node/src/normalizers.rs
sha256:dcf3cb410837f44ddbe98af55e8b0a6e351661060daebd7d0ff8bd730603b0ff tokenizers-0.15.2/bindings/node/src/pre_tokenizers.rs
sha256:27c5d78c318c9f15ca440715fc5297b79655fb0b787fc1e1eb9cc5a5dca7ccb3 tokenizers-0.15.2/bindings/node/src/processors.rs
tokenizers-0.15.2/bindings/node/src/tasks/
sha256:97302dbce5e72f37cf4a74de57e17084a6ef98f41a9cc832348842d8ae9e4de1 tokenizers-0.15.2/bindings/node/src/tasks/mod.rs
sha256:7cb04541da8f4b29346caacf41dcfe873b6f9ffab4f4756f14710d3f06025f05 tokenizers-0.15.2/bindings/node/src/tasks/models.rs
sha256:06de8bd57a41b18dce7b169f3798d6c3dadcf361a9006cfcfb792cdbdf9ee4c1 tokenizers-0.15.2/bindings/node/src/tasks/tokenizer.rs
sha256:ca3815779b929c0ae48d24afa477e4b71f9eb84736fca2513a9f389d6217aa0c tokenizers-0.15.2/bindings/node/src/tokenizer.rs
sha256:67a3f03a137bf83c2351f736c68ab5bce13016d7f76f838a1b3461c0c0ef061f tokenizers-0.15.2/bindings/node/src/trainers.rs
sha256:7c7ab6fcdd34f3fc94a6652c1af166e03b886a1b7a7b40366011efd04efdec5f tokenizers-0.15.2/bindings/node/src/utils.rs
sha256:3be08c632028bf9f54b8aa711469f6207127e2bcae5cf88b4b9da18d788a3db4 tokenizers-0.15.2/bindings/node/tsconfig.json
sha256:110ee41ab1de180ecc708c8094a815b5023e62a309d9417ebd2d6cf014298896 tokenizers-0.15.2/bindings/node/types.ts
sha256:544e20c2d79a24710ce0fbca67384e33716fcea9f0f52f17868118a837451397 tokenizers-0.15.2/bindings/node/yarn.lock
tokenizers-0.15.2/bindings/python/
tokenizers-0.15.2/bindings/python/.cargo/
sha256:0f8f7159b2550efdd76855ec4620c6fd21d85648caa9eb71f41aad19404ee92b tokenizers-0.15.2/bindings/python/.cargo/config.toml
sha256:6667b2d1aab6a00caa5aee5af8ad9f1465e567abf1c209d15727d57b3e8f6e5f tokenizers-0.15.2/bindings/python/.gitignore
sha256:c0778b9c5f36c46141cb570ba45615150476d130fa00a128e1f0331417b361fc tokenizers-0.15.2/bindings/python/CHANGELOG.md
sha256:622298acad6fc8337efa439e920eb694f2b5c99fa9a6572cbfd92aeeab0d42c1 tokenizers-0.15.2/bindings/python/Cargo.lock
sha256:ac5de2eddddcb876c42a0240a34c5b2b309c4a81aa83f9bea5f68713bb3127ab tokenizers-0.15.2/bindings/python/Cargo.toml
sha256:b5bba29c1d37582948f7837cd44ca176452f386a8c4d41f2fccd95ab17859abf tokenizers-0.15.2/bindings/python/MANIFEST.in
sha256:a193eaba11184b09cb4c359ec937230c16be6641003a8c26fd3e61d666e3630a tokenizers-0.15.2/bindings/python/Makefile
sha256:b228767941e834617aa3a7c02f278edf433ebc34adb9236c9d7f160eab883971 tokenizers-0.15.2/bindings/python/README.md
sha256:d5fec928f9fee467cbac5b91750c56ce65a646784a8395225dd6b0016b910845 tokenizers-0.15.2/bindings/python/conftest.py
tokenizers-0.15.2/bindings/python/examples/
sha256:0b23ff83ffb7e7d429e2a1295140f91bb7ee00f40ac46ad1a5819cced6e4d54e tokenizers-0.15.2/bindings/python/examples/custom_components.py
sha256:844fec7f969b708caaab51fb43e637a20591d55642057441f67881e31b25d21d tokenizers-0.15.2/bindings/python/examples/example.py
sha256:58ebba107e08b4c45703c9ba9aad14d143142c8592949e4903ae8e06c830421e tokenizers-0.15.2/bindings/python/examples/train_bert_wordpiece.py
sha256:ec2d5897b2646b8017acbbfa404c212e0142dae05d7df0cf4a3c43e4d0b68d26 tokenizers-0.15.2/bindings/python/examples/train_bytelevel_bpe.py
sha256:8df17880da3aab6b4b7da525bea41bfb99e7fcf03c2bb78265cb565c98ca6312 tokenizers-0.15.2/bindings/python/examples/train_with_datasets.py
sha256:04528de21bd78ca77df86a5c802cce91c56155b90c9429f4160bfc180dc6857f tokenizers-0.15.2/bindings/python/examples/using_the_visualizer.ipynb
tokenizers-0.15.2/bindings/python/py_src/
tokenizers-0.15.2/bindings/python/py_src/tokenizers/
sha256:644e596a052fa1b05272b1c141d1286e1c78c2a3346ecabd17d68b62404d8d84 tokenizers-0.15.2/bindings/python/py_src/tokenizers/__init__.py
sha256:8307e1f05191c00007b37b1e745cd1a5f5431670e82da0ad9b701ae271bfb0fb tokenizers-0.15.2/bindings/python/py_src/tokenizers/__init__.pyi
tokenizers-0.15.2/bindings/python/py_src/tokenizers/decoders/
sha256:946a77da1f2a7ab134178f20c992fcc0699e4159668d5a5e22c45bd5233d91fe tokenizers-0.15.2/bindings/python/py_src/tokenizers/decoders/__init__.py
sha256:a85e5d7002bb1e2a9d8ff797093686ffe01d0bf9af2f2f84f7afc7ce8bc8ce02 tokenizers-0.15.2/bindings/python/py_src/tokenizers/decoders/__init__.pyi
tokenizers-0.15.2/bindings/python/py_src/tokenizers/implementations/
sha256:57302ca65688a3bae5e0014ef0c8aeee283b31f663be89f055b959c34d7347a3 tokenizers-0.15.2/bindings/python/py_src/tokenizers/implementations/__init__.py
sha256:d9315984bba968988c0d8189b94366c5826ff9c9d1f1b0c799b333698a4544eb tokenizers-0.15.2/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
sha256:b0a0ae9b414a3d875280914937c2c37ab541a130d14aacaa49dadc9be96f42a2 tokenizers-0.15.2/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
sha256:380fe3cb2dc44266136ba8677fe10ac0b385bb2ae8a8560e273db9cac3360549 tokenizers-0.15.2/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
sha256:436644016d313101c5ed8094b66a65c1ac5b53e2743f634ae0f24633151bfbf7 tokenizers-0.15.2/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
sha256:b32357a10657d49b48f14d40f574837288a1945ddb20655354de77609f37a55e tokenizers-0.15.2/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
sha256:8827fd34a3c0c5359f3469b0c74028a00d6e6cde386c081376f8d4ecb867be13 tokenizers-0.15.2/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
tokenizers-0.15.2/bindings/python/py_src/tokenizers/models/
sha256:7896781d3010669c672882cdca559a4c5ab15f2f816ba38ab3058de3b7de795f tokenizers-0.15.2/bindings/python/py_src/tokenizers/models/__init__.py
sha256:d9179f2d326528e175a34648eb0a411c318844cb17be7db9fb6eb258261abcf6 tokenizers-0.15.2/bindings/python/py_src/tokenizers/models/__init__.pyi
tokenizers-0.15.2/bindings/python/py_src/tokenizers/normalizers/
sha256:84a3b09ea58cf8895c56fec70d64fd4988657337afb82343409634e591719339 tokenizers-0.15.2/bindings/python/py_src/tokenizers/normalizers/__init__.py
sha256:a51ecdc1c643c859dc9b265af569f5823ab3a296aae47ca23e444d1d8ba1cd99 tokenizers-0.15.2/bindings/python/py_src/tokenizers/normalizers/__init__.pyi
tokenizers-0.15.2/bindings/python/py_src/tokenizers/pre_tokenizers/
sha256:c1de8a61003f46c19240af87786f68a53461bf8b6d4919329e8d9d93a6b3f8f3 tokenizers-0.15.2/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py
sha256:288dfa1a01e5477a7d1ad2ed85ea8d7459adea39bd3d5ccd2aae1ac1ccb574a5 tokenizers-0.15.2/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
tokenizers-0.15.2/bindings/python/py_src/tokenizers/processors/
sha256:c4cd8310ac0ab4722e987b2ccccf00324abe0256aabc16455d680b53c48d84e6 tokenizers-0.15.2/bindings/python/py_src/tokenizers/processors/__init__.py
sha256:0195c7f590d66f48efce9d03282ee52a309e3b0962ef4fd9bd1df3a8c29feefe tokenizers-0.15.2/bindings/python/py_src/tokenizers/processors/__init__.pyi
tokenizers-0.15.2/bindings/python/py_src/tokenizers/tools/
sha256:c46f1c681f4e1c2f1c6c1d354b9bd8575e0767184ee9e59b2de86c6fbd29a62a tokenizers-0.15.2/bindings/python/py_src/tokenizers/tools/__init__.py
sha256:cc0c9dab5a06583f10125978f9ec8bf0b970d01d6cb72fe1a48137b58c4bd369 tokenizers-0.15.2/bindings/python/py_src/tokenizers/tools/visualizer-styles.css
sha256:d0a52b2e19012e13ef83718092fb2206891be75edb30231237ebee618d6a984a tokenizers-0.15.2/bindings/python/py_src/tokenizers/tools/visualizer.py
tokenizers-0.15.2/bindings/python/py_src/tokenizers/trainers/
sha256:513bb6d8019ca7be88be95b8e712d16c95844f4e0dc4f5ba35f09bd98633d043 tokenizers-0.15.2/bindings/python/py_src/tokenizers/trainers/__init__.py
sha256:ab982c3179e9d9404ee7e8070656a36fd9613a8f0539e2810b0546192036dd6b tokenizers-0.15.2/bindings/python/py_src/tokenizers/trainers/__init__.pyi
sha256:61d042f8c0231635e42ec064420919b6ee5f5d45baa2cb0c97c4164af70f3231 tokenizers-0.15.2/bindings/python/pyproject.toml
sha256:2b92ea252be0fbc26f70317cdaa7b6411ea634b50d55338cd8c495e4dbf25d1d tokenizers-0.15.2/bindings/python/rust-toolchain
tokenizers-0.15.2/bindings/python/scripts/
sha256:afe16e3256335febe837cde06ed5e9035d3f1705cd49023ac74b34aed4122eac tokenizers-0.15.2/bindings/python/scripts/convert.py
sha256:cab2399ce5c49842a8522c1ec58891cbc45af9b5aa6db59a4f5cb699dd2c90bc tokenizers-0.15.2/bindings/python/scripts/sentencepiece_extractor.py
sha256:c7a60f464b573084e4b49c55b04d7c7de54fa70504dfa8eb04bc1acdd3d69917 tokenizers-0.15.2/bindings/python/scripts/spm_parity_check.py
sha256:32e7d02fa00cb911b747f419942188de07ebd841aa89bde7ce9c256173ffd696 tokenizers-0.15.2/bindings/python/setup.cfg
tokenizers-0.15.2/bindings/python/src/
sha256:160b5c27ee9e940593c4fcb9b3f8767a519faa7f525c1033e38eb4d97a2d42eb tokenizers-0.15.2/bindings/python/src/decoders.rs
sha256:d14b84d99b788fe04dff2ca9921055fd5ed4f13853e55fac1325c2789e1fee59 tokenizers-0.15.2/bindings/python/src/encoding.rs
sha256:9831272131902513b5cae5b76fd7fde484fc03de4488d7123d9a7dd00a109495 tokenizers-0.15.2/bindings/python/src/error.rs
sha256:fb56210ebe39b1383c5383cbf010d449a990c760e7ef3667f1d8fbf721041b0c tokenizers-0.15.2/bindings/python/src/lib.rs
sha256:6e01af584c0900c763e037d1173f5f6355015330244440513f4f9e1236824112 tokenizers-0.15.2/bindings/python/src/models.rs
sha256:20f3099b654d3ccde595e1b37b46f80acb4948ad527988d6e367c023738478e4 tokenizers-0.15.2/bindings/python/src/normalizers.rs
sha256:ea2c8a615c52564ca7a864af21da9c8cc6b1ef71b8fe4410c1c005e4f0083777 tokenizers-0.15.2/bindings/python/src/pre_tokenizers.rs
sha256:78706eb9f5179b50defe410a9431e9095a91f6f4bd0b0c49cc1359dc0c568fde tokenizers-0.15.2/bindings/python/src/processors.rs
sha256:9e49b34d599baa0135cb2f9ba0e380fff813eeb5bf96b29976dac5f7fb68f1a6 tokenizers-0.15.2/bindings/python/src/token.rs
sha256:30cbae88275134f33aaae95933d86562cf9d100238086fcdd30b750b2bb25a07 tokenizers-0.15.2/bindings/python/src/tokenizer.rs
sha256:afc018fecf162e8ac6f76040a8e0453dd256eed16f34119b0e2331984f412831 tokenizers-0.15.2/bindings/python/src/trainers.rs
tokenizers-0.15.2/bindings/python/src/utils/
sha256:f5e9cc874c2c460f23c4eb216f84d6f8d2173d2760821ae5a0c3e0f926a755d9 tokenizers-0.15.2/bindings/python/src/utils/iterators.rs
sha256:d9232e08b7fcff98e484618b68edfe408543ddddf7b0b2299a3ad2fa688f384b tokenizers-0.15.2/bindings/python/src/utils/mod.rs
sha256:07dde7eac2b2c99f6b98f59d0aa6168da23d3ad401b62b2bf3050359319669a3 tokenizers-0.15.2/bindings/python/src/utils/normalization.rs
sha256:958e18940c00611c0cc467cb3507067a1c47fd46631f2407aca8b52ea660dd8c tokenizers-0.15.2/bindings/python/src/utils/pretokenization.rs
sha256:2227963a53509e864f4a72a6ef99c759da7ae2a96002cf912bb447956a2e0815 tokenizers-0.15.2/bindings/python/src/utils/regex.rs
sha256:c270709bc8e4729d59a093136a128080ee7dff2399ad3630005a258a32bd6132 tokenizers-0.15.2/bindings/python/stub.py
sha256:70e03c19780c5e7c1f31580930d36f0aa4a64e5fc8e601dd50b74b301729b3ee tokenizers-0.15.2/bindings/python/test.txt
tokenizers-0.15.2/bindings/python/tests/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.15.2/bindings/python/tests/__init__.py
tokenizers-0.15.2/bindings/python/tests/bindings/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.15.2/bindings/python/tests/bindings/__init__.py
sha256:838a641e4df943af4877322bd863ee787082c28aac39ddbf8c4561ccd531613f tokenizers-0.15.2/bindings/python/tests/bindings/test_decoders.py
sha256:5f4b218bbfdaa27751080e4451b8fd4d31fda7c41d661da90c604dbd00339b47 tokenizers-0.15.2/bindings/python/tests/bindings/test_encoding.py
sha256:9ef0476b6da3f183b681b146fd0605061729064961f354bab85f4062c3c4ba3e tokenizers-0.15.2/bindings/python/tests/bindings/test_models.py
sha256:31be6fc5fc5c24c9951c559b2c151cb5e18e5f56b008bf7ae79756b5b6f2300d tokenizers-0.15.2/bindings/python/tests/bindings/test_normalizers.py
sha256:5dff1771a6f9d7385d3d3e74c850a69338583011bdb839d2fed509fc69d4edfa tokenizers-0.15.2/bindings/python/tests/bindings/test_pre_tokenizers.py
sha256:d208993177837ccf0bb173c56cde5148492956540217061b42f585934c1308b4 tokenizers-0.15.2/bindings/python/tests/bindings/test_processors.py
sha256:9a071b37bb21bc224fba42749dbab2dd9d15f8e5eec14d2933f7f5d60351793c tokenizers-0.15.2/bindings/python/tests/bindings/test_tokenizer.py
sha256:ff322fca83fe9070d98dc035389f3ce51332a565309c4ade497a52beacda4729 tokenizers-0.15.2/bindings/python/tests/bindings/test_trainers.py
tokenizers-0.15.2/bindings/python/tests/documentation/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.15.2/bindings/python/tests/documentation/__init__.py
sha256:9f4d95f4df2e532924dfa7363fd05ea3c4d60b20782d54f3a0ea8f182289106f tokenizers-0.15.2/bindings/python/tests/documentation/test_pipeline.py
sha256:eeebefd68bcd7179a1e0fd3fd7318ea82659b94cb7ef62477ad00c1d578be1a1 tokenizers-0.15.2/bindings/python/tests/documentation/test_quicktour.py
sha256:70e8f0fdd0e2e04168cc57ea60d00e16ec459d85fa8bdfcf0faa3cd8781ee48c tokenizers-0.15.2/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py
tokenizers-0.15.2/bindings/python/tests/implementations/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.15.2/bindings/python/tests/implementations/__init__.py
sha256:f527dfefdb56f01c5cc541f2f79d28d11d02d4bfa21dff8d16a56b71bce33fbb tokenizers-0.15.2/bindings/python/tests/implementations/test_base_tokenizer.py
sha256:c0649576679a5b9d27203679a3dc978b217355cb474117590097cc9f4019858c tokenizers-0.15.2/bindings/python/tests/implementations/test_bert_wordpiece.py
sha256:5da8fdc0bd0070c84b718d9cca79e58f2e43cbeb3d50ad8e73315e11ebb7914e tokenizers-0.15.2/bindings/python/tests/implementations/test_byte_level_bpe.py
sha256:277561580028e90f34b3cbcd2151121741f478a67bc402599fe47a6e5b2a0dab tokenizers-0.15.2/bindings/python/tests/implementations/test_char_bpe.py
sha256:b6b31a25a2781e628b39a2965dde19e746d9a2540b7c2cdb9b3636b90ba74365 tokenizers-0.15.2/bindings/python/tests/implementations/test_sentencepiece.py
sha256:8506993a88a25b0fc188572fd70169b33ed95b1b4ee5421e56049f885f88f255 tokenizers-0.15.2/bindings/python/tests/test_serialization.py
sha256:b4e4576d427cf2c8b7b634c402ed1c8f45cbb030d1a4e30841502ec69f369b0d tokenizers-0.15.2/bindings/python/tests/utils.py
tokenizers-0.15.2/docs/
sha256:64da2857b2b8b721109ede3b1b8c1469281129af0c45c2213b17916c1c1efd73 tokenizers-0.15.2/docs/Makefile
sha256:9a5efb500e56a297d44431026a0911a85440854ad8be8d80a955eb76f2be361b tokenizers-0.15.2/docs/README.md
tokenizers-0.15.2/docs/source-doc-builder/
sha256:8c9e3c6b89eae2b85b98ee7fd27341be9495d601d0778b8db0bb06c4f9a413bc tokenizers-0.15.2/docs/source-doc-builder/_toctree.yml
tokenizers-0.15.2/docs/source-doc-builder/api/
sha256:6c80b2c6fafea22cc8210e8c38fc8a22934e532f8b5d7057ccfb9da03794c0c5 tokenizers-0.15.2/docs/source-doc-builder/api/added-tokens.mdx
sha256:aa600d7ecf65f56ad2c6fb115950c5e421d360e09a354e0722bbb59fbb92d504 tokenizers-0.15.2/docs/source-doc-builder/api/decoders.mdx
sha256:af65169fdb9a777f17c7ac74dc47af1308882c669ba14309bdb13151eb534ab9 tokenizers-0.15.2/docs/source-doc-builder/api/encode-inputs.mdx
sha256:7ab4617e68210c301a5cec7652fe499351b52b22775c14e19d51ae09f6e9852d tokenizers-0.15.2/docs/source-doc-builder/api/encoding.mdx
sha256:c26868132cce48ddfcdbc287e7ec54af20e79fc54942211eb1e54de329a2a224 tokenizers-0.15.2/docs/source-doc-builder/api/input-sequences.mdx
sha256:0fa43188b0569e2452ce4a2cdad61c4d966653f1a5b016e6e34dee691360c74e tokenizers-0.15.2/docs/source-doc-builder/api/models.mdx
sha256:5b8e0dfb7ac14288d96825dfd0cb60ec0265ae42dea19f59a30b209a210e72ff tokenizers-0.15.2/docs/source-doc-builder/api/normalizers.mdx
sha256:af1dcc0d43802ed5aac6fdc271c99312b0741685712738e11884240b2df17274 tokenizers-0.15.2/docs/source-doc-builder/api/post-processors.mdx
sha256:7f6172e90e8768ab13eff60a68486c3a67ec0b18ab61029f263f71e678f881ad tokenizers-0.15.2/docs/source-doc-builder/api/pre-tokenizers.mdx
sha256:33c9d09bcc5dd3f5775f8521ad5e0665f357873c389615aa260bea4ed7eafb39 tokenizers-0.15.2/docs/source-doc-builder/api/tokenizer.mdx
sha256:743339cf86db82446d01a4afbbe24b86c1fd42998c76948bb7f7fa5792208829 tokenizers-0.15.2/docs/source-doc-builder/api/trainers.mdx
sha256:3b08ec9077407db0076bf25b9efc271473b3917264f77348ee40dc7dca4b0ec3 tokenizers-0.15.2/docs/source-doc-builder/api/visualizer.mdx
sha256:a94c55d07b6b966d7d930d64a4ed7b2c8d13a5f066cfc9744f9849d3cff6f4fe tokenizers-0.15.2/docs/source-doc-builder/components.mdx
sha256:bf6a3a26a19f0d90bdc24fea638c8b4f12956ac23718e20a425ad20fdf9f537c tokenizers-0.15.2/docs/source-doc-builder/index.mdx
sha256:675c62857dfedbbf779a67b184c1bc71504cd051aa2e5c86c8e68ef8a87350d0 tokenizers-0.15.2/docs/source-doc-builder/installation.mdx
sha256:bf32ea9c2049df10e000962f381453fbe96a51cbb2e0cfccf42c30d557a06220 tokenizers-0.15.2/docs/source-doc-builder/pipeline.mdx
sha256:c719c152095b152e9dd8100f85f0ac8b9b62e43a5d773d4ef68c82f8fd099010 tokenizers-0.15.2/docs/source-doc-builder/quicktour.mdx
sha256:1bd0aeccbcc56d57ad00a272f46773d262f6ce24827cf02227e00354a1779eb4 tokenizers-0.15.2/docs/source-doc-builder/training_from_memory.mdx
tokenizers-0.15.2/docs/source/
tokenizers-0.15.2/docs/source/_ext/
sha256:18d280dbaa0a211a5745e18b271ca6f20b5133a1dbb59ef412cc24c032301939 tokenizers-0.15.2/docs/source/_ext/entities.py
sha256:f6b1fd1f777b23885ed350bd0dea306b7ff6bfabcaae067f56f7ebb7847ef895 tokenizers-0.15.2/docs/source/_ext/rust_doc.py
sha256:d1079093942989915b32985c10183b83445e305370c1aeba026b0bcdeb8644fd tokenizers-0.15.2/docs/source/_ext/toctree_tags.py
tokenizers-0.15.2/docs/source/_static/
tokenizers-0.15.2/docs/source/_static/css/
sha256:d68e6fef383ffcee718a5728690415a204fddda208cd49a2df4931257bb3c476 tokenizers-0.15.2/docs/source/_static/css/Calibre-Light.ttf
sha256:2cc588a746277a0e8a4bc64b95b63aed4ce60061f43cef06d44899b9d0395eec tokenizers-0.15.2/docs/source/_static/css/Calibre-Medium.otf
sha256:8f3af66227ee366466149c568be461f579bac53e411a39acc33583c7b310dfef tokenizers-0.15.2/docs/source/_static/css/Calibre-Regular.otf
sha256:83148cb95241197bcbb93352458548c630dfb707e822f07d2f0f03a3f61b0870 tokenizers-0.15.2/docs/source/_static/css/Calibre-Thin.otf
sha256:a6826e671da01a2fa5a0409e4f9f2edbcf1d3059a988548dd439774906ec448b tokenizers-0.15.2/docs/source/_static/css/code-snippets.css
sha256:c712f6e603226ead1b3a08733599979d1fbb7c063cd14223b65ed9faed78cad7 tokenizers-0.15.2/docs/source/_static/css/huggingface.css
tokenizers-0.15.2/docs/source/_static/js/
sha256:1595a8ffbda81644d7385d6f8ecb8dba96d8343fd6fe55653f34526d79b4e69d tokenizers-0.15.2/docs/source/_static/js/custom.js
tokenizers-0.15.2/docs/source/api/
sha256:ff2cc6e66aba30b79ec0aadd3e2b214095d47ef1f7c1d7dac701cfc056560f24 tokenizers-0.15.2/docs/source/api/node.inc
sha256:591900f42ad20d8cd396a5fb77cc440be3a60e698b61ae3babc818e662e0c68a tokenizers-0.15.2/docs/source/api/python.inc
sha256:c21848d284baef80f8c8ecfc057648c509e91b400fde506304da004e2f22cd34 tokenizers-0.15.2/docs/source/api/reference.rst
sha256:8253c97c3382c969f82ac09ce2854f40cbe530f97ac03d2404abfe5141e31dc3 tokenizers-0.15.2/docs/source/api/rust.inc
sha256:d100fbadb3a88f75476af2acc0fd4ba4d90d59e5cfa919f31293b24d1656ea27 tokenizers-0.15.2/docs/source/components.rst
sha256:93c5dafd101322d8f98c26943744735f99b5f67d93e809dd721b4ebf90371310 tokenizers-0.15.2/docs/source/conf.py
sha256:eb7dfb878646f949ce8e2eae062af4768a79e9afcb200fa124f50201570059e8 tokenizers-0.15.2/docs/source/entities.inc
sha256:3a6453b0107cf0421a04d046a09b736cc248a4ffa847e9e3a7325854455606f7 tokenizers-0.15.2/docs/source/index.rst
tokenizers-0.15.2/docs/source/installation/
sha256:7d7d111f50cec5ea6f39b7b35db11688e4cc0c8acba096e27ff1d053506244d5 tokenizers-0.15.2/docs/source/installation/main.rst
sha256:6590d59131ae3dc72cf0e88106b9a75fa4fbf4ddf4cc713c19536832e13bbaeb tokenizers-0.15.2/docs/source/installation/node.inc
sha256:f8e6a397b54a762200b5adfa6dec0be3ae4d553ace7f570e43052fa6e268e673 tokenizers-0.15.2/docs/source/installation/python.inc
sha256:858c8ab604d54821c8f1b43f9d18638b5c8b703fb098058f3e329f4828e3910a tokenizers-0.15.2/docs/source/installation/rust.inc
sha256:95af1721b255e4b7b4d00ec75a82d80868a0fb771c0bd580d6001976155585dd tokenizers-0.15.2/docs/source/pipeline.rst
sha256:78452aac2be736ee2313929e49b8c342dd1182bc8ad5b75e6d6a1998e1353e08 tokenizers-0.15.2/docs/source/quicktour.rst
tokenizers-0.15.2/docs/source/tutorials/
tokenizers-0.15.2/docs/source/tutorials/python/
sha256:c78ea0da873d0899476945f9a00091bffbc2fc2a6036d5df2a706abf9ba26d4e tokenizers-0.15.2/docs/source/tutorials/python/training_from_memory.rst
tokenizers-0.15.2/tokenizers/
sha256:30abc9fa16351a9c4299828c1edffe5c50342036602e764a0427d6025cd6af31 tokenizers-0.15.2/tokenizers/CHANGELOG.md
sha256:90d6de8601b226b8fecf7380350900a854ce7a1c3e726ce8a6e98e27329383b0 tokenizers-0.15.2/tokenizers/Cargo.toml
tokenizers-0.15.2/tokenizers/LICENSE -> ../LICENSE
sha256:149699cd22a48e6b227f48350d0e1637343d48cb4a436a89a16834025f607938 tokenizers-0.15.2/tokenizers/Makefile
sha256:40fd4e3e650f7af0d4fa62fff0c6600efc9e2b45f8ab3e8440642cc7c6753077 tokenizers-0.15.2/tokenizers/README.md
sha256:487d57adf5a2cb4729326ce4a0c85e80c965e8951f3b38e6bc3868902519b39f tokenizers-0.15.2/tokenizers/README.tpl
tokenizers-0.15.2/tokenizers/benches/
sha256:ea372dbeea560a0576563ce5b89df97b637f226e9cde77027c2036f95805b77a tokenizers-0.15.2/tokenizers/benches/bert_benchmark.rs
sha256:2513f244f08d112f65449f114128b7822ca912621bffb6ae84a6434068a88330 tokenizers-0.15.2/tokenizers/benches/bpe_benchmark.rs
tokenizers-0.15.2/tokenizers/benches/common/
sha256:2fdeba251f5d18b991b868a5902d8aa6f5eba8837fd62debb3422351c83ac184 tokenizers-0.15.2/tokenizers/benches/common/mod.rs
sha256:bc71182748a4b70f26cf45e4f819adff1bd48654af6f0253d4c70ecd1bef2a6a tokenizers-0.15.2/tokenizers/benches/layout_benchmark.rs
sha256:fb517f09a7c131e138f952359aeaf8f2f9fb9a814aa2935323f7d378c9c3de12 tokenizers-0.15.2/tokenizers/benches/unigram_benchmark.rs
tokenizers-0.15.2/tokenizers/examples/
sha256:edbe11c1ee633c57ebddd72e6bde619aaebb1f066df6698c20a01c4c214809a6 tokenizers-0.15.2/tokenizers/examples/serialization.rs
tokenizers-0.15.2/tokenizers/examples/unstable_wasm/
sha256:f5ba0dc3ff1ad2d818ccbe695f554c31822aea61713639e7d5b76d9a98f2ecf5 tokenizers-0.15.2/tokenizers/examples/unstable_wasm/.gitignore
sha256:c0a32c90ef7eda62d0dcadee83f91759d1251297c715555a39586d59df98ac1b tokenizers-0.15.2/tokenizers/examples/unstable_wasm/Cargo.toml
sha256:b83a774dea05794aa3996310404dad4806617358488558a6c96f5d5922269b7b tokenizers-0.15.2/tokenizers/examples/unstable_wasm/README.md
tokenizers-0.15.2/tokenizers/examples/unstable_wasm/src/
sha256:700108192809e07d83d25abd3c5465d0d8c268f9283428c6c38be3d68876fc94 tokenizers-0.15.2/tokenizers/examples/unstable_wasm/src/lib.rs
sha256:4202fe474a0ae76174696d04bc001ae1f838071e1b14ebe9a4913e9efdfade36 tokenizers-0.15.2/tokenizers/examples/unstable_wasm/src/utils.rs
tokenizers-0.15.2/tokenizers/examples/unstable_wasm/tests/
sha256:181970ff03a4a3115a9ead0e90015f0aefef46089c4fa6cd4d113c5d13e64e09 tokenizers-0.15.2/tokenizers/examples/unstable_wasm/tests/web.rs
tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/
tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/.bin/
sha256:2bf3871139ef28ae0d47e9b89ac0bb4b67ed5c794fca8d6092eaa37d9c9230e3 tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/.bin/create-wasm-app.js
sha256:c8015948b6eaefde5eaa3862f2fe5ce0245fcde4d6e66f7441dcdcbbed46e652 tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/.gitignore
sha256:06fc329654011dddca329034f2a0f57650b9e0fd50930509f411d43ad05b12ea tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/.travis.yml
sha256:8173d5c29b4f956d532781d2b86e4e30f83e6b7878dce18c919451d6ba707c90 tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/LICENSE-APACHE
sha256:ef36e00d5e8dd8c6260a46855a1a578ff546916c7624119a60832e5ce5c510e4 tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/LICENSE-MIT
sha256:06d60699af573a691b095bded90a83e5d2464e98c56e36d0d6bd1608c4d45a8f tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/README.md
sha256:5b38ab61e867664a13a5d8af50be6904726da873186752497d54bba0ff857c6a tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/bootstrap.js
sha256:13fb2506ed4b9c6152dca7cabe2ec8d8e6052f2a27642e22d81ae58f75acc98a tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/index.html
sha256:4e7d9c000d676dcaecf56808298b3d3c5efe50a3a386326b5fb96bf7f439b1ca tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/index.js
sha256:7e6e6b8cde0cf7452e4297d16c5ec10c983378b50da2af2eba1452ccd8b0af23 tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/package-lock.json
sha256:a28345c99b3be9287a042467cd5b55be673d5bd8431b391506589ef2667eba9a tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/package.json
sha256:1be82d86fa4bf762b6744e612ea837ceda33640e71ae481e9ecbddebf9e35fe6 tokenizers-0.15.2/tokenizers/examples/unstable_wasm/www/webpack.config.js
sha256:2b92ea252be0fbc26f70317cdaa7b6411ea634b50d55338cd8c495e4dbf25d1d tokenizers-0.15.2/tokenizers/rust-toolchain
tokenizers-0.15.2/tokenizers/src/
sha256:3b9257390a3f99ec35a3ca36278c58f266ebb8a983ad3a2b32b4c423666cbfa9 tokenizers-0.15.2/tokenizers/src/cli.rs
tokenizers-0.15.2/tokenizers/src/decoders/
sha256:9c363604a82b2e1439afaa78efe6b11c2d33348525bb934c797a37f2007bf3bb tokenizers-0.15.2/tokenizers/src/decoders/bpe.rs
sha256:58bd61910667e4aac6700d420adb038653199e3cad7944cf8abb950046314ca6 tokenizers-0.15.2/tokenizers/src/decoders/byte_fallback.rs
sha256:c091493ea218c9543c2258ab0354458f986040f61bf3112867932b506d882999 tokenizers-0.15.2/tokenizers/src/decoders/ctc.rs
sha256:19b7bfca165bdee6ab084ed5429a49afc9d52664c51187a500335a80707850fa tokenizers-0.15.2/tokenizers/src/decoders/fuse.rs
sha256:0a0374cadb272f91aa3a116a65d12152f4724294c6754089daf4279a1a574a1c tokenizers-0.15.2/tokenizers/src/decoders/mod.rs
sha256:22697620e1edfa3c6ac470e047b3a007a6f30fea28fb0643164eb2de304ac17a tokenizers-0.15.2/tokenizers/src/decoders/sequence.rs
sha256:82a317f29d62f2cbbc68bf993db2a5c6d41deb9c7622da12fcf37eb14d8ed36a tokenizers-0.15.2/tokenizers/src/decoders/strip.rs
sha256:08770ae7e05bef4d2f729becca7aa7063fe06539fe7616cbefd9a21a16a4f43d tokenizers-0.15.2/tokenizers/src/decoders/wordpiece.rs
sha256:191c9abbf820cd4cf25c78be2bd74d419708059247f7fc0bb9126bc83ef924ea tokenizers-0.15.2/tokenizers/src/lib.rs
tokenizers-0.15.2/tokenizers/src/models/
tokenizers-0.15.2/tokenizers/src/models/bpe/
sha256:dcb06b6767b18ac4530b110de19f53c416e23b1b09deb5133a3b18d3b0fc068b tokenizers-0.15.2/tokenizers/src/models/bpe/mod.rs
sha256:7236aadbe024cdb1b3f73ece87ec95456bef49774c2c229c4889968327e43769 tokenizers-0.15.2/tokenizers/src/models/bpe/model.rs
sha256:8834c78ec24f8de4c2fbb98c27e0b8fafeda6ab23948a29442a3442c9b7a8c26 tokenizers-0.15.2/tokenizers/src/models/bpe/serialization.rs
sha256:fd243b3d2e5346af14a507d8d37cce4c216f1d52fd4d42f05ca1a10f16a27e95 tokenizers-0.15.2/tokenizers/src/models/bpe/trainer.rs
sha256:90c8014d1691eed20abb99cd385777344e6c556443a0005647af795aee7f43c3 tokenizers-0.15.2/tokenizers/src/models/bpe/word.rs
sha256:a018fe7e16a633eea092f79de542cfd7785051bb70da711b3fed13ab78e39009 tokenizers-0.15.2/tokenizers/src/models/mod.rs
tokenizers-0.15.2/tokenizers/src/models/unigram/
sha256:d0952d809c182b07df13114961dfd068eabd53e82a6e6a5a4fc5fb4a2cf72a88 tokenizers-0.15.2/tokenizers/src/models/unigram/lattice.rs
sha256:cf317d99ceebe2b4326237aa8f8bfdc2293a5dd0a7fc69a6e9e0028b804a000e tokenizers-0.15.2/tokenizers/src/models/unigram/mod.rs
sha256:9756fb725f0958bdec4d81f4533d0902c71fffeae8293dc3ce020d4ffdbaed3b tokenizers-0.15.2/tokenizers/src/models/unigram/model.rs
sha256:b848b6a26ff76617d7717194666d7d47d5765dd684a8abb1c0708b87cd1519aa tokenizers-0.15.2/tokenizers/src/models/unigram/serialization.rs
sha256:7125823b970c874d2187982dab8a11ec74c85398e9e8c0d13438cbdf813b19b8 tokenizers-0.15.2/tokenizers/src/models/unigram/trainer.rs
sha256:304f7f387d7be43bd01e72b05a950f0a41d95ff51b219f0b123d46b84f70f30e tokenizers-0.15.2/tokenizers/src/models/unigram/trie.rs
tokenizers-0.15.2/tokenizers/src/models/wordlevel/
sha256:dff4dcd3792d335e52b31ebdd136f81b9a30a36585cefdaf8addf07b3a6ec847 tokenizers-0.15.2/tokenizers/src/models/wordlevel/mod.rs
sha256:1af61e4f303e4e849162f0a50744d592e2a0ef3fcaee202b474ebaaeaeb6b453 tokenizers-0.15.2/tokenizers/src/models/wordlevel/serialization.rs
sha256:11fa23268c8f145a2e18e6a3a7da3519632ce72a9be7624ad950e8fa76fdb528 tokenizers-0.15.2/tokenizers/src/models/wordlevel/trainer.rs
tokenizers-0.15.2/tokenizers/src/models/wordpiece/
sha256:ca270710797843c3aa726eeeee8112007618c22c7b20851fbba40421614dfe2e tokenizers-0.15.2/tokenizers/src/models/wordpiece/mod.rs
sha256:188ece9ae1641994bb5145a2ee600fc9b4d616ca3c0b9792b6c740b8348b0569 tokenizers-0.15.2/tokenizers/src/models/wordpiece/serialization.rs
sha256:4aa1ffd863a4be04dce17c8941ea68e3b4b4d257317593aacec3282aef698888 tokenizers-0.15.2/tokenizers/src/models/wordpiece/trainer.rs
tokenizers-0.15.2/tokenizers/src/normalizers/
sha256:a938fdac02447e33393d98edcaad015f58e2095d633521cee85a2b60ccd995bd tokenizers-0.15.2/tokenizers/src/normalizers/bert.rs
sha256:bb7f272e6e0476b3ea624a7f1b7b75494ba65e773c2d347780940ce8391533a2 tokenizers-0.15.2/tokenizers/src/normalizers/mod.rs
sha256:69acb2a2be78760c59a39a2c1f7db6469c75708900b53a7f9bbe189ed83aa6da tokenizers-0.15.2/tokenizers/src/normalizers/precompiled.rs
sha256:99d7336d9c9ba8baca0f30b435c2a7930332100ec0d58f6221eff94da373fa19 tokenizers-0.15.2/tokenizers/src/normalizers/prepend.rs
sha256:2f4566ee4f7b108559cc0946a6e4c91866d0bb98de67e61d139cfce8e1045cb0 tokenizers-0.15.2/tokenizers/src/normalizers/replace.rs
sha256:3d088593a2fa931261f4cd906440b83c053351a0e0e12cb797e541b98d7e2b06 tokenizers-0.15.2/tokenizers/src/normalizers/strip.rs
sha256:6c2312c8a2ba9461b850147fd63432f59f81e942042995d7c28e097f8198139e tokenizers-0.15.2/tokenizers/src/normalizers/unicode.rs
sha256:4b09adb7232537923d55352f2b0707f01c14929117d28fc34dd4b13a70049328 tokenizers-0.15.2/tokenizers/src/normalizers/utils.rs
tokenizers-0.15.2/tokenizers/src/pre_tokenizers/
sha256:32dd35c1f815072c6ba32df390389d10e5f4073b480d1c4ade06143ba23b571d tokenizers-0.15.2/tokenizers/src/pre_tokenizers/bert.rs
sha256:48e0fae79ee9f5a9035cac9639d79c52a77e03ecca97670d32488753d0698142 tokenizers-0.15.2/tokenizers/src/pre_tokenizers/byte_level.rs
sha256:7d2d0170eeb5d864742df867958f379ec564bc5af67e835f5793b06bfab4185f tokenizers-0.15.2/tokenizers/src/pre_tokenizers/delimiter.rs
sha256:1452f56c9fcb0265ee91c5439f860033209e44fe61565b743970d64216b9b41a tokenizers-0.15.2/tokenizers/src/pre_tokenizers/digits.rs
sha256:fb313084748a989a7cc063a0314d0d5fad399a27c0db37a6dea2efb7f1b03fb1 tokenizers-0.15.2/tokenizers/src/pre_tokenizers/metaspace.rs
sha256:989cfc1c38e5f1a3d2bb48f4b2493e28fdd8562d82a98a133ea26f490036d8ea tokenizers-0.15.2/tokenizers/src/pre_tokenizers/mod.rs
sha256:cfb82a699365f2bad1fe36be7ec5523276c90d4cfcf86e8e985e0ba0de6fd9e3 tokenizers-0.15.2/tokenizers/src/pre_tokenizers/punctuation.rs
sha256:1742957bb2cbe971fa791dac41b9e5fad5bc9843cb4eaa0e8e604f6cdc75e1df tokenizers-0.15.2/tokenizers/src/pre_tokenizers/sequence.rs
sha256:d12ee03b7a4d8006979906883ed3d993132cf2b9503f31ebfb0b0d376020d220 tokenizers-0.15.2/tokenizers/src/pre_tokenizers/split.rs
tokenizers-0.15.2/tokenizers/src/pre_tokenizers/unicode_scripts/
sha256:5740e30dba4e6e819309a24144c95d228d9aca437950fc0285b134f238f1788e tokenizers-0.15.2/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs
sha256:4ed0d408409ec35bcba141d75a2647d7503bc17d1e6f4641627325c99bd56edb tokenizers-0.15.2/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
sha256:c614fe1c9a79b1e4cbbb9f3cc3ad130dd6506e12ff74688d611a8eeff47c2167 tokenizers-0.15.2/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs
sha256:0d0d8284ab86d9b0f06b0b0f7024d50618f74de2b1c2ffc2877032a587a0753f tokenizers-0.15.2/tokenizers/src/pre_tokenizers/whitespace.rs
tokenizers-0.15.2/tokenizers/src/processors/
sha256:db17c4467ab2cb95db5dbe22d3abd6ca61d0c463537c49f40896a148bcf37ec1 tokenizers-0.15.2/tokenizers/src/processors/bert.rs
sha256:613ea5420b5029e396eda84c359a5785feb55e8dca6ccea04b03ec04d501460a tokenizers-0.15.2/tokenizers/src/processors/mod.rs
sha256:713d3ade0492cfcd9d1aecb269a5cc8aacf321a716b13fe2dfbfc7b023c0a67a tokenizers-0.15.2/tokenizers/src/processors/roberta.rs
sha256:96bfe2c23419a88b270bb3d6edb34d9e61e12b9e50cc8cd8b852a48e81a2c18e tokenizers-0.15.2/tokenizers/src/processors/sequence.rs
sha256:3f5115c656c044a9db63abbe27fafe018c674a50c1a7010639bb3ffece607920 tokenizers-0.15.2/tokenizers/src/processors/template.rs
tokenizers-0.15.2/tokenizers/src/tokenizer/
sha256:947c9354fb9f6c9c4f1eb49215a360d05436adc70d32a5a09e21d2eaa7d198bc tokenizers-0.15.2/tokenizers/src/tokenizer/added_vocabulary.rs
sha256:deb947238b2b93e8dfc31c00d44c8ceb5da7ab81e6733ab5b9c9ff62aa42577e tokenizers-0.15.2/tokenizers/src/tokenizer/encoding.rs
sha256:dc1710fcbeb55478f7a1bf9121a078010fa35e061d92cbca71a003ae7ef5535b tokenizers-0.15.2/tokenizers/src/tokenizer/mod.rs
sha256:c0edac6fc35e7db28ea47f2f34b2673995d92203e5004d721bbbe4772df36b74 tokenizers-0.15.2/tokenizers/src/tokenizer/normalizer.rs
sha256:036af136c7991371dd30571d13875c13c775e007dbd270cb5efdb128aba7d6a4 tokenizers-0.15.2/tokenizers/src/tokenizer/pattern.rs
sha256:8c4db8cacb5f5f4a904c391ca86e71d40090a8fab90f5f1eba6718e5659d01fa tokenizers-0.15.2/tokenizers/src/tokenizer/pre_tokenizer.rs
sha256:9c8a9d27ad8b60b0bc8be34f68b58b60657ce8edd7ea26f0a926a9c6692e18b1 tokenizers-0.15.2/tokenizers/src/tokenizer/serialization.rs
tokenizers-0.15.2/tokenizers/src/utils/
sha256:3b5fb562f3de1822fda832bd9c7a8c17f5374cc36d28da1d36c68c362a76c643 tokenizers-0.15.2/tokenizers/src/utils/cache.rs
sha256:bd4760ee176d3c3f088e0e6f37cc06afc18745fb8375d6a6a312586cfcc697fa tokenizers-0.15.2/tokenizers/src/utils/fancy.rs
sha256:719a46100a8b6ec6662bd037bc0f72e2213eefd3ddb1137984648a3ca3ea9529 tokenizers-0.15.2/tokenizers/src/utils/from_pretrained.rs
sha256:004540ac0fc434f6ec25c1cf8e6818998550da936a728aa5a301458c6c66a540 tokenizers-0.15.2/tokenizers/src/utils/iter.rs
sha256:ef57380e0ebd207fb7bc09f90a1aa23db851249d80fdf56d6738e8ac55cb6dbd tokenizers-0.15.2/tokenizers/src/utils/mod.rs
sha256:f6f58fb3fcf2dfbee2e8edc9b763247443fa102aea185481ff591f7d52c5256e tokenizers-0.15.2/tokenizers/src/utils/onig.rs
sha256:17e2b82c50ae7a3c296cda7898519c57911a8887a6ff5e9e57c27d215f5ed9e7 tokenizers-0.15.2/tokenizers/src/utils/padding.rs
sha256:f4629256e1e94a860b14ab26cdd2474a92c49b2cac31dd835ff3e4c6ab0c124a tokenizers-0.15.2/tokenizers/src/utils/parallelism.rs
sha256:2bae922cafe5167d36cc6cd973edf4fcf6d0224be4d27fa47b9285dcca447ca8 tokenizers-0.15.2/tokenizers/src/utils/progress.rs
sha256:d73513e1391cf656258e2752e4fc392b0946172776f6764fa9eedda419e88d04 tokenizers-0.15.2/tokenizers/src/utils/truncation.rs
tokenizers-0.15.2/tokenizers/tests/
sha256:493c7097ed9a51ac4e6837ff729e71cc2ccd1bba1623e882147ca9447449bff6 tokenizers-0.15.2/tokenizers/tests/added_tokens.rs
tokenizers-0.15.2/tokenizers/tests/common/
sha256:3f1cb483fac09c7fecc4df337f94eb986b6e02e92895bb185899f61a4465fbc8 tokenizers-0.15.2/tokenizers/tests/common/mod.rs
sha256:6bef0acc974bcf1488470eec9a166c0aee27835112a6cdda7b855b0b8052aa05 tokenizers-0.15.2/tokenizers/tests/documentation.rs
sha256:c44fc0afd1f9ec6c9ab30218b84866175fee2ae8e4315603fec0af9d1ae0f813 tokenizers-0.15.2/tokenizers/tests/from_pretrained.rs
sha256:f0f790bbe60a47112b4c250991c5769f10fa0ad3bf40fb200922b59caf841933 tokenizers-0.15.2/tokenizers/tests/offsets.rs
sha256:383ad85e2187c2db29a88ffe18553e4c3dfdd79e9220a400a702cf703b6071aa tokenizers-0.15.2/tokenizers/tests/serialization.rs
sha256:f0433b135226479693729e4ebe15d91767545b17a5ecefe87717afc42e0dcb19 tokenizers-0.15.2/tokenizers/tests/training.rs
sha256:740b12eb54aea2731c4f922bac5b100e09c254f40bd8362e8ee5516856e4952e tokenizers-0.15.2/tokenizers/tests/unigram.rs
Return to home page.