sha256:393e8fa95b0ff298ccd1321c5932fdbfdb601224dea7170bcde06e803c10bb45
Alias: sha256(gz(tar))
The hash you've requested:
sha256:21e9a235c72e49cafb7ed29829650f6a49f0a951194e1f0168b3b2f547362569
is considered an alias for sha256(tar).
Visit the canonical page.
Build input of
- openSUSE: python-tokenizers 0.20.3-1.1 (tokenizers-0.20.3.tar.gz) sha256:21e9a235c72e49cafb7ed29829650f6a49f0a951194e1f0168b3b2f547362569
- openSUSE: python-tokenizers 0.20.3-1.2 (tokenizers-0.20.3.tar.gz) sha256:21e9a235c72e49cafb7ed29829650f6a49f0a951194e1f0168b3b2f547362569
Contains Dependency Lockfiles
-
sha256:c3d48b6639e952c95e86e11700813aa1b474a2efc61b255fb2fd8a51ea267fedtokenizers-0.20.3/bindings/node/yarn.lock -
sha256:41ae195f80e1bb41ced42f4a195bbb7874be878c2b8673623d45a0fdac42e596tokenizers-0.20.3/bindings/python/Cargo.lock -
sha256:b3a6eb06a5d73eec232ef54a329bb448bfefea5adff6013de7eee94353f41ef9tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/package-lock.json
Diff
Content
tokenizers-0.20.3/
tokenizers-0.20.3/.github/
tokenizers-0.20.3/.github/conda/
sha256:a6591b816fed5e1831c613814bf70c7180e0f015057bad2a5ab6f8d50d6e0c22 tokenizers-0.20.3/.github/conda/bld.bat
sha256:27c731a659d61a4fce9877fea566686a01c3b5bc804f73b9949c2dd3d534b2f6 tokenizers-0.20.3/.github/conda/build.sh
sha256:45f202ad857770620e4e919ebebee205711b49975812ad28f11decc984fdfe7e tokenizers-0.20.3/.github/conda/meta.yaml
sha256:9f92c3d0c0082e9409d62ddf4a7c4612acf8692dcf1b4bed33fdffbe43f153fe tokenizers-0.20.3/.github/stale.yml
tokenizers-0.20.3/.github/workflows/
sha256:c3324b307eb3eb71f160357c07ef49f5a86b9a1c6f16672a0946c0195d29c96b tokenizers-0.20.3/.github/workflows/build_documentation.yml
sha256:4e6c12654c7e95e4e51652e096fad2e6bccd3ec5193543ea9b4c8eada7fe80e0 tokenizers-0.20.3/.github/workflows/build_pr_documentation.yml
sha256:57ae5e4c90a9d327f25a9ad148fe7ace589865f1f8d9565d78460facda3f748c tokenizers-0.20.3/.github/workflows/delete_doc_comment.yml
sha256:db118b73b00dc7ccd3aaea04e90b31b6f2bca549160604697182b297d653bb50 tokenizers-0.20.3/.github/workflows/delete_doc_comment_trigger.yml
sha256:d530957189f5d2a22f2ddd659800c921c30c384e93f22d9c11f9c5c4f218d0e1 tokenizers-0.20.3/.github/workflows/docs-check.yml
sha256:24ff5b5d2305bb9a1873e7323ac6b766e83da8bda4ec68e59cb77f7f4c65251c tokenizers-0.20.3/.github/workflows/node-release.yml
sha256:91d4604a2f722d440580e030003e7225ad67e31a7444deff2f22330a333121a2 tokenizers-0.20.3/.github/workflows/node.yml
sha256:0252236cc5193e507abc8a6fa9edfec5622b3d6c88bdd9f516a1c2d4fbed9a92 tokenizers-0.20.3/.github/workflows/python-release-conda.yml
sha256:ad34a7883f36e4e8ed99bb471626405565f9adb80e5419f999652a4061f28532 tokenizers-0.20.3/.github/workflows/python-release.yml
sha256:b35af18adfe0f90bede585935d08812a697d10e1577e27caedb3524b313d030b tokenizers-0.20.3/.github/workflows/python.yml
sha256:e331d54ecc193759b5f41c391b8da0570cb70bf2b5704ec16377c30507f70f07 tokenizers-0.20.3/.github/workflows/rust-release.yml
sha256:1ab870c40c54776c10fbac18f0ddc012c1635c5e32a767c89f7b3c413069c483 tokenizers-0.20.3/.github/workflows/rust.yml
sha256:819e3a9ecb1c3df334a25a296165e686f5f246e237d133c0d1f2fc12e9b6634c tokenizers-0.20.3/.github/workflows/stale.yml
sha256:6a2cbf68e4a715645e75ae9395a10efa1770649bb038ade5f2959f97e7a2fd73 tokenizers-0.20.3/.github/workflows/trufflehog.yml
sha256:53ff3a72fd52c75b89371910748b4bc05e3718ab640b9751bb7df7cd8d97231f tokenizers-0.20.3/.github/workflows/upload_pr_documentation.yml
sha256:91da3b625c669f733040eaba80a5a8067890e64714ef6089d36a261b532cf7d0 tokenizers-0.20.3/.gitignore
sha256:4d377e818cff7934516a38936b80230a34153c0bbb42dff20f64e84bfad19826 tokenizers-0.20.3/CITATION.cff
sha256:c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4 tokenizers-0.20.3/LICENSE
sha256:296eb3baa8876f09b2a13747116e78fec18df22ac1e6a1868c77b0b23f8621da tokenizers-0.20.3/README.md
sha256:c98eaf43f6046659e2b0e33b24d7ad8041e38be1956925df8d9794bbd1c9b8d5 tokenizers-0.20.3/RELEASE.md
tokenizers-0.20.3/bindings/
tokenizers-0.20.3/bindings/node/
tokenizers-0.20.3/bindings/node/.cargo/
sha256:95eccdbcb0b22f08a34041c422ff5821bd783f9392d8bd1136305239fc1f7287 tokenizers-0.20.3/bindings/node/.cargo/config.toml
sha256:30adc741ffc18728591498ec0d123497e6262c963bf75a91177a0056c50ae9e1 tokenizers-0.20.3/bindings/node/.editorconfig
sha256:f90a518eaf74b6a9c515d44ac0159cf87e3ebc1913792ec0dc38334b6d626eec tokenizers-0.20.3/bindings/node/.eslintrc.yml
sha256:d005ffc2cefc69de8caf1394a6a074754a8bb808fb5ec1d92fc223db3d96bbed tokenizers-0.20.3/bindings/node/.gitattributes
sha256:27071e79228833618f823a63dcaf2d0bca21a2db81ece8edf7422cd9bb992274 tokenizers-0.20.3/bindings/node/.gitignore
sha256:c7206e1a30314ad09058859077f48794851e3079811b7c970a15275b39645a38 tokenizers-0.20.3/bindings/node/.prettierignore
sha256:d32b93a2df8144b2d8e82ac0196e5fa1f162591d00434ff884f1ded5cc50d331 tokenizers-0.20.3/bindings/node/.taplo.toml
tokenizers-0.20.3/bindings/node/.yarn/
tokenizers-0.20.3/bindings/node/.yarn/releases/
sha256:64c0b63e5fa0e21f354b5eff7c9a1b486f32046bfc3283670569e3c6a9cad102 tokenizers-0.20.3/bindings/node/.yarn/releases/yarn-3.5.1.cjs
sha256:bda4132acb74d9f289cf423e33ab81782f03377641413b7626444fa759f52b1d tokenizers-0.20.3/bindings/node/.yarnrc.yml
sha256:d95c7d42be256ab9e0ac5f008b11edbfdfc2d261e8857f624dbd0c9d66756247 tokenizers-0.20.3/bindings/node/Cargo.toml
sha256:02965ed67e29f871a53f7a7afa593a0ddc8020f86ccce0faaf1aa59215b5269e tokenizers-0.20.3/bindings/node/LICENSE
sha256:e9398e4e77c114500ecb33b03abc0ff98ecd973ee770ad481549232beef59a06 tokenizers-0.20.3/bindings/node/Makefile
sha256:61377fafc00be92c730d6ca920c145de5b1910ddd9a98419309376699da742b2 tokenizers-0.20.3/bindings/node/README.md
sha256:71b61de74caafb5f27db3d24d197b64282f49f807b969e140ca8ce181e5464fc tokenizers-0.20.3/bindings/node/build.rs
tokenizers-0.20.3/bindings/node/examples/
tokenizers-0.20.3/bindings/node/examples/documentation/
sha256:32099423a4192e4ae956656314cfd9c5fefc4b0f0c0783dcd5f5a5206a7a398c tokenizers-0.20.3/bindings/node/examples/documentation/pipeline.test.ts
sha256:f1e06f10edd4dc2144769b5f28ad0f24cb5f08fd26e1e20949ce0611ea012cd9 tokenizers-0.20.3/bindings/node/examples/documentation/quicktour.test.ts
sha256:e928ddd08356b8813a25919156f953fe20584eb49caa5625b1ce0d6d56102b34 tokenizers-0.20.3/bindings/node/index.d.ts
sha256:87cfc88607e694f3eac84d5af93a56dde83efb16567034d4db021a8ef7d591d3 tokenizers-0.20.3/bindings/node/index.js
sha256:fdbdbd048885ea46a8ce806675fb47bb9a04975d55132d7080a458a78a6a2b94 tokenizers-0.20.3/bindings/node/jest.config.js
tokenizers-0.20.3/bindings/node/lib/
tokenizers-0.20.3/bindings/node/lib/bindings/
tokenizers-0.20.3/bindings/node/lib/bindings/__mocks__/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.20.3/bindings/node/lib/bindings/__mocks__/merges.txt
sha256:ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356 tokenizers-0.20.3/bindings/node/lib/bindings/__mocks__/vocab.json
sha256:e2c1d7f033886700bc4d6718e92a3b00547268a5c083ce91eeca79f0771cf26b tokenizers-0.20.3/bindings/node/lib/bindings/__mocks__/vocab.txt
sha256:bae90354fdc246dc1819d8e813f01f7a0b37b8cbc278de5f48c320805819ec25 tokenizers-0.20.3/bindings/node/lib/bindings/decoders.test.ts
sha256:930510ac93ffd04dc780138b29c30f63709a71071aa8b194ce518ad8ed108c1b tokenizers-0.20.3/bindings/node/lib/bindings/encoding.test.ts
sha256:ea23c29304840ecccd24f78a117db88a1380b35d6d3a811626d83e597bdd471c tokenizers-0.20.3/bindings/node/lib/bindings/models.test.ts
sha256:3201d9ca7d7c42d9934f9ae3e3331f9c0488f0adc713f6ff6ec64251c4ab6178 tokenizers-0.20.3/bindings/node/lib/bindings/normalizers.test.ts
sha256:29db4dbd89bd55c88c4a43b6ae24b2610bc05858fc6a55f33cf1c973b72c2198 tokenizers-0.20.3/bindings/node/lib/bindings/post-processors.test.ts
sha256:529ecc9b4adcc0d28ef1b52445898086ed0e5e70bea9d49862820e691b2b6215 tokenizers-0.20.3/bindings/node/lib/bindings/pre-tokenizers.test.ts
sha256:3fd18a74349b98d8bc9392e7a10b3e9ea0335c2b58ee8afe1f97b58a85df1a1a tokenizers-0.20.3/bindings/node/lib/bindings/tokenizer.test.ts
sha256:68c1660db395387412cd5ec38cfd6e01c69f0da362e477acbdbcee561e624c46 tokenizers-0.20.3/bindings/node/lib/bindings/utils.test.ts
tokenizers-0.20.3/bindings/node/npm/
tokenizers-0.20.3/bindings/node/npm/android-arm-eabi/
sha256:05a1d365a6029297f0e0b1209776c25b42cdf7a51a52e8c76b85b542c8c7ab20 tokenizers-0.20.3/bindings/node/npm/android-arm-eabi/README.md
sha256:90a397203ba933dc527dff748946c38d7418ca9cfd62035a20d4e36458e06dec tokenizers-0.20.3/bindings/node/npm/android-arm-eabi/package.json
tokenizers-0.20.3/bindings/node/npm/android-arm64/
sha256:7c0cd0416e0548f05948cf6f4c4d950d966d2107aa3c62e0c4fa47e2ff077240 tokenizers-0.20.3/bindings/node/npm/android-arm64/README.md
sha256:43fed355ff5375b82e34895553f69e3d7a55a6a3173d28038909c5db0d662a73 tokenizers-0.20.3/bindings/node/npm/android-arm64/package.json
tokenizers-0.20.3/bindings/node/npm/darwin-arm64/
sha256:ee3950cafecfb51853d3762ae9369813f4cd397ff0477900ae5623fc0e33540c tokenizers-0.20.3/bindings/node/npm/darwin-arm64/README.md
sha256:a21b46740ea8236cc37ca25a65e5354609bdcf2cdeabfa24461f61f700f470db tokenizers-0.20.3/bindings/node/npm/darwin-arm64/package.json
tokenizers-0.20.3/bindings/node/npm/darwin-x64/
sha256:23b156cb161d34e087477df0a946cfedf47dd665a8cb77bb637a2ea8dfdf80ff tokenizers-0.20.3/bindings/node/npm/darwin-x64/README.md
sha256:aad332092b741c591c39e3898e8f24ccfebb41fc7c1995a2114e2ab84700488d tokenizers-0.20.3/bindings/node/npm/darwin-x64/package.json
tokenizers-0.20.3/bindings/node/npm/freebsd-x64/
sha256:c21c445c4f03fb72275dea0459b584b3d1b107321af13a80b018aacfe89f57e6 tokenizers-0.20.3/bindings/node/npm/freebsd-x64/README.md
sha256:e838711b46934e6b1bfa6477ae6b0d549f07c3f70dcb0b59fcd735ec4e38e22f tokenizers-0.20.3/bindings/node/npm/freebsd-x64/package.json
tokenizers-0.20.3/bindings/node/npm/linux-arm-gnueabihf/
sha256:c485213dbc76af8e6f54bd83be63777f719060e67c87921d426a229ac9363424 tokenizers-0.20.3/bindings/node/npm/linux-arm-gnueabihf/README.md
sha256:0c826d635835adcbc4bb7dde99cac6e9fac6954247c60958b77a63a3a4fb6b6f tokenizers-0.20.3/bindings/node/npm/linux-arm-gnueabihf/package.json
tokenizers-0.20.3/bindings/node/npm/linux-arm64-gnu/
sha256:fca73a7053f15219b1cd717dfe93dad41f1ce8ed0cbe77f06e0bae890be2bcf2 tokenizers-0.20.3/bindings/node/npm/linux-arm64-gnu/README.md
sha256:5de0ee27d1daa0c4d28eba57d66e891a51dc27d65c0cb1843ed47de10ca8399e tokenizers-0.20.3/bindings/node/npm/linux-arm64-gnu/package.json
tokenizers-0.20.3/bindings/node/npm/linux-arm64-musl/
sha256:a42323205e03c06ab50640eb6616e8e29de7f0e1a24bc87a85b5adc39bbc743f tokenizers-0.20.3/bindings/node/npm/linux-arm64-musl/README.md
sha256:ad55cfef52a07822aad94d364c895ee7a9ddaf51be38681ac2b62f88a5beef52 tokenizers-0.20.3/bindings/node/npm/linux-arm64-musl/package.json
tokenizers-0.20.3/bindings/node/npm/linux-x64-gnu/
sha256:b8f3cdc50472110fee2373517b0ef96ce803641d4c74b95f82ffc3f85b449841 tokenizers-0.20.3/bindings/node/npm/linux-x64-gnu/README.md
sha256:9f936b7429d60f3dab9319422da2216b31ba0330311284668e2161c6cac0f22d tokenizers-0.20.3/bindings/node/npm/linux-x64-gnu/package.json
tokenizers-0.20.3/bindings/node/npm/linux-x64-musl/
sha256:9b4cc26fdf788f433f1d7dcbd9aed1979b655418cc8b9810120eb01651f7f18d tokenizers-0.20.3/bindings/node/npm/linux-x64-musl/README.md
sha256:4260f5c07c5cf19f2e9f44500f17162c9c77955582b0887d6a26b143ea5acba2 tokenizers-0.20.3/bindings/node/npm/linux-x64-musl/package.json
tokenizers-0.20.3/bindings/node/npm/win32-arm64-msvc/
sha256:cf057a5481c42a26b03fffa4d33b6d196a10a81223992f1a17271b27290d06a4 tokenizers-0.20.3/bindings/node/npm/win32-arm64-msvc/README.md
sha256:6c35853dc168d4204bdd98c2b1709be3f0fc6b1d898b2c5fdad106fa2c28841b tokenizers-0.20.3/bindings/node/npm/win32-arm64-msvc/package.json
tokenizers-0.20.3/bindings/node/npm/win32-ia32-msvc/
sha256:df2c8f0c9d538872f1c10995666f6a9a6b13b6af4eb202efb76b90c24bc980b5 tokenizers-0.20.3/bindings/node/npm/win32-ia32-msvc/README.md
sha256:366da5b97cdde4fe1f9ba87346cafa178042fc528a9d41726dcea4863f7e55f4 tokenizers-0.20.3/bindings/node/npm/win32-ia32-msvc/package.json
tokenizers-0.20.3/bindings/node/npm/win32-x64-msvc/
sha256:08b6de0abc7178dd64cc5054624a04545d0ac3e1019822452809867fa56b45df tokenizers-0.20.3/bindings/node/npm/win32-x64-msvc/README.md
sha256:6ba80cc97010f6c17faa664569295b97add7252b504ad46a99818e31fd89161c tokenizers-0.20.3/bindings/node/npm/win32-x64-msvc/package.json
sha256:9cf3cca1abebba89176c5fd7a7e6e13ef75706b3db6395081b08119abea31031 tokenizers-0.20.3/bindings/node/package.json
sha256:d53b7e5653f4a60e4f6558b9d7704045ec0ff0687a768ff3163a67acab4c9f6d tokenizers-0.20.3/bindings/node/rustfmt.toml
tokenizers-0.20.3/bindings/node/src/
sha256:0c288ea5db6f2fcf4f9fb37e46472b5d27ace113310440b9681ad745abe8462e tokenizers-0.20.3/bindings/node/src/arc_rwlock_serde.rs
sha256:478285b75a75c90832a4461315cc4cd8c77a8989db74f81f2d0e8f0df6cbe5f6 tokenizers-0.20.3/bindings/node/src/decoders.rs
sha256:03cbe893bb55b4609205d94627e77e60b54dd7ef751a9291efac35265bbc2018 tokenizers-0.20.3/bindings/node/src/encoding.rs
sha256:23e01dcea44b1fc841dc18ad525f6fd5c5e4a94843c1368e802de32bb06e90ad tokenizers-0.20.3/bindings/node/src/lib.rs
sha256:2dc0b75569ede4063f503211cde250abab990460afb495abc79393392f655def tokenizers-0.20.3/bindings/node/src/models.rs
sha256:f7dc43ad7c794ffe23fc29ef4dfa6e3df50fe5f1145c23d7c3abac332a78ba57 tokenizers-0.20.3/bindings/node/src/normalizers.rs
sha256:d54f97b5a1dbe3d539185c1384c158ac300f4dba77a9ee0fe8891bfad9e7c9f2 tokenizers-0.20.3/bindings/node/src/pre_tokenizers.rs
sha256:27c5d78c318c9f15ca440715fc5297b79655fb0b787fc1e1eb9cc5a5dca7ccb3 tokenizers-0.20.3/bindings/node/src/processors.rs
tokenizers-0.20.3/bindings/node/src/tasks/
sha256:97302dbce5e72f37cf4a74de57e17084a6ef98f41a9cc832348842d8ae9e4de1 tokenizers-0.20.3/bindings/node/src/tasks/mod.rs
sha256:7cb04541da8f4b29346caacf41dcfe873b6f9ffab4f4756f14710d3f06025f05 tokenizers-0.20.3/bindings/node/src/tasks/models.rs
sha256:06de8bd57a41b18dce7b169f3798d6c3dadcf361a9006cfcfb792cdbdf9ee4c1 tokenizers-0.20.3/bindings/node/src/tasks/tokenizer.rs
sha256:b50650707bca5a89b6037d0e9f1a074d708a084b2c701f3735fb102874fe17a6 tokenizers-0.20.3/bindings/node/src/tokenizer.rs
sha256:67a3f03a137bf83c2351f736c68ab5bce13016d7f76f838a1b3461c0c0ef061f tokenizers-0.20.3/bindings/node/src/trainers.rs
sha256:7c7ab6fcdd34f3fc94a6652c1af166e03b886a1b7a7b40366011efd04efdec5f tokenizers-0.20.3/bindings/node/src/utils.rs
sha256:3be08c632028bf9f54b8aa711469f6207127e2bcae5cf88b4b9da18d788a3db4 tokenizers-0.20.3/bindings/node/tsconfig.json
sha256:110ee41ab1de180ecc708c8094a815b5023e62a309d9417ebd2d6cf014298896 tokenizers-0.20.3/bindings/node/types.ts
sha256:c3d48b6639e952c95e86e11700813aa1b474a2efc61b255fb2fd8a51ea267fed tokenizers-0.20.3/bindings/node/yarn.lock
tokenizers-0.20.3/bindings/python/
tokenizers-0.20.3/bindings/python/.cargo/
sha256:0f8f7159b2550efdd76855ec4620c6fd21d85648caa9eb71f41aad19404ee92b tokenizers-0.20.3/bindings/python/.cargo/config.toml
sha256:6667b2d1aab6a00caa5aee5af8ad9f1465e567abf1c209d15727d57b3e8f6e5f tokenizers-0.20.3/bindings/python/.gitignore
sha256:c0778b9c5f36c46141cb570ba45615150476d130fa00a128e1f0331417b361fc tokenizers-0.20.3/bindings/python/CHANGELOG.md
sha256:41ae195f80e1bb41ced42f4a195bbb7874be878c2b8673623d45a0fdac42e596 tokenizers-0.20.3/bindings/python/Cargo.lock
sha256:212322ef5157ac6f3a8198f47b95767a97909006fd067d2e1bfe35edcde91690 tokenizers-0.20.3/bindings/python/Cargo.toml
sha256:b5bba29c1d37582948f7837cd44ca176452f386a8c4d41f2fccd95ab17859abf tokenizers-0.20.3/bindings/python/MANIFEST.in
sha256:8b982270fe04dbf1158d45e5c0bd10045cf03c7d351f78577e0738e2452da12c tokenizers-0.20.3/bindings/python/Makefile
sha256:b228767941e834617aa3a7c02f278edf433ebc34adb9236c9d7f160eab883971 tokenizers-0.20.3/bindings/python/README.md
tokenizers-0.20.3/bindings/python/benches/
sha256:3c1718bb820865e98d14ca25d0e9a95bfab1b2296f2226a489ed8bc272c283c8 tokenizers-0.20.3/bindings/python/benches/test_tiktoken.py
sha256:d5fec928f9fee467cbac5b91750c56ce65a646784a8395225dd6b0016b910845 tokenizers-0.20.3/bindings/python/conftest.py
tokenizers-0.20.3/bindings/python/examples/
sha256:0b23ff83ffb7e7d429e2a1295140f91bb7ee00f40ac46ad1a5819cced6e4d54e tokenizers-0.20.3/bindings/python/examples/custom_components.py
sha256:7259e3f692c90639d339931d594eeabb878b8cc8e031e960430946f1dcc2fca5 tokenizers-0.20.3/bindings/python/examples/example.py
sha256:58ebba107e08b4c45703c9ba9aad14d143142c8592949e4903ae8e06c830421e tokenizers-0.20.3/bindings/python/examples/train_bert_wordpiece.py
sha256:ec2d5897b2646b8017acbbfa404c212e0142dae05d7df0cf4a3c43e4d0b68d26 tokenizers-0.20.3/bindings/python/examples/train_bytelevel_bpe.py
sha256:31ec2b613b7114b20dc83c198130cfea07e3cfcb6d7e98527cc9dcb1a343ef87 tokenizers-0.20.3/bindings/python/examples/train_with_datasets.py
sha256:26f2b0631f6168d440caba51c5b3c0f037a9e3f2b9cea167c928c8b60474cead tokenizers-0.20.3/bindings/python/examples/using_the_visualizer.ipynb
tokenizers-0.20.3/bindings/python/py_src/
tokenizers-0.20.3/bindings/python/py_src/tokenizers/
sha256:644e596a052fa1b05272b1c141d1286e1c78c2a3346ecabd17d68b62404d8d84 tokenizers-0.20.3/bindings/python/py_src/tokenizers/__init__.py
sha256:8f0df85995da62ef0d041276fdcca97cea8963123b09728fce5bde8ac5f0e174 tokenizers-0.20.3/bindings/python/py_src/tokenizers/__init__.pyi
tokenizers-0.20.3/bindings/python/py_src/tokenizers/decoders/
sha256:946a77da1f2a7ab134178f20c992fcc0699e4159668d5a5e22c45bd5233d91fe tokenizers-0.20.3/bindings/python/py_src/tokenizers/decoders/__init__.py
sha256:c6c45ea3b385442890e1b05963da2061bd622c9e434e02397a5341f94820a1cb tokenizers-0.20.3/bindings/python/py_src/tokenizers/decoders/__init__.pyi
tokenizers-0.20.3/bindings/python/py_src/tokenizers/implementations/
sha256:57302ca65688a3bae5e0014ef0c8aeee283b31f663be89f055b959c34d7347a3 tokenizers-0.20.3/bindings/python/py_src/tokenizers/implementations/__init__.py
sha256:d9315984bba968988c0d8189b94366c5826ff9c9d1f1b0c799b333698a4544eb tokenizers-0.20.3/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
sha256:b0a0ae9b414a3d875280914937c2c37ab541a130d14aacaa49dadc9be96f42a2 tokenizers-0.20.3/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
sha256:380fe3cb2dc44266136ba8677fe10ac0b385bb2ae8a8560e273db9cac3360549 tokenizers-0.20.3/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
sha256:436644016d313101c5ed8094b66a65c1ac5b53e2743f634ae0f24633151bfbf7 tokenizers-0.20.3/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
sha256:2f0ae87e8a219d47cc13694ada5418a3240884ff3844fd022251d16a3d21c8db tokenizers-0.20.3/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
sha256:4988955cbf19b6a2d72a9baac27c26adfc601a8b6ef320243aeedd2f3b445c8a tokenizers-0.20.3/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
tokenizers-0.20.3/bindings/python/py_src/tokenizers/models/
sha256:7896781d3010669c672882cdca559a4c5ab15f2f816ba38ab3058de3b7de795f tokenizers-0.20.3/bindings/python/py_src/tokenizers/models/__init__.py
sha256:7253d3c22ca3cfb16555d12ec28ff759afd3990adb6615b44869a737295ea676 tokenizers-0.20.3/bindings/python/py_src/tokenizers/models/__init__.pyi
tokenizers-0.20.3/bindings/python/py_src/tokenizers/normalizers/
sha256:ff4eb0e1ca9122dbde12021d75868b3127209123a4200308cd809eb1be400385 tokenizers-0.20.3/bindings/python/py_src/tokenizers/normalizers/__init__.py
sha256:7707d5b2f83461b798a006814a62ac226a8bfadcb1883c8769a4c5b192b8699c tokenizers-0.20.3/bindings/python/py_src/tokenizers/normalizers/__init__.pyi
tokenizers-0.20.3/bindings/python/py_src/tokenizers/pre_tokenizers/
sha256:c1de8a61003f46c19240af87786f68a53461bf8b6d4919329e8d9d93a6b3f8f3 tokenizers-0.20.3/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py
sha256:74bb5ac4e81c05af39bd00ba6f2bdf28608e593122e1ce3621caa299f6ad92c4 tokenizers-0.20.3/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
tokenizers-0.20.3/bindings/python/py_src/tokenizers/processors/
sha256:c4cd8310ac0ab4722e987b2ccccf00324abe0256aabc16455d680b53c48d84e6 tokenizers-0.20.3/bindings/python/py_src/tokenizers/processors/__init__.py
sha256:871efaed963c4878716ff8625cf4719be7ff29ca11e170f1eef7cad9cd2547e4 tokenizers-0.20.3/bindings/python/py_src/tokenizers/processors/__init__.pyi
tokenizers-0.20.3/bindings/python/py_src/tokenizers/tools/
sha256:c46f1c681f4e1c2f1c6c1d354b9bd8575e0767184ee9e59b2de86c6fbd29a62a tokenizers-0.20.3/bindings/python/py_src/tokenizers/tools/__init__.py
sha256:cc0c9dab5a06583f10125978f9ec8bf0b970d01d6cb72fe1a48137b58c4bd369 tokenizers-0.20.3/bindings/python/py_src/tokenizers/tools/visualizer-styles.css
sha256:822f84d8d08fec5b86eae8e940e75a9524d7525695f3957a348f993cf4ef03fe tokenizers-0.20.3/bindings/python/py_src/tokenizers/tools/visualizer.py
tokenizers-0.20.3/bindings/python/py_src/tokenizers/trainers/
sha256:513bb6d8019ca7be88be95b8e712d16c95844f4e0dc4f5ba35f09bd98633d043 tokenizers-0.20.3/bindings/python/py_src/tokenizers/trainers/__init__.py
sha256:dd3c052adb3899eef341f5517124e6b576223f85f07118df018b70aa8655b684 tokenizers-0.20.3/bindings/python/py_src/tokenizers/trainers/__init__.pyi
sha256:82a7c285b73bfe65fa74f29188151d6e2e520b8e1457efcdc8d2d50277b18212 tokenizers-0.20.3/bindings/python/pyproject.toml
sha256:2b92ea252be0fbc26f70317cdaa7b6411ea634b50d55338cd8c495e4dbf25d1d tokenizers-0.20.3/bindings/python/rust-toolchain
tokenizers-0.20.3/bindings/python/scripts/
sha256:4175726b76ac3ea2ce39bf4661f5e26e9d14e9dcefea5c6e4afc91f01710bb0e tokenizers-0.20.3/bindings/python/scripts/convert.py
sha256:68c16a7e0e00b540cc8a74c49823fd1a0b0f5de7b595f3426ba90d4bafd7d9ad tokenizers-0.20.3/bindings/python/scripts/sentencepiece_extractor.py
sha256:85726a5fe68222c61ddaf1a4dc4bfa59bfdfd9a5da6e1fe5dd76064399a92112 tokenizers-0.20.3/bindings/python/scripts/spm_parity_check.py
sha256:32e7d02fa00cb911b747f419942188de07ebd841aa89bde7ce9c256173ffd696 tokenizers-0.20.3/bindings/python/setup.cfg
tokenizers-0.20.3/bindings/python/src/
sha256:f0fafec9ef3a8852069082481f412627ec2bd35f5bc2a0ccb77f5004e7160c1d tokenizers-0.20.3/bindings/python/src/decoders.rs
sha256:1fbd8477eddef8330c5f50974edb1593dc07f9e127f6f493f77e1c7250b41c8e tokenizers-0.20.3/bindings/python/src/encoding.rs
sha256:293562d81f0774b14112669fc58375eef154968ee32f61e0a276d0adeb277602 tokenizers-0.20.3/bindings/python/src/error.rs
sha256:529e77102e60fa3680beb66f0f2f213b5e32402ef71cc32535865b446cfa0920 tokenizers-0.20.3/bindings/python/src/lib.rs
sha256:4c57549f06c6590384a845dfb20eaa35f3b9c4adfd5153ed88288689cb6405a9 tokenizers-0.20.3/bindings/python/src/models.rs
sha256:ba37a4b00e03cf188ddbc2eb718df9c29da5d4e48e69208493c67fb2bff9a380 tokenizers-0.20.3/bindings/python/src/normalizers.rs
sha256:650088d8fe8b1d31711693ccfdaaa2e761e49cdf76a848f5ab638de25eeaab67 tokenizers-0.20.3/bindings/python/src/pre_tokenizers.rs
sha256:cd58aff14eda161784d7296564314bd15b11ef464eec216f69e033fbae99ab37 tokenizers-0.20.3/bindings/python/src/processors.rs
sha256:9e49b34d599baa0135cb2f9ba0e380fff813eeb5bf96b29976dac5f7fb68f1a6 tokenizers-0.20.3/bindings/python/src/token.rs
sha256:57ab24cc654c2bcaf8dc6eef6f83d115bbdd1d21cbcd11505a62f5d2165fa4f5 tokenizers-0.20.3/bindings/python/src/tokenizer.rs
sha256:412b6db0f152bc597c9a4eab344a5ce93ca706ca335c0b3565b1f8160dab063e tokenizers-0.20.3/bindings/python/src/trainers.rs
tokenizers-0.20.3/bindings/python/src/utils/
sha256:67bd9d57c12adeb0bdbfd986761a1dbeaba996c28f6d90dbb95280ee775443db tokenizers-0.20.3/bindings/python/src/utils/iterators.rs
sha256:f10237229107d96fc7af97cf93b4528b68960c793ba256c6b2987c2390c7fa76 tokenizers-0.20.3/bindings/python/src/utils/mod.rs
sha256:4772af24de5869db58c9befd8cf49dcbb63feeed1f9fc8da982abb389cbe98a2 tokenizers-0.20.3/bindings/python/src/utils/normalization.rs
sha256:88302c8952a4a203bbfac1bba13a7072ed5c6e4f75f8b96356885b93a4aa7fd8 tokenizers-0.20.3/bindings/python/src/utils/pretokenization.rs
sha256:9d98c86f47aad177f85b1ebe2dec22443ae45b0972c8ee489d40842e7e011ab8 tokenizers-0.20.3/bindings/python/src/utils/regex.rs
sha256:9dfb15fcb41cdd0e465500a67469b6be4e0583e1d4a0c3476338d401db072dad tokenizers-0.20.3/bindings/python/src/utils/serde_pyo3.rs
sha256:e22f20243d93b22e2b0a8893291fc2498848e9eb16ce79640d4ab9157d7ae0fe tokenizers-0.20.3/bindings/python/stub.py
sha256:70e03c19780c5e7c1f31580930d36f0aa4a64e5fc8e601dd50b74b301729b3ee tokenizers-0.20.3/bindings/python/test.txt
tokenizers-0.20.3/bindings/python/tests/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.20.3/bindings/python/tests/__init__.py
tokenizers-0.20.3/bindings/python/tests/bindings/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.20.3/bindings/python/tests/bindings/__init__.py
sha256:7493e7592df728bb99d33f3074b327a944d4bca2a76210112e014f093beebc75 tokenizers-0.20.3/bindings/python/tests/bindings/test_decoders.py
sha256:5f4b218bbfdaa27751080e4451b8fd4d31fda7c41d661da90c604dbd00339b47 tokenizers-0.20.3/bindings/python/tests/bindings/test_encoding.py
sha256:a816aa009317234855304ed576b804a15035d2efa39e944497212f78adc4d17d tokenizers-0.20.3/bindings/python/tests/bindings/test_models.py
sha256:18e2d28d3853211c4999702fec14cb21ed48695c5b87c92d5b9f80d845e4b059 tokenizers-0.20.3/bindings/python/tests/bindings/test_normalizers.py
sha256:553ac2d6f4333b0faa159c214c9ace503ad3a6d527b204d34ed7011143b00452 tokenizers-0.20.3/bindings/python/tests/bindings/test_pre_tokenizers.py
sha256:ed828e36350ed49c27c4d023371f0ee97defc510d08f0919fcc616a0bb21cb23 tokenizers-0.20.3/bindings/python/tests/bindings/test_processors.py
sha256:236ea260ac612007353c44265482657515627017033d1cb119cb5c35d4c7e693 tokenizers-0.20.3/bindings/python/tests/bindings/test_tokenizer.py
sha256:ff322fca83fe9070d98dc035389f3ce51332a565309c4ade497a52beacda4729 tokenizers-0.20.3/bindings/python/tests/bindings/test_trainers.py
tokenizers-0.20.3/bindings/python/tests/documentation/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.20.3/bindings/python/tests/documentation/__init__.py
sha256:3daf9785f1f14c8f0deccd58dd62fa91f735e54ce2677181d4efbb278dbee4cb tokenizers-0.20.3/bindings/python/tests/documentation/test_pipeline.py
sha256:897010ff492659bedccb321c1eb65006e5ea5e789f0973e34e012f02ae967690 tokenizers-0.20.3/bindings/python/tests/documentation/test_quicktour.py
sha256:ada99dd658322f83f5beee7475a65f686afc234c277d5f09812280b8945efe2b tokenizers-0.20.3/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py
tokenizers-0.20.3/bindings/python/tests/implementations/
sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tokenizers-0.20.3/bindings/python/tests/implementations/__init__.py
sha256:43a84bea7130ba6b1773f3aba409ffd395e4a3b632c0ebede89d96a8a30c75b2 tokenizers-0.20.3/bindings/python/tests/implementations/test_base_tokenizer.py
sha256:51cab1df7f9f4693b143f670b2aa1b3c5fc01dc609b18f1127f9e8d07743794a tokenizers-0.20.3/bindings/python/tests/implementations/test_bert_wordpiece.py
sha256:24683c9bab55fe699f4887b3fbfe5b4990b3ad815fa90ef256380822cd60f237 tokenizers-0.20.3/bindings/python/tests/implementations/test_byte_level_bpe.py
sha256:f6cb83befa8ee6d7ea2cfbb105d0fdbb0fd8fd5cba045b4f1a5f523eddeb193d tokenizers-0.20.3/bindings/python/tests/implementations/test_char_bpe.py
sha256:9f54e6ed6d8856c4ec0a3af33936e26f8e3a402b141d2e69397fd02c27564b0c tokenizers-0.20.3/bindings/python/tests/implementations/test_sentencepiece.py
sha256:9f26a1151cd0440d9f4a6f523c69187c2e6157783410ac6f3b5321ba884fd756 tokenizers-0.20.3/bindings/python/tests/test_serialization.py
sha256:b4e4576d427cf2c8b7b634c402ed1c8f45cbb030d1a4e30841502ec69f369b0d tokenizers-0.20.3/bindings/python/tests/utils.py
tokenizers-0.20.3/docs/
sha256:64da2857b2b8b721109ede3b1b8c1469281129af0c45c2213b17916c1c1efd73 tokenizers-0.20.3/docs/Makefile
sha256:9a5efb500e56a297d44431026a0911a85440854ad8be8d80a955eb76f2be361b tokenizers-0.20.3/docs/README.md
tokenizers-0.20.3/docs/source-doc-builder/
sha256:8c9e3c6b89eae2b85b98ee7fd27341be9495d601d0778b8db0bb06c4f9a413bc tokenizers-0.20.3/docs/source-doc-builder/_toctree.yml
tokenizers-0.20.3/docs/source-doc-builder/api/
sha256:6c80b2c6fafea22cc8210e8c38fc8a22934e532f8b5d7057ccfb9da03794c0c5 tokenizers-0.20.3/docs/source-doc-builder/api/added-tokens.mdx
sha256:aa600d7ecf65f56ad2c6fb115950c5e421d360e09a354e0722bbb59fbb92d504 tokenizers-0.20.3/docs/source-doc-builder/api/decoders.mdx
sha256:af65169fdb9a777f17c7ac74dc47af1308882c669ba14309bdb13151eb534ab9 tokenizers-0.20.3/docs/source-doc-builder/api/encode-inputs.mdx
sha256:7ab4617e68210c301a5cec7652fe499351b52b22775c14e19d51ae09f6e9852d tokenizers-0.20.3/docs/source-doc-builder/api/encoding.mdx
sha256:c26868132cce48ddfcdbc287e7ec54af20e79fc54942211eb1e54de329a2a224 tokenizers-0.20.3/docs/source-doc-builder/api/input-sequences.mdx
sha256:0fa43188b0569e2452ce4a2cdad61c4d966653f1a5b016e6e34dee691360c74e tokenizers-0.20.3/docs/source-doc-builder/api/models.mdx
sha256:5b8e0dfb7ac14288d96825dfd0cb60ec0265ae42dea19f59a30b209a210e72ff tokenizers-0.20.3/docs/source-doc-builder/api/normalizers.mdx
sha256:af1dcc0d43802ed5aac6fdc271c99312b0741685712738e11884240b2df17274 tokenizers-0.20.3/docs/source-doc-builder/api/post-processors.mdx
sha256:7f6172e90e8768ab13eff60a68486c3a67ec0b18ab61029f263f71e678f881ad tokenizers-0.20.3/docs/source-doc-builder/api/pre-tokenizers.mdx
sha256:33c9d09bcc5dd3f5775f8521ad5e0665f357873c389615aa260bea4ed7eafb39 tokenizers-0.20.3/docs/source-doc-builder/api/tokenizer.mdx
sha256:743339cf86db82446d01a4afbbe24b86c1fd42998c76948bb7f7fa5792208829 tokenizers-0.20.3/docs/source-doc-builder/api/trainers.mdx
sha256:3b08ec9077407db0076bf25b9efc271473b3917264f77348ee40dc7dca4b0ec3 tokenizers-0.20.3/docs/source-doc-builder/api/visualizer.mdx
sha256:a94c55d07b6b966d7d930d64a4ed7b2c8d13a5f066cfc9744f9849d3cff6f4fe tokenizers-0.20.3/docs/source-doc-builder/components.mdx
sha256:bf6a3a26a19f0d90bdc24fea638c8b4f12956ac23718e20a425ad20fdf9f537c tokenizers-0.20.3/docs/source-doc-builder/index.mdx
sha256:675c62857dfedbbf779a67b184c1bc71504cd051aa2e5c86c8e68ef8a87350d0 tokenizers-0.20.3/docs/source-doc-builder/installation.mdx
sha256:bf32ea9c2049df10e000962f381453fbe96a51cbb2e0cfccf42c30d557a06220 tokenizers-0.20.3/docs/source-doc-builder/pipeline.mdx
sha256:c719c152095b152e9dd8100f85f0ac8b9b62e43a5d773d4ef68c82f8fd099010 tokenizers-0.20.3/docs/source-doc-builder/quicktour.mdx
sha256:1bd0aeccbcc56d57ad00a272f46773d262f6ce24827cf02227e00354a1779eb4 tokenizers-0.20.3/docs/source-doc-builder/training_from_memory.mdx
tokenizers-0.20.3/docs/source/
tokenizers-0.20.3/docs/source/_ext/
sha256:18d280dbaa0a211a5745e18b271ca6f20b5133a1dbb59ef412cc24c032301939 tokenizers-0.20.3/docs/source/_ext/entities.py
sha256:f6b1fd1f777b23885ed350bd0dea306b7ff6bfabcaae067f56f7ebb7847ef895 tokenizers-0.20.3/docs/source/_ext/rust_doc.py
sha256:d1079093942989915b32985c10183b83445e305370c1aeba026b0bcdeb8644fd tokenizers-0.20.3/docs/source/_ext/toctree_tags.py
tokenizers-0.20.3/docs/source/_static/
tokenizers-0.20.3/docs/source/_static/css/
sha256:d68e6fef383ffcee718a5728690415a204fddda208cd49a2df4931257bb3c476 tokenizers-0.20.3/docs/source/_static/css/Calibre-Light.ttf
sha256:2cc588a746277a0e8a4bc64b95b63aed4ce60061f43cef06d44899b9d0395eec tokenizers-0.20.3/docs/source/_static/css/Calibre-Medium.otf
sha256:8f3af66227ee366466149c568be461f579bac53e411a39acc33583c7b310dfef tokenizers-0.20.3/docs/source/_static/css/Calibre-Regular.otf
sha256:83148cb95241197bcbb93352458548c630dfb707e822f07d2f0f03a3f61b0870 tokenizers-0.20.3/docs/source/_static/css/Calibre-Thin.otf
sha256:a6826e671da01a2fa5a0409e4f9f2edbcf1d3059a988548dd439774906ec448b tokenizers-0.20.3/docs/source/_static/css/code-snippets.css
sha256:c712f6e603226ead1b3a08733599979d1fbb7c063cd14223b65ed9faed78cad7 tokenizers-0.20.3/docs/source/_static/css/huggingface.css
tokenizers-0.20.3/docs/source/_static/js/
sha256:82a457d4b85c2d1a9c8af4de785a98acba9bce5e0e5c27cba823c72cef3d5acd tokenizers-0.20.3/docs/source/_static/js/custom.js
tokenizers-0.20.3/docs/source/api/
sha256:ff2cc6e66aba30b79ec0aadd3e2b214095d47ef1f7c1d7dac701cfc056560f24 tokenizers-0.20.3/docs/source/api/node.inc
sha256:591900f42ad20d8cd396a5fb77cc440be3a60e698b61ae3babc818e662e0c68a tokenizers-0.20.3/docs/source/api/python.inc
sha256:c21848d284baef80f8c8ecfc057648c509e91b400fde506304da004e2f22cd34 tokenizers-0.20.3/docs/source/api/reference.rst
sha256:8253c97c3382c969f82ac09ce2854f40cbe530f97ac03d2404abfe5141e31dc3 tokenizers-0.20.3/docs/source/api/rust.inc
sha256:d100fbadb3a88f75476af2acc0fd4ba4d90d59e5cfa919f31293b24d1656ea27 tokenizers-0.20.3/docs/source/components.rst
sha256:93c5dafd101322d8f98c26943744735f99b5f67d93e809dd721b4ebf90371310 tokenizers-0.20.3/docs/source/conf.py
sha256:eb7dfb878646f949ce8e2eae062af4768a79e9afcb200fa124f50201570059e8 tokenizers-0.20.3/docs/source/entities.inc
sha256:3a6453b0107cf0421a04d046a09b736cc248a4ffa847e9e3a7325854455606f7 tokenizers-0.20.3/docs/source/index.rst
tokenizers-0.20.3/docs/source/installation/
sha256:7d7d111f50cec5ea6f39b7b35db11688e4cc0c8acba096e27ff1d053506244d5 tokenizers-0.20.3/docs/source/installation/main.rst
sha256:6590d59131ae3dc72cf0e88106b9a75fa4fbf4ddf4cc713c19536832e13bbaeb tokenizers-0.20.3/docs/source/installation/node.inc
sha256:f8e6a397b54a762200b5adfa6dec0be3ae4d553ace7f570e43052fa6e268e673 tokenizers-0.20.3/docs/source/installation/python.inc
sha256:858c8ab604d54821c8f1b43f9d18638b5c8b703fb098058f3e329f4828e3910a tokenizers-0.20.3/docs/source/installation/rust.inc
sha256:95af1721b255e4b7b4d00ec75a82d80868a0fb771c0bd580d6001976155585dd tokenizers-0.20.3/docs/source/pipeline.rst
sha256:78452aac2be736ee2313929e49b8c342dd1182bc8ad5b75e6d6a1998e1353e08 tokenizers-0.20.3/docs/source/quicktour.rst
tokenizers-0.20.3/docs/source/tutorials/
tokenizers-0.20.3/docs/source/tutorials/python/
sha256:c78ea0da873d0899476945f9a00091bffbc2fc2a6036d5df2a706abf9ba26d4e tokenizers-0.20.3/docs/source/tutorials/python/training_from_memory.rst
tokenizers-0.20.3/tokenizers/
sha256:30abc9fa16351a9c4299828c1edffe5c50342036602e764a0427d6025cd6af31 tokenizers-0.20.3/tokenizers/CHANGELOG.md
sha256:c83937e08408da0fe873738c45b3712073e9ffaef1b703cd4cb922d19725dc8f tokenizers-0.20.3/tokenizers/Cargo.toml
tokenizers-0.20.3/tokenizers/LICENSE -> ../LICENSE
sha256:149699cd22a48e6b227f48350d0e1637343d48cb4a436a89a16834025f607938 tokenizers-0.20.3/tokenizers/Makefile
sha256:f8978980957df4e6067dd511ab4f874f582a2f90f9fc8950412d3854d9297ed3 tokenizers-0.20.3/tokenizers/README.md
sha256:487d57adf5a2cb4729326ce4a0c85e80c965e8951f3b38e6bc3868902519b39f tokenizers-0.20.3/tokenizers/README.tpl
tokenizers-0.20.3/tokenizers/benches/
sha256:62e3df468e2f41c8f3c555164553ac72c2a5e7bc0f530f39d16f833b8ebca2db tokenizers-0.20.3/tokenizers/benches/bert_benchmark.rs
sha256:684ea83441a4bf1249e89d47c1966cc90f8acdea02afef8733a9c6cb8250e873 tokenizers-0.20.3/tokenizers/benches/bpe_benchmark.rs
tokenizers-0.20.3/tokenizers/benches/common/
sha256:2fdeba251f5d18b991b868a5902d8aa6f5eba8837fd62debb3422351c83ac184 tokenizers-0.20.3/tokenizers/benches/common/mod.rs
sha256:bc71182748a4b70f26cf45e4f819adff1bd48654af6f0253d4c70ecd1bef2a6a tokenizers-0.20.3/tokenizers/benches/layout_benchmark.rs
sha256:c137b80a7d5f1f9f08f7b5ee427089e70a06dffd02c18e40e283530137a2a49e tokenizers-0.20.3/tokenizers/benches/llama3.rs
sha256:1c3b8edfc263f509453742306aa839eaa6cc53b30e0fbe2f0260c7e5a4fbf39c tokenizers-0.20.3/tokenizers/benches/unigram_benchmark.rs
tokenizers-0.20.3/tokenizers/examples/
sha256:2bd564996ff859e1c996238f38f576657e566688bbd435c8f100e3bb0cdc43af tokenizers-0.20.3/tokenizers/examples/encode_batch.rs
sha256:8f35712a217fef4bccf538a3bc0c71bb460096ff751169a057a2165bc9ee57e1 tokenizers-0.20.3/tokenizers/examples/serialization.rs
tokenizers-0.20.3/tokenizers/examples/unstable_wasm/
sha256:f5ba0dc3ff1ad2d818ccbe695f554c31822aea61713639e7d5b76d9a98f2ecf5 tokenizers-0.20.3/tokenizers/examples/unstable_wasm/.gitignore
sha256:c0a32c90ef7eda62d0dcadee83f91759d1251297c715555a39586d59df98ac1b tokenizers-0.20.3/tokenizers/examples/unstable_wasm/Cargo.toml
sha256:b83a774dea05794aa3996310404dad4806617358488558a6c96f5d5922269b7b tokenizers-0.20.3/tokenizers/examples/unstable_wasm/README.md
tokenizers-0.20.3/tokenizers/examples/unstable_wasm/src/
sha256:700108192809e07d83d25abd3c5465d0d8c268f9283428c6c38be3d68876fc94 tokenizers-0.20.3/tokenizers/examples/unstable_wasm/src/lib.rs
sha256:4202fe474a0ae76174696d04bc001ae1f838071e1b14ebe9a4913e9efdfade36 tokenizers-0.20.3/tokenizers/examples/unstable_wasm/src/utils.rs
tokenizers-0.20.3/tokenizers/examples/unstable_wasm/tests/
sha256:181970ff03a4a3115a9ead0e90015f0aefef46089c4fa6cd4d113c5d13e64e09 tokenizers-0.20.3/tokenizers/examples/unstable_wasm/tests/web.rs
tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/
tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/.bin/
sha256:2bf3871139ef28ae0d47e9b89ac0bb4b67ed5c794fca8d6092eaa37d9c9230e3 tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/.bin/create-wasm-app.js
sha256:c8015948b6eaefde5eaa3862f2fe5ce0245fcde4d6e66f7441dcdcbbed46e652 tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/.gitignore
sha256:06fc329654011dddca329034f2a0f57650b9e0fd50930509f411d43ad05b12ea tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/.travis.yml
sha256:8173d5c29b4f956d532781d2b86e4e30f83e6b7878dce18c919451d6ba707c90 tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/LICENSE-APACHE
sha256:ef36e00d5e8dd8c6260a46855a1a578ff546916c7624119a60832e5ce5c510e4 tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/LICENSE-MIT
sha256:06d60699af573a691b095bded90a83e5d2464e98c56e36d0d6bd1608c4d45a8f tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/README.md
sha256:5b38ab61e867664a13a5d8af50be6904726da873186752497d54bba0ff857c6a tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/bootstrap.js
sha256:13fb2506ed4b9c6152dca7cabe2ec8d8e6052f2a27642e22d81ae58f75acc98a tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/index.html
sha256:4e7d9c000d676dcaecf56808298b3d3c5efe50a3a386326b5fb96bf7f439b1ca tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/index.js
sha256:b3a6eb06a5d73eec232ef54a329bb448bfefea5adff6013de7eee94353f41ef9 tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/package-lock.json
sha256:a28345c99b3be9287a042467cd5b55be673d5bd8431b391506589ef2667eba9a tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/package.json
sha256:1be82d86fa4bf762b6744e612ea837ceda33640e71ae481e9ecbddebf9e35fe6 tokenizers-0.20.3/tokenizers/examples/unstable_wasm/www/webpack.config.js
sha256:2b92ea252be0fbc26f70317cdaa7b6411ea634b50d55338cd8c495e4dbf25d1d tokenizers-0.20.3/tokenizers/rust-toolchain
tokenizers-0.20.3/tokenizers/src/
tokenizers-0.20.3/tokenizers/src/decoders/
sha256:9c363604a82b2e1439afaa78efe6b11c2d33348525bb934c797a37f2007bf3bb tokenizers-0.20.3/tokenizers/src/decoders/bpe.rs
sha256:58bd61910667e4aac6700d420adb038653199e3cad7944cf8abb950046314ca6 tokenizers-0.20.3/tokenizers/src/decoders/byte_fallback.rs
sha256:c091493ea218c9543c2258ab0354458f986040f61bf3112867932b506d882999 tokenizers-0.20.3/tokenizers/src/decoders/ctc.rs
sha256:19b7bfca165bdee6ab084ed5429a49afc9d52664c51187a500335a80707850fa tokenizers-0.20.3/tokenizers/src/decoders/fuse.rs
sha256:9963dedcd9a965ae80888e72a7d83fbe2ee1c4f426b313cf1b7014c384419a08 tokenizers-0.20.3/tokenizers/src/decoders/mod.rs
sha256:6bbaf543e2ac97e658bd657ccdb56b8930218c9381492e712d4c93131b663eca tokenizers-0.20.3/tokenizers/src/decoders/sequence.rs
sha256:82a317f29d62f2cbbc68bf993db2a5c6d41deb9c7622da12fcf37eb14d8ed36a tokenizers-0.20.3/tokenizers/src/decoders/strip.rs
sha256:de59951937e6adc45b6f549379c63fcae9d961d624bf3c274cd0bc17f4e9b55f tokenizers-0.20.3/tokenizers/src/decoders/wordpiece.rs
sha256:b6b8e2d214ead8d524e462266fc1f6a62e7c805ebabc73a1cb4742fcd066449a tokenizers-0.20.3/tokenizers/src/lib.rs
tokenizers-0.20.3/tokenizers/src/models/
tokenizers-0.20.3/tokenizers/src/models/bpe/
sha256:bd38f20a24f390ea319148bc494010319cb613948edf42e1acd817b5801e6d8f tokenizers-0.20.3/tokenizers/src/models/bpe/mod.rs
sha256:5de418cae49a07840821df5c841249c2832076fbbad99dfc6a45cf8b8d051032 tokenizers-0.20.3/tokenizers/src/models/bpe/model.rs
sha256:da2d119b8dc1352ceef158ef040403b43b60b412ff6fb9ebf6d647b04f8e7fd4 tokenizers-0.20.3/tokenizers/src/models/bpe/serialization.rs
sha256:e0dbba6681680706677b5d2ae3a8e0fef48456a7a4aaf010c78670b36dd327e6 tokenizers-0.20.3/tokenizers/src/models/bpe/trainer.rs
sha256:90c8014d1691eed20abb99cd385777344e6c556443a0005647af795aee7f43c3 tokenizers-0.20.3/tokenizers/src/models/bpe/word.rs
sha256:afe52d85f18b25d73a2aa9e9b3e644b990d74a0267678d8b780caf37bb2fe517 tokenizers-0.20.3/tokenizers/src/models/mod.rs
tokenizers-0.20.3/tokenizers/src/models/unigram/
sha256:d0952d809c182b07df13114961dfd068eabd53e82a6e6a5a4fc5fb4a2cf72a88 tokenizers-0.20.3/tokenizers/src/models/unigram/lattice.rs
sha256:cf317d99ceebe2b4326237aa8f8bfdc2293a5dd0a7fc69a6e9e0028b804a000e tokenizers-0.20.3/tokenizers/src/models/unigram/mod.rs
sha256:d4b147f6fc9d50425739207471583567385e40d3478bbbcbb4aa4d0c902ab19d tokenizers-0.20.3/tokenizers/src/models/unigram/model.rs
sha256:2358d3985ae5330b2caebdbd5567621572368869dcbb36e3a95696325e941ce3 tokenizers-0.20.3/tokenizers/src/models/unigram/serialization.rs
sha256:7125823b970c874d2187982dab8a11ec74c85398e9e8c0d13438cbdf813b19b8 tokenizers-0.20.3/tokenizers/src/models/unigram/trainer.rs
sha256:304f7f387d7be43bd01e72b05a950f0a41d95ff51b219f0b123d46b84f70f30e tokenizers-0.20.3/tokenizers/src/models/unigram/trie.rs
tokenizers-0.20.3/tokenizers/src/models/wordlevel/
sha256:bb3382e4eb8087de2dbaf1e24f3b5b95afc8e2105fc9a456ada43cd93e575036 tokenizers-0.20.3/tokenizers/src/models/wordlevel/mod.rs
sha256:1af61e4f303e4e849162f0a50744d592e2a0ef3fcaee202b474ebaaeaeb6b453 tokenizers-0.20.3/tokenizers/src/models/wordlevel/serialization.rs
sha256:11fa23268c8f145a2e18e6a3a7da3519632ce72a9be7624ad950e8fa76fdb528 tokenizers-0.20.3/tokenizers/src/models/wordlevel/trainer.rs
tokenizers-0.20.3/tokenizers/src/models/wordpiece/
sha256:5f83214247ccaac4db9ad86d71cdeb28c37faf73d35f03971c804daf12ebae5d tokenizers-0.20.3/tokenizers/src/models/wordpiece/mod.rs
sha256:188ece9ae1641994bb5145a2ee600fc9b4d616ca3c0b9792b6c740b8348b0569 tokenizers-0.20.3/tokenizers/src/models/wordpiece/serialization.rs
sha256:4aa1ffd863a4be04dce17c8941ea68e3b4b4d257317593aacec3282aef698888 tokenizers-0.20.3/tokenizers/src/models/wordpiece/trainer.rs
tokenizers-0.20.3/tokenizers/src/normalizers/
sha256:a938fdac02447e33393d98edcaad015f58e2095d633521cee85a2b60ccd995bd tokenizers-0.20.3/tokenizers/src/normalizers/bert.rs
sha256:699e75ad10cd48e8416d93cb56cf0cd866b50043119574d5052934ecd808ed22 tokenizers-0.20.3/tokenizers/src/normalizers/byte_level.rs
sha256:145809163e0bd0161093f88e5cc316eea5407f2ecc1480530c5b77f96608ff6b tokenizers-0.20.3/tokenizers/src/normalizers/mod.rs
sha256:69acb2a2be78760c59a39a2c1f7db6469c75708900b53a7f9bbe189ed83aa6da tokenizers-0.20.3/tokenizers/src/normalizers/precompiled.rs
sha256:99d7336d9c9ba8baca0f30b435c2a7930332100ec0d58f6221eff94da373fa19 tokenizers-0.20.3/tokenizers/src/normalizers/prepend.rs
sha256:2f4566ee4f7b108559cc0946a6e4c91866d0bb98de67e61d139cfce8e1045cb0 tokenizers-0.20.3/tokenizers/src/normalizers/replace.rs
sha256:3d088593a2fa931261f4cd906440b83c053351a0e0e12cb797e541b98d7e2b06 tokenizers-0.20.3/tokenizers/src/normalizers/strip.rs
sha256:6c2312c8a2ba9461b850147fd63432f59f81e942042995d7c28e097f8198139e tokenizers-0.20.3/tokenizers/src/normalizers/unicode.rs
sha256:4b09adb7232537923d55352f2b0707f01c14929117d28fc34dd4b13a70049328 tokenizers-0.20.3/tokenizers/src/normalizers/utils.rs
tokenizers-0.20.3/tokenizers/src/pre_tokenizers/
sha256:32dd35c1f815072c6ba32df390389d10e5f4073b480d1c4ade06143ba23b571d tokenizers-0.20.3/tokenizers/src/pre_tokenizers/bert.rs
sha256:5c962ea0b56d70ed464b96b88c4bba7badc1872a3d3b3cf084140e21d5cf8f38 tokenizers-0.20.3/tokenizers/src/pre_tokenizers/byte_level.rs
sha256:7d2d0170eeb5d864742df867958f379ec564bc5af67e835f5793b06bfab4185f tokenizers-0.20.3/tokenizers/src/pre_tokenizers/delimiter.rs
sha256:1452f56c9fcb0265ee91c5439f860033209e44fe61565b743970d64216b9b41a tokenizers-0.20.3/tokenizers/src/pre_tokenizers/digits.rs
sha256:e3fe6724cf38f700c8ae549c06889ae82d889c25b2975f8b6cec636ec81a26e6 tokenizers-0.20.3/tokenizers/src/pre_tokenizers/metaspace.rs
sha256:5f0383e59902675f7c1d7ff446249665ce99414a9d28fad0f613c03143d6f468 tokenizers-0.20.3/tokenizers/src/pre_tokenizers/mod.rs
sha256:cfb82a699365f2bad1fe36be7ec5523276c90d4cfcf86e8e985e0ba0de6fd9e3 tokenizers-0.20.3/tokenizers/src/pre_tokenizers/punctuation.rs
sha256:1742957bb2cbe971fa791dac41b9e5fad5bc9843cb4eaa0e8e604f6cdc75e1df tokenizers-0.20.3/tokenizers/src/pre_tokenizers/sequence.rs
sha256:d12ee03b7a4d8006979906883ed3d993132cf2b9503f31ebfb0b0d376020d220 tokenizers-0.20.3/tokenizers/src/pre_tokenizers/split.rs
tokenizers-0.20.3/tokenizers/src/pre_tokenizers/unicode_scripts/
sha256:5740e30dba4e6e819309a24144c95d228d9aca437950fc0285b134f238f1788e tokenizers-0.20.3/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs
sha256:4ed0d408409ec35bcba141d75a2647d7503bc17d1e6f4641627325c99bd56edb tokenizers-0.20.3/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
sha256:c614fe1c9a79b1e4cbbb9f3cc3ad130dd6506e12ff74688d611a8eeff47c2167 tokenizers-0.20.3/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs
sha256:0d0d8284ab86d9b0f06b0b0f7024d50618f74de2b1c2ffc2877032a587a0753f tokenizers-0.20.3/tokenizers/src/pre_tokenizers/whitespace.rs
tokenizers-0.20.3/tokenizers/src/processors/
sha256:db17c4467ab2cb95db5dbe22d3abd6ca61d0c463537c49f40896a148bcf37ec1 tokenizers-0.20.3/tokenizers/src/processors/bert.rs
sha256:1dfcff4557f06c4306cc353846ff68775cceebd774e507108a5aafc81037eed5 tokenizers-0.20.3/tokenizers/src/processors/mod.rs
sha256:713d3ade0492cfcd9d1aecb269a5cc8aacf321a716b13fe2dfbfc7b023c0a67a tokenizers-0.20.3/tokenizers/src/processors/roberta.rs
sha256:96bfe2c23419a88b270bb3d6edb34d9e61e12b9e50cc8cd8b852a48e81a2c18e tokenizers-0.20.3/tokenizers/src/processors/sequence.rs
sha256:61e503e8c5e55ccdf71c6f1c708a777404518670b4d9ee9a7c5e5ac4423bc2ec tokenizers-0.20.3/tokenizers/src/processors/template.rs
tokenizers-0.20.3/tokenizers/src/tokenizer/
sha256:925958a7989906624616218ccd5d63cd685c16cceebaa9da8974a7deea594b31 tokenizers-0.20.3/tokenizers/src/tokenizer/added_vocabulary.rs
sha256:deb947238b2b93e8dfc31c00d44c8ceb5da7ab81e6733ab5b9c9ff62aa42577e tokenizers-0.20.3/tokenizers/src/tokenizer/encoding.rs
sha256:6e038515b55bf8280a71f88da4fc546bfeea569bda796896e957fabae413f268 tokenizers-0.20.3/tokenizers/src/tokenizer/mod.rs
sha256:3ae957303d9ce77b07e283b32a4fe76b959de1ebc6a61f0e16908d935d9fe5dd tokenizers-0.20.3/tokenizers/src/tokenizer/normalizer.rs
sha256:036af136c7991371dd30571d13875c13c775e007dbd270cb5efdb128aba7d6a4 tokenizers-0.20.3/tokenizers/src/tokenizer/pattern.rs
sha256:2050974242070d86f2492277d4a37892a25afc9d3d60215e6fecfac6516440e7 tokenizers-0.20.3/tokenizers/src/tokenizer/pre_tokenizer.rs
sha256:b5ce8abfe70dbafb5e79961ba16de382b3e208eeb41779245d483f0cbcc2d066 tokenizers-0.20.3/tokenizers/src/tokenizer/serialization.rs
tokenizers-0.20.3/tokenizers/src/utils/
sha256:3b5fb562f3de1822fda832bd9c7a8c17f5374cc36d28da1d36c68c362a76c643 tokenizers-0.20.3/tokenizers/src/utils/cache.rs
sha256:e8ed67303c94b141c027de9e440dd8391c4a1a39ba5a170f51768db8d859d2d3 tokenizers-0.20.3/tokenizers/src/utils/fancy.rs
sha256:a60f23b619a3c5c5df1d9dff3fde33cc641d70f36f7267d741facde53dda1191 tokenizers-0.20.3/tokenizers/src/utils/from_pretrained.rs
sha256:004540ac0fc434f6ec25c1cf8e6818998550da936a728aa5a301458c6c66a540 tokenizers-0.20.3/tokenizers/src/utils/iter.rs
sha256:ef57380e0ebd207fb7bc09f90a1aa23db851249d80fdf56d6738e8ac55cb6dbd tokenizers-0.20.3/tokenizers/src/utils/mod.rs
sha256:f6f58fb3fcf2dfbee2e8edc9b763247443fa102aea185481ff591f7d52c5256e tokenizers-0.20.3/tokenizers/src/utils/onig.rs
sha256:17e2b82c50ae7a3c296cda7898519c57911a8887a6ff5e9e57c27d215f5ed9e7 tokenizers-0.20.3/tokenizers/src/utils/padding.rs
sha256:f207764fa62d362885fd04ac5e14e9e32645a976e57e72e7663e64eaac15d531 tokenizers-0.20.3/tokenizers/src/utils/parallelism.rs
sha256:2bae922cafe5167d36cc6cd973edf4fcf6d0224be4d27fa47b9285dcca447ca8 tokenizers-0.20.3/tokenizers/src/utils/progress.rs
sha256:d73513e1391cf656258e2752e4fc392b0946172776f6764fa9eedda419e88d04 tokenizers-0.20.3/tokenizers/src/utils/truncation.rs
tokenizers-0.20.3/tokenizers/tests/
sha256:493c7097ed9a51ac4e6837ff729e71cc2ccd1bba1623e882147ca9447449bff6 tokenizers-0.20.3/tokenizers/tests/added_tokens.rs
tokenizers-0.20.3/tokenizers/tests/common/
sha256:75e506cb6cabd1a07989f086d8b573201a010318057fbb0ba287ea5e2b2d0686 tokenizers-0.20.3/tokenizers/tests/common/mod.rs
sha256:6a0054bef05508196cbb1510888011e0332be2c5ddf2ec494c8357f7ff6050af tokenizers-0.20.3/tokenizers/tests/documentation.rs
sha256:c44fc0afd1f9ec6c9ab30218b84866175fee2ae8e4315603fec0af9d1ae0f813 tokenizers-0.20.3/tokenizers/tests/from_pretrained.rs
sha256:f0f790bbe60a47112b4c250991c5769f10fa0ad3bf40fb200922b59caf841933 tokenizers-0.20.3/tokenizers/tests/offsets.rs
sha256:f6cedde9ff05d6b24917efdb7a52b75a31f2d400b52dacf4513894bc479b3857 tokenizers-0.20.3/tokenizers/tests/serialization.rs
sha256:f0433b135226479693729e4ebe15d91767545b17a5ecefe87717afc42e0dcb19 tokenizers-0.20.3/tokenizers/tests/training.rs
sha256:73192b32a0a6d53a7b22e50e74f48e9b85c1295cd956e0b6def51b5a1f71be4f tokenizers-0.20.3/tokenizers/tests/unigram.rs
Return to home page.