tokka-bench/pyproject.toml at main · bgub/tokka-bench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
[project.scripts]
benchmark = "tokka_bench.cli:main"
dashboard = "visualization.dashboard:launch_dashboard"
test = "pytest:main"

[project]
name = "tokka-bench"
version = "0.1.0"
description = "Tokenizer benchmarking across multiple languages"
readme = "README.md"
requires-python = ">=3.9"
dependencies = [
    "transformers>=4.21.0",
    "datasets>=2.0.0",
    "pandas>=1.5.0",
    "omegaconf>=2.3.0",
    "streamlit>=1.28.0",
    "plotly>=5.15.0",
    "tiktoken>=0.11.0",
    "blobfile>=3.0.0",
    "sentencepiece>=0.2.1",
]


[dependency-groups]
dev = [
    "mypy>=1.16.1",
    "pytest>=8.4.1",
]

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.build.targets.wheel]
packages = ["src/tokka_bench", "src/visualization"]