STX_tokenstats.txt

Stats for text:
  min: 52
  max: 105
  mean: 64.87760669090575
  median: 61.0
  std: 12.097560338377741
  skew: 1.217709088451885
  kurt: 3.8640282248984685
  count: 261011
  sum: 16933769
  99.9%: 105.0
  1%: 52.0
  2%: 52.0
  3%: 52.0
  4%: 52.0
  5%: 52.0
  6%: 52.0
  7%: 53.0
  8%: 53.0
  9%: 53.0
  10%: 53.0
  11%: 53.0
  12%: 53.0
  13%: 53.0
  14%: 54.0
  15%: 54.0
  16%: 54.0
  17%: 54.0
  18%: 54.0
  19%: 54.0
  20%: 55.0
  21%: 55.0
  22%: 55.0
  23%: 55.0
  24%: 55.0
  25%: 55.0
  26%: 56.0
  27%: 56.0
  28%: 56.0
  29%: 56.0
  30%: 56.0
  31%: 57.0
  32%: 57.0
  33%: 57.0
  34%: 57.0
  35%: 57.0
  36%: 58.0
  37%: 58.0
  38%: 58.0
  39%: 58.0
  40%: 59.0
  41%: 59.0
  42%: 59.0
  43%: 59.0
  44%: 59.0
  45%: 60.0
  46%: 60.0
  47%: 60.0
  48%: 61.0
  49%: 61.0
  50%: 61.0
  51%: 61.0
  52%: 62.0
  53%: 62.0
  54%: 62.0
  55%: 63.0
  56%: 63.0
  57%: 63.0
  58%: 64.0
  59%: 64.0
  60%: 64.0
  61%: 65.0
  62%: 65.0
  63%: 65.0
  64%: 66.0
  65%: 66.0
  66%: 66.0
  67%: 67.0
  68%: 67.0
  69%: 68.0
  70%: 68.0
  71%: 69.0
  72%: 69.0
  73%: 70.0
  74%: 70.0
  75%: 71.0
  76%: 71.0
  77%: 72.0
  78%: 73.0
  79%: 73.0
  80%: 74.0
  81%: 75.0
  82%: 75.0
  83%: 76.0
  84%: 77.0
  85%: 78.0
  86%: 79.0
  87%: 80.0
  88%: 81.0
  89%: 82.0
  90%: 83.0
  91%: 85.0
  92%: 86.0
  93%: 87.0
  94%: 89.0
  95%: 91.0
  96%: 93.0
  97%: 96.0
  98%: 98.0
  99%: 102.0
  100%: 105.0
  total_chars: 79444533
  total_words: 11946913
  avg_chars: 304.37235595434674
  avg_words: 45.77168395201735
  avg_chars_per_word: 6.649795892880445
  avg_chars_per_sample: 304.37235595434674
  avg_words_per_sample: 45.77168395201735
  tokens_per_char: 0.21315209946542202
  bin_0-8: 0
  bin_8-16: 0
  bin_16-32: 0
  bin_32-64: 151348
  bin_64-128: 109663
  bin_128-256: 0
  bin_256-384: 0
  bin_384-512: 0
  bin_512-768: 0
  bin_768-1024: 0
  bin_1024-2048: 0
  bin_2048-4096: 0

Total tokens across all columns: 16933769