chains_tokenstats.txt

Stats for text:
  min: 24
  max: 105
  mean: 54.49921874232255
  median: 51.0
  std: 17.583205401615015
  skew: 0.8053512547009355
  kurt: 2.9856355167866555
  count: 101759
  sum: 5545786
  99.9%: 105.0
  1%: 29.0
  2%: 30.0
  3%: 31.0
  4%: 32.0
  5%: 32.0
  6%: 33.0
  7%: 33.0
  8%: 34.0
  9%: 34.0
  10%: 35.0
  11%: 35.0
  12%: 36.0
  13%: 36.0
  14%: 37.0
  15%: 37.0
  16%: 37.0
  17%: 38.0
  18%: 38.0
  19%: 39.0
  20%: 39.0
  21%: 39.0
  22%: 40.0
  23%: 40.0
  24%: 41.0
  25%: 41.0
  26%: 41.0
  27%: 42.0
  28%: 42.0
  29%: 42.0
  30%: 43.0
  31%: 43.0
  32%: 43.0
  33%: 44.0
  34%: 44.0
  35%: 45.0
  36%: 45.0
  37%: 45.0
  38%: 46.0
  39%: 46.0
  40%: 47.0
  41%: 47.0
  42%: 47.0
  43%: 48.0
  44%: 48.0
  45%: 49.0
  46%: 49.0
  47%: 49.0
  48%: 50.0
  49%: 50.0
  50%: 51.0
  51%: 51.0
  52%: 52.0
  53%: 52.0
  54%: 53.0
  55%: 53.0
  56%: 53.0
  57%: 54.0
  58%: 54.0
  59%: 55.0
  60%: 55.0
  61%: 56.0
  62%: 57.0
  63%: 57.0
  64%: 58.0
  65%: 58.0
  66%: 59.0
  67%: 59.0
  68%: 60.0
  69%: 61.0
  70%: 61.0
  71%: 62.0
  72%: 63.0
  73%: 63.0
  74%: 64.0
  75%: 65.0
  76%: 66.0
  77%: 67.0
  78%: 67.0
  79%: 68.0
  80%: 69.0
  81%: 70.0
  82%: 71.0
  83%: 72.0
  84%: 73.0
  85%: 74.0
  86%: 75.0
  87%: 77.0
  88%: 78.0
  89%: 79.0
  90%: 81.0
  91%: 83.0
  92%: 84.0
  93%: 86.0
  94%: 88.0
  95%: 90.0
  96%: 93.0
  97%: 95.0
  98%: 98.0
  99%: 102.0
  100%: 105.0
  total_chars: 27535725
  total_words: 3026595
  avg_chars: 270.59744101258855
  avg_words: 29.742774594876128
  avg_chars_per_word: 9.097921922160051
  avg_chars_per_sample: 270.59744101258855
  avg_words_per_sample: 29.742774594876128
  tokens_per_char: 0.20140330425292963
  bin_0-8: 0
  bin_8-16: 0
  bin_16-32: 3904
  bin_32-64: 70458
  bin_64-128: 27397
  bin_128-256: 0
  bin_256-384: 0
  bin_384-512: 0
  bin_512-768: 0
  bin_768-1024: 0
  bin_1024-2048: 0
  bin_2048-4096: 0

Total tokens across all columns: 5545786