Upload 2 files

STXtokenstats.txt

@@ -0,0 +1,133 @@
+Stats for text:
+  min: 52
+  max: 105
+  mean: 64.87760669090575
+  median: 61.0
+  std: 12.097560338377741
+  skew: 1.217709088451885
+  kurt: 3.8640282248984685
+  count: 261011
+  sum: 16933769
+  99.9%: 105.0
+  1%: 52.0
+  2%: 52.0
+  3%: 52.0
+  4%: 52.0
+  5%: 52.0
+  6%: 52.0
+  7%: 53.0
+  8%: 53.0
+  9%: 53.0
+  10%: 53.0
+  11%: 53.0
+  12%: 53.0
+  13%: 53.0
+  14%: 54.0
+  15%: 54.0
+  16%: 54.0
+  17%: 54.0
+  18%: 54.0
+  19%: 54.0
+  20%: 55.0
+  21%: 55.0
+  22%: 55.0
+  23%: 55.0
+  24%: 55.0
+  25%: 55.0
+  26%: 56.0
+  27%: 56.0
+  28%: 56.0
+  29%: 56.0
+  30%: 56.0
+  31%: 57.0
+  32%: 57.0
+  33%: 57.0
+  34%: 57.0
+  35%: 57.0
+  36%: 58.0
+  37%: 58.0
+  38%: 58.0
+  39%: 58.0
+  40%: 59.0
+  41%: 59.0
+  42%: 59.0
+  43%: 59.0
+  44%: 59.0
+  45%: 60.0
+  46%: 60.0
+  47%: 60.0
+  48%: 61.0
+  49%: 61.0
+  50%: 61.0
+  51%: 61.0
+  52%: 62.0
+  53%: 62.0
+  54%: 62.0
+  55%: 63.0
+  56%: 63.0
+  57%: 63.0
+  58%: 64.0
+  59%: 64.0
+  60%: 64.0
+  61%: 65.0
+  62%: 65.0
+  63%: 65.0
+  64%: 66.0
+  65%: 66.0
+  66%: 66.0
+  67%: 67.0
+  68%: 67.0
+  69%: 68.0
+  70%: 68.0
+  71%: 69.0
+  72%: 69.0
+  73%: 70.0
+  74%: 70.0
+  75%: 71.0
+  76%: 71.0
+  77%: 72.0
+  78%: 73.0
+  79%: 73.0
+  80%: 74.0
+  81%: 75.0
+  82%: 75.0
+  83%: 76.0
+  84%: 77.0
+  85%: 78.0
+  86%: 79.0
+  87%: 80.0
+  88%: 81.0
+  89%: 82.0
+  90%: 83.0
+  91%: 85.0
+  92%: 86.0
+  93%: 87.0
+  94%: 89.0
+  95%: 91.0
+  96%: 93.0
+  97%: 96.0
+  98%: 98.0
+  99%: 102.0
+  100%: 105.0
+  total_chars: 79444533
+  total_words: 11946913
+  avg_chars: 304.37235595434674
+  avg_words: 45.77168395201735
+  avg_chars_per_word: 6.649795892880445
+  avg_chars_per_sample: 304.37235595434674
+  avg_words_per_sample: 45.77168395201735
+  tokens_per_char: 0.21315209946542202
+  bin_0-8: 0
+  bin_8-16: 0
+  bin_16-32: 0
+  bin_32-64: 151348
+  bin_64-128: 109663
+  bin_128-256: 0
+  bin_256-384: 0
+  bin_384-512: 0
+  bin_512-768: 0
+  bin_768-1024: 0
+  bin_1024-2048: 0
+  bin_2048-4096: 0
+
+Total tokens across all columns: 16933769

chainstokenstats.txt

@@ -0,0 +1,133 @@
+Stats for text:
+  min: 24
+  max: 105
+  mean: 54.49921874232255
+  median: 51.0
+  std: 17.583205401615015
+  skew: 0.8053512547009355
+  kurt: 2.9856355167866555
+  count: 101759
+  sum: 5545786
+  99.9%: 105.0
+  1%: 29.0
+  2%: 30.0
+  3%: 31.0
+  4%: 32.0
+  5%: 32.0
+  6%: 33.0
+  7%: 33.0
+  8%: 34.0
+  9%: 34.0
+  10%: 35.0
+  11%: 35.0
+  12%: 36.0
+  13%: 36.0
+  14%: 37.0
+  15%: 37.0
+  16%: 37.0
+  17%: 38.0
+  18%: 38.0
+  19%: 39.0
+  20%: 39.0
+  21%: 39.0
+  22%: 40.0
+  23%: 40.0
+  24%: 41.0
+  25%: 41.0
+  26%: 41.0
+  27%: 42.0
+  28%: 42.0
+  29%: 42.0
+  30%: 43.0
+  31%: 43.0
+  32%: 43.0
+  33%: 44.0
+  34%: 44.0
+  35%: 45.0
+  36%: 45.0
+  37%: 45.0
+  38%: 46.0
+  39%: 46.0
+  40%: 47.0
+  41%: 47.0
+  42%: 47.0
+  43%: 48.0
+  44%: 48.0
+  45%: 49.0
+  46%: 49.0
+  47%: 49.0
+  48%: 50.0
+  49%: 50.0
+  50%: 51.0
+  51%: 51.0
+  52%: 52.0
+  53%: 52.0
+  54%: 53.0
+  55%: 53.0
+  56%: 53.0
+  57%: 54.0
+  58%: 54.0
+  59%: 55.0
+  60%: 55.0
+  61%: 56.0
+  62%: 57.0
+  63%: 57.0
+  64%: 58.0
+  65%: 58.0
+  66%: 59.0
+  67%: 59.0
+  68%: 60.0
+  69%: 61.0
+  70%: 61.0
+  71%: 62.0
+  72%: 63.0
+  73%: 63.0
+  74%: 64.0
+  75%: 65.0
+  76%: 66.0
+  77%: 67.0
+  78%: 67.0
+  79%: 68.0
+  80%: 69.0
+  81%: 70.0
+  82%: 71.0
+  83%: 72.0
+  84%: 73.0
+  85%: 74.0
+  86%: 75.0
+  87%: 77.0
+  88%: 78.0
+  89%: 79.0
+  90%: 81.0
+  91%: 83.0
+  92%: 84.0
+  93%: 86.0
+  94%: 88.0
+  95%: 90.0
+  96%: 93.0
+  97%: 95.0
+  98%: 98.0
+  99%: 102.0
+  100%: 105.0
+  total_chars: 27535725
+  total_words: 3026595
+  avg_chars: 270.59744101258855
+  avg_words: 29.742774594876128
+  avg_chars_per_word: 9.097921922160051
+  avg_chars_per_sample: 270.59744101258855
+  avg_words_per_sample: 29.742774594876128
+  tokens_per_char: 0.20140330425292963
+  bin_0-8: 0
+  bin_8-16: 0
+  bin_16-32: 3904
+  bin_32-64: 70458
+  bin_64-128: 27397
+  bin_128-256: 0
+  bin_256-384: 0
+  bin_384-512: 0
+  bin_512-768: 0
+  bin_768-1024: 0
+  bin_1024-2048: 0
+  bin_2048-4096: 0
+
+Total tokens across all columns: 5545786