[
{"query": [0,0],
"sample0": [ 27, 91, 437, 1659, 5239, 91, 29, 55, 25,23349, 198, 51,
25,14874, 1252,25308,22495, 198, 44, 25, 17, 14, 19, 198,
43, 25, 16, 14, 23, 198, 42, 25, 34, 198, 38, 535,
33, 91, 32, 17, 32, 17, 91, 38, 4339, 33, 91, 66,
17, 89, 17, 91, 38, 535, 33, 91, 32, 17, 32, 17,
91, 38, 4339, 33, 91, 66, 17, 32, 17, 91, 198, 38,
4339, 33, 91, 66, 17, 32, 17, 91, 38, 4339, 33, 91,
66, 17, 32, 17, 91, 38, 4339, 33, 91, 66, 17, 32,
17, 91, 38, 4339, 33, 91, 66, 17, 32, 17,15886, 59,
77, 27, 91, 437, 1659, 5239, 91, 29, 198, 27, 91, 437,
1659, 5239, 91, 29, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27],
"sample1": [ 27, 91, 437, 1659, 5239, 91, 29, 55, 25,14208, 2816, 198,
51, 25, 47,13218,34831, 338, 198, 44, 25, 19, 14, 19,
198, 43, 25, 16, 14, 23, 198, 42, 25, 35, 76, 1228,
198, 91, 25, 33, 198, 91, 93, 32, 17, 67, 32, 61,
38, 6242, 67, 91, 7, 18, 32, 4339, 37, 2782, 33, 66,
32, 91,12473, 93, 36, 17, 38, 18, 32, 91, 33, 67,
70, 5036, 17,36077, 91, 198, 93, 32, 17, 67, 32, 61,
38, 6242, 67, 91, 7, 18, 32, 4339, 37, 2782, 33, 66,
32, 91, 33, 67, 93, 67, 17, 276, 33, 67, 91, 8579,
38, 1961, 18, 25, 91, 198, 91, 25, 32, 91, 67, 17,
69, 9395,16344, 91, 2782, 69, 2934, 17,36077, 91, 93, 67,
17, 69, 9395,16344, 91, 70, 69, 891, 67, 17,36077, 91,
198, 93, 32, 17, 69, 9395,16344, 91, 2782, 69, 2934, 17,
36077, 91, 93, 70, 18,19082, 33, 67, 91, 8579, 38, 1961,
18, 25, 91, 59, 77, 27, 91, 437, 1659, 5239, 91, 29,
198, 27, 91, 437, 1659, 5239, 91, 29, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27],
"best": 1,
"broken": 1}
]For scripting purposes, we’d like a CLI filter which takes text and prints out the BPE encoding. I hacked up the encode.py from the nshepperd codebase to make encode2.py, which reads from stdin, converts, and pads to our target length of 256: