{ "dataset_groups": [ { "bytes": 3977331, "delta_bits": 1345478.5344710723, "documents": 129648, "gap_bpb": 0.3382867894251377, "model_a_bpb": 0.5736822739748099, "model_b_bpb": 0.23539548454967224, "name": "epic:5005" }, { "bytes": 3977331, "delta_bits": 1345478.5344710723, "documents": 129648, "gap_bpb": 0.3382867894251377, "model_a_bpb": 0.5736822739748099, "model_b_bpb": 0.23539548454967224, "name": "eval_only" }, { "bytes": 3977331, "delta_bits": 1345478.5344710723, "documents": 129648, "gap_bpb": 0.3382867894251377, "model_a_bpb": 0.5736822739748099, "model_b_bpb": 0.23539548454967224, "name": "issue:6103" }, { "bytes": 3977331, "delta_bits": 1345478.5344710723, "documents": 129648, "gap_bpb": 0.3382867894251377, "model_a_bpb": 0.5736822739748099, "model_b_bpb": 0.23539548454967224, "name": "loss:target_only" }, { "bytes": 3977331, "delta_bits": 1345478.5344710723, "documents": 129648, "gap_bpb": 0.3382867894251377, "model_a_bpb": 0.5736822739748099, "model_b_bpb": 0.23539548454967224, "name": "ppl_circuit_coverage" }, { "bytes": 3977331, "delta_bits": 1345478.5344710723, "documents": 129648, "gap_bpb": 0.3382867894251377, "model_a_bpb": 0.5736822739748099, "model_b_bpb": 0.23539548454967224, "name": "seed:6103" }, { "bytes": 3977331, "delta_bits": 1345478.5344710723, "documents": 129648, "gap_bpb": 0.3382867894251377, "model_a_bpb": 0.5736822739748099, "model_b_bpb": 0.23539548454967224, "name": "source:generated_ppl_circuit_coverage_v1" }, { "bytes": 3977331, "delta_bits": 1345478.5344710723, "documents": 129648, "gap_bpb": 0.3382867894251377, "model_a_bpb": 0.5736822739748099, "model_b_bpb": 0.23539548454967224, "name": "tokenizer:marin-community" }, { "bytes": 3977331, "delta_bits": 1345478.5344710723, "documents": 129648, "gap_bpb": 0.3382867894251377, "model_a_bpb": 0.5736822739748099, "model_b_bpb": 0.23539548454967224, "name": "tokenizer:marin-community/marin-tokenizer" }, { "bytes": 3909567, "delta_bits": 1321717.1655325263, "documents": 126648, "gap_bpb": 0.33807251942031596, "model_a_bpb": 0.5633475927750925, "model_b_bpb": 0.22527507335477653, "name": "examples:tokenizer_vocab_coverage" }, { "bytes": 3909567, "delta_bits": 1321717.1655325263, "documents": 126648, "gap_bpb": 0.33807251942031596, "model_a_bpb": 0.5633475927750925, "model_b_bpb": 0.22527507335477653, "name": "family:vocab_mechanics" }, { "bytes": 3909567, "delta_bits": 1321717.1655325263, "documents": 126648, "gap_bpb": 0.33807251942031596, "model_a_bpb": 0.5633475927750925, "model_b_bpb": 0.22527507335477653, "name": "ppl_circuit_coverage/vocab_mechanics" }, { "bytes": 3909567, "delta_bits": 1321717.1655325263, "documents": 126648, "gap_bpb": 0.33807251942031596, "model_a_bpb": 0.5633475927750925, "model_b_bpb": 0.22527507335477653, "name": "task:token_chars" }, { "bytes": 67764, "delta_bits": 23761.368938546115, "documents": 3000, "gap_bpb": 0.35064885394230144, "model_a_bpb": 1.1699299655800255, "model_b_bpb": 0.8192811116377242, "name": "examples:1000" }, { "bytes": 64751, "delta_bits": 19613.099714707503, "documents": 2000, "gap_bpb": 0.3029003369014765, "model_a_bpb": 1.0656726461622585, "model_b_bpb": 0.7627723092607821, "name": "family:text_mechanics" }, { "bytes": 64751, "delta_bits": 19613.099714707503, "documents": 2000, "gap_bpb": 0.3029003369014765, "model_a_bpb": 1.0656726461622585, "model_b_bpb": 0.7627723092607821, "name": "ppl_circuit_coverage/text_mechanics" }, { "bytes": 58483, "delta_bits": 11320.221457130616, "documents": 1000, "gap_bpb": 0.19356430855343632, "model_a_bpb": 0.9362593456192041, "model_b_bpb": 0.7426950370657677, "name": "task:line_rhythm" }, { "bytes": 6268, "delta_bits": 8292.878257576887, "documents": 1000, "gap_bpb": 1.3230501368182654, "model_a_bpb": 2.2731515956931228, "model_b_bpb": 0.9501014588748572, "name": "task:character_indices" }, { "bytes": 3013, "delta_bits": 4148.26922383861, "documents": 1000, "gap_bpb": 1.376790316574381, "model_a_bpb": 3.410476161935762, "model_b_bpb": 2.0336858453613815, "name": "family:python_interpretation" }, { "bytes": 3013, "delta_bits": 4148.26922383861, "documents": 1000, "gap_bpb": 1.376790316574381, "model_a_bpb": 3.410476161935762, "model_b_bpb": 2.0336858453613815, "name": "ppl_circuit_coverage/python_interpretation" }, { "bytes": 3013, "delta_bits": 4148.26922383861, "documents": 1000, "gap_bpb": 1.376790316574381, "model_a_bpb": 3.410476161935762, "model_b_bpb": 2.0336858453613815, "name": "task:arithmetic_repl" } ], "datasets": [ { "bytes": 3909567, "delta_bits": 1321717.1655325263, "documents": 126648, "gap_bpb": 0.33807251942031596, "model_a_bpb": 0.5633475927750925, "model_b_bpb": 0.22527507335477653, "name": "ppl_circuit_coverage/vocab_mechanics/token_chars" }, { "bytes": 58483, "delta_bits": 11320.221457130616, "documents": 1000, "gap_bpb": 0.19356430855343632, "model_a_bpb": 0.9362593456192041, "model_b_bpb": 0.7426950370657677, "name": "ppl_circuit_coverage/text_mechanics/line_rhythm" }, { "bytes": 6268, "delta_bits": 8292.878257576887, "documents": 1000, "gap_bpb": 1.3230501368182654, "model_a_bpb": 2.2731515956931228, "model_b_bpb": 0.9501014588748572, "name": "ppl_circuit_coverage/text_mechanics/character_indices" }, { "bytes": 3013, "delta_bits": 4148.26922383861, "documents": 1000, "gap_bpb": 1.376790316574381, "model_a_bpb": 3.410476161935762, "model_b_bpb": 2.0336858453613815, "name": "ppl_circuit_coverage/python_interpretation/arithmetic_repl" } ], "model_a": "marin-community/marin-32b-base", "model_b": "Qwen/Qwen3-32B", "pattern_buckets": [ { "bytes": 2401100, "delta_bits": 764853.2136891272, "documents": 1432112, "gap_bpb": 0.31854284023536183, "model_a_bpb": 0.5768083416224548, "model_b_bpb": 0.258265501387093, "name": "text/punctuation" }, { "bytes": 579285, "delta_bits": 271035.99878886534, "documents": 555937, "gap_bpb": 0.46788022957415665, "model_a_bpb": 0.6397647863626839, "model_b_bpb": 0.17188455678852727, "name": "text/word" }, { "bytes": 155800, "delta_bits": 176884.15735680782, "documents": 68470, "gap_bpb": 1.1353283527394598, "model_a_bpb": 1.404737052546774, "model_b_bpb": 0.2694086998073142, "name": "text/non_ascii_word" }, { "bytes": 673787, "delta_bits": 64733.577631564025, "documents": 673787, "gap_bpb": 0.09607424546861845, "model_a_bpb": 0.23532359319435836, "model_b_bpb": 0.1392493477257399, "name": "whitespace/single_space" }, { "bytes": 129648, "delta_bits": 44921.619721292955, "documents": 129648, "gap_bpb": 0.3464891068222645, "model_a_bpb": 0.6364287039102098, "model_b_bpb": 0.28993959708794514, "name": "whitespace/newline" }, { "bytes": 29213, "delta_bits": 15452.205938990759, "documents": 8769, "gap_bpb": 0.5289496436172512, "model_a_bpb": 1.8228142074068105, "model_b_bpb": 1.2938645637895594, "name": "text/number" }, { "bytes": 8498, "delta_bits": 7597.7613444359, "documents": 1311, "gap_bpb": 0.8940646439675101, "model_a_bpb": 1.5257764618471563, "model_b_bpb": 0.6317118178796463, "name": "text/non_ascii" } ], "top_documents": { "model_a_worse": [ { "bytes": 78, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 94.87529903756109, "gap_bpb": 1.2163499876610397, "model_a_bpb": 1.2865832395441112, "model_b_bpb": 0.07023325188307175, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0445\u0430\u0440\u0430\u043a\u0442\u0435\u0440\u0438\u0441\u0442\u0438\"\"\")\u23ce['\u2420',\u2420'\u0445',\u2420'\u0430',\u2420'\u0440',\u2420'\u0430',\u2420'\u043a',\u2420'\u0442',\u2420'\u0435',\u2420'\u0440',\u2420'\u0438',\u2420'\u0441',\u2420'\u0442',\u2420'\u0438']\u23ce", "row_index": 110774, "score_byte_end": 127, "score_byte_start": 49, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 9.599535673221894, "worst_text": "\u0445" }, { "bytes": 84, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 94.05128471623631, "gap_bpb": 1.1196581513837656, "model_a_bpb": 1.1899040392668776, "model_b_bpb": 0.07024588788311205, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0430\u0434\u043c\u0456\u043d\u0456\u0441\u0442\u0440\u0430\u0442\u0438\u0432\"\"\")\u23ce['\u2420',\u2420'\u0430',\u2420'\u0434',\u2420'\u043c',\u2420'\u0456',\u2420'\u043d',\u2420'\u0456',\u2420'\u0441',\u2420'\u0442',\u2420'\u0440',\u2420'\u0430',\u2420'\u0442',\u2420'\u0438',\u2420'\u0432']\u23ce", "row_index": 125284, "score_byte_end": 135, "score_byte_start": 51, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 7.839022196983278, "worst_text": "\u0430" }, { "bytes": 43, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 90.34608274145425, "gap_bpb": 2.1010716916617267, "model_a_bpb": 2.1169964852123164, "model_b_bpb": 0.015924793550589744, "preview": ">>>\u2420token_chars(\"\"\"\u7f51\u520a\u4e0b\u8f7d\u6b21\u6570\"\"\")\u23ce['\u7f51',\u2420'\u520a',\u2420'\u4e0b',\u2420'\u8f7d',\u2420'\u6b21',\u2420'\u6570']\u23ce", "row_index": 124525, "score_byte_end": 85, "score_byte_start": 42, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 8.554271212628164, "worst_text": "\u7f51" }, { "bytes": 62, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 85.08023853778387, "gap_bpb": 1.37226191189974, "model_a_bpb": 1.4925985499607135, "model_b_bpb": 0.1203366380609736, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u010ceskosloven\"\"\")\u23ce['\u2420',\u2420'\u010c',\u2420'e',\u2420's',\u2420'k',\u2420'o',\u2420's',\u2420'l',\u2420'o',\u2420'v',\u2420'e',\u2420'n']\u23ce", "row_index": 122295, "score_byte_end": 99, "score_byte_start": 37, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 12.375292318519609, "worst_text": "\u010c" }, { "bytes": 66, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 83.90319316369906, "gap_bpb": 1.2712605024802888, "model_a_bpb": 1.3493836112115287, "model_b_bpb": 0.07812310873123987, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u043f\u0440\u043e\u0444\u0438\u043b\u0430\u043a\u0442\u0438\"\"\")\u23ce['\u2420',\u2420'\u043f',\u2420'\u0440',\u2420'\u043e',\u2420'\u0444',\u2420'\u0438',\u2420'\u043b',\u2420'\u0430',\u2420'\u043a',\u2420'\u0442',\u2420'\u0438']\u23ce", "row_index": 125681, "score_byte_end": 111, "score_byte_start": 45, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 9.202382328959837, "worst_text": "\u043f" }, { "bytes": 78, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 82.22087509307283, "gap_bpb": 1.0541137832445235, "model_a_bpb": 1.1122362814425297, "model_b_bpb": 0.05812249819800639, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0437\u0430\u0431\u0435\u0437\u043f\u0435\u0447\u0435\u043d\u043d\u044f\"\"\")\u23ce['\u2420',\u2420'\u0437',\u2420'\u0430',\u2420'\u0431',\u2420'\u0435',\u2420'\u0437',\u2420'\u043f',\u2420'\u0435',\u2420'\u0447',\u2420'\u0435',\u2420'\u043d',\u2420'\u043d',\u2420'\u044f']\u23ce", "row_index": 112666, "score_byte_end": 127, "score_byte_start": 49, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 6.041667973138053, "worst_text": "\u0437" }, { "bytes": 66, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 79.81420033230887, "gap_bpb": 1.2093060656410435, "model_a_bpb": 1.339189719213794, "model_b_bpb": 0.12988365357275036, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0445\u0430\u0440\u0430\u043a\u0442\u0435\u0440\u0438\u0437\"\"\")\u23ce['\u2420',\u2420'\u0445',\u2420'\u0430',\u2420'\u0440',\u2420'\u0430',\u2420'\u043a',\u2420'\u0442',\u2420'\u0435',\u2420'\u0440',\u2420'\u0438',\u2420'\u0437']\u23ce", "row_index": 119418, "score_byte_end": 111, "score_byte_start": 45, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 9.837921513513427, "worst_text": "\u0445" }, { "bytes": 57, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 78.9278293046073, "gap_bpb": 1.3846987597299527, "model_a_bpb": 1.4593085509132895, "model_b_bpb": 0.07460979118333662, "preview": ">>>\u2420token_chars(\"\"\"\u2420ar\u015fivlendi\"\"\")\u23ce['\u2420',\u2420'a',\u2420'r',\u2420'\u015f',\u2420'i',\u2420'v',\u2420'l',\u2420'e',\u2420'n',\u2420'd',\u2420'i']\u23ce", "row_index": 110743, "score_byte_end": 93, "score_byte_start": 36, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 6.678156501924411, "worst_text": "\u015f" }, { "bytes": 78, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 78.82364762679366, "gap_bpb": 1.010559584958893, "model_a_bpb": 1.0908227062870677, "model_b_bpb": 0.08026312132817455, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0432\u0438\u043a\u043e\u0440\u0438\u0441\u0442\u0430\u043d\u043d\u044f\"\"\")\u23ce['\u2420',\u2420'\u0432',\u2420'\u0438',\u2420'\u043a',\u2420'\u043e',\u2420'\u0440',\u2420'\u0438',\u2420'\u0441',\u2420'\u0442',\u2420'\u0430',\u2420'\u043d',\u2420'\u043d',\u2420'\u044f']\u23ce", "row_index": 112113, "score_byte_end": 127, "score_byte_start": 49, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 5.040427975477492, "worst_text": "\u0438" }, { "bytes": 61, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 77.96187612143487, "gap_bpb": 1.278063542974342, "model_a_bpb": 1.2969570495083407, "model_b_bpb": 0.018893506533998728, "preview": ">>>\u2420token_chars(\"\"\"\u0410\u0440\u0445\u0456\u0432\u043e\u0432\u0430\u043d\u043e\"\"\")\u23ce['\u0410',\u2420'\u0440',\u2420'\u0445',\u2420'\u0456',\u2420'\u0432',\u2420'\u043e',\u2420'\u0432',\u2420'\u0430',\u2420'\u043d',\u2420'\u043e']\u23ce", "row_index": 116452, "score_byte_end": 105, "score_byte_start": 44, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 12.91348657096243, "worst_text": "\u0410" }, { "bytes": 72, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 77.69026754599905, "gap_bpb": 1.0790314936944312, "model_a_bpb": 1.1994711351854905, "model_b_bpb": 0.12043964149105918, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u043f\u0456\u0434\u043f\u0440\u0438\u0454\u043c\u0441\u0442\u0432\"\"\")\u23ce['\u2420',\u2420'\u043f',\u2420'\u0456',\u2420'\u0434',\u2420'\u043f',\u2420'\u0440',\u2420'\u0438',\u2420'\u0454',\u2420'\u043c',\u2420'\u0441',\u2420'\u0442',\u2420'\u0432']\u23ce", "row_index": 125266, "score_byte_end": 119, "score_byte_start": 47, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 4.781713097923367, "worst_text": "\u0456" }, { "bytes": 96, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 76.60003891732954, "gap_bpb": 0.7979170720555161, "model_a_bpb": 0.8753754639324746, "model_b_bpb": 0.0774583918769585, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0432\u0438\u043a\u043e\u0440\u0438\u0441\u0442\u043e\u0432\u0443\u0432\u0430\u0442\u0438\"\"\")\u23ce['\u2420',\u2420'\u0432',\u2420'\u0438',\u2420'\u043a',\u2420'\u043e',\u2420'\u0440',\u2420'\u0438',\u2420'\u0441',\u2420'\u0442',\u2420'\u043e',\u2420'\u0432',\u2420'\u0443',\u2420'\u0432',\u2420'\u0430',\u2420'\u0442',\u2420'\u0438\u2026", "row_index": 123014, "score_byte_end": 151, "score_byte_start": 55, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 4.4629202088739355, "worst_text": "\u0438" }, { "bytes": 78, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 74.94831823712263, "gap_bpb": 0.9608758748349056, "model_a_bpb": 1.0612716380580054, "model_b_bpb": 0.10039576322309979, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0432\u0456\u0434\u0431\u0443\u0432\u0430\u0454\u0442\u044c\u0441\u044f\"\"\")\u23ce['\u2420',\u2420'\u0432',\u2420'\u0456',\u2420'\u0434',\u2420'\u0431',\u2420'\u0443',\u2420'\u0432',\u2420'\u0430',\u2420'\u0454',\u2420'\u0442',\u2420'\u044c',\u2420'\u0441',\u2420'\u044f']\u23ce", "row_index": 123297, "score_byte_end": 127, "score_byte_start": 49, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 5.302612431030133, "worst_text": "\u0431" }, { "bytes": 56, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 74.71036654016092, "gap_bpb": 1.3341136882171594, "model_a_bpb": 1.4791424318303448, "model_b_bpb": 0.14502874361318532, "preview": ">>>\u2420token_chars(\"\"\"\u2420eoqkrvldkf\"\"\")\u23ce['\u2420',\u2420'e',\u2420'o',\u2420'q',\u2420'k',\u2420'r',\u2420'v',\u2420'l',\u2420'd',\u2420'k',\u2420'f']\u23ce", "row_index": 107930, "score_byte_end": 91, "score_byte_start": 35, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": 12.030910130848284, "worst_text": "e" }, { "bytes": 78, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 74.0202262955497, "gap_bpb": 0.9489772601993552, "model_a_bpb": 1.0506610764615631, "model_b_bpb": 0.1016838162622079, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\"\"\")\u23ce['\u2420',\u2420'\u0438',\u2420'\u0441',\u2420'\u043f',\u2420'\u043e',\u2420'\u043b',\u2420'\u044c',\u2420'\u0437',\u2420'\u043e',\u2420'\u0432',\u2420'\u0430',\u2420'\u043d',\u2420'\u0438']\u23ce", "row_index": 117125, "score_byte_end": 127, "score_byte_start": 49, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 8.068352806741597, "worst_text": "\u0438" }, { "bytes": 54, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 73.20790776457329, "gap_bpb": 1.355701995640246, "model_a_bpb": 1.4269961423493172, "model_b_bpb": 0.07129414670907097, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u043f\u0440\u043e\u0444\u0435\u0441\u0441\u0438\"\"\")\u23ce['\u2420',\u2420'\u043f',\u2420'\u0440',\u2420'\u043e',\u2420'\u0444',\u2420'\u0435',\u2420'\u0441',\u2420'\u0441',\u2420'\u0438']\u23ce", "row_index": 115730, "score_byte_end": 95, "score_byte_start": 41, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 12.465070013992403, "worst_text": "\u043f" }, { "bytes": 78, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 72.71176273013336, "gap_bpb": 0.932202086283761, "model_a_bpb": 1.0319446329400146, "model_b_bpb": 0.09974254665625376, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0434\u0435\u044f\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438\"\"\")\u23ce['\u2420',\u2420'\u0434',\u2420'\u0435',\u2420'\u044f',\u2420'\u0442',\u2420'\u0435',\u2420'\u043b',\u2420'\u044c',\u2420'\u043d',\u2420'\u043e',\u2420'\u0441',\u2420'\u0442',\u2420'\u0438']\u23ce", "row_index": 111338, "score_byte_end": 127, "score_byte_start": 49, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 7.695808116026196, "worst_text": "\u0434" }, { "bytes": 34, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 72.32439701258497, "gap_bpb": 2.1271881474289698, "model_a_bpb": 2.245386238989138, "model_b_bpb": 0.1181980915601686, "preview": ">>>\u2420token_chars(\"\"\"\u2420\ub178\ucd9c\ub4f1\ub85d\"\"\")\u23ce['\u2420',\u2420'\ub178',\u2420'\ucd9c',\u2420'\ub4f1',\u2420'\ub85d']\u23ce", "row_index": 117045, "score_byte_end": 71, "score_byte_start": 37, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 7.080218918972693, "worst_text": "\ub178" }, { "bytes": 72, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 71.94794093118989, "gap_bpb": 0.9992769573776373, "model_a_bpb": 1.1496969776415866, "model_b_bpb": 0.15042002026394913, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u03bd\u03b5\u03c6\u03bf\u03ba\u03ac\u03bb\u03c5\u03c8\u03b7\u03c2\"\"\")\u23ce['\u2420',\u2420'\u03bd',\u2420'\u03b5',\u2420'\u03c6',\u2420'\u03bf',\u2420'\u03ba',\u2420'\u03ac',\u2420'\u03bb',\u2420'\u03c5',\u2420'\u03c8',\u2420'\u03b7',\u2420'\u03c2']\u23ce", "row_index": 123707, "score_byte_end": 119, "score_byte_start": 47, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 7.381851035153731, "worst_text": "\u03bd" }, { "bytes": 66, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 71.71899069868518, "gap_bpb": 1.0866513742225026, "model_a_bpb": 1.3252560806778522, "model_b_bpb": 0.2386047064553496, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u043f\u0440\u0435\u0434\u043d\u0430\u0437\u043d\u0430\u0447\"\"\")\u23ce['\u2420',\u2420'\u043f',\u2420'\u0440',\u2420'\u0435',\u2420'\u0434',\u2420'\u043d',\u2420'\u0430',\u2420'\u0437',\u2420'\u043d',\u2420'\u0430',\u2420'\u0447']\u23ce", "row_index": 125016, "score_byte_end": 111, "score_byte_start": 45, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 10.713642725164187, "worst_text": "\u043f" }, { "bytes": 47, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 71.56871885282101, "gap_bpb": 1.5227386989961917, "model_a_bpb": 1.5528843449649934, "model_b_bpb": 0.030145645968801697, "preview": ">>>\u2420token_chars(\"\"\"\u040e\u044b\u045fN\u040e\u044b\u045fN\"\"\")\u23ce['\u040e',\u2420'\u044b',\u2420'\u045f',\u2420'N',\u2420'\u040e',\u2420'\u044b',\u2420'\u045f',\u2420'N']\u23ce", "row_index": 125772, "score_byte_end": 85, "score_byte_start": 38, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 8.473788835469199, "worst_text": "\u040e" }, { "bytes": 51, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 71.50812660245072, "gap_bpb": 1.402120129459818, "model_a_bpb": 1.4216784551363388, "model_b_bpb": 0.0195583256765209, "preview": ">>>\u2420token_chars(\"\"\"ilmektedir\"\"\")\u23ce['i',\u2420'l',\u2420'm',\u2420'e',\u2420'k',\u2420't',\u2420'e',\u2420'd',\u2420'i',\u2420'r']\u23ce", "row_index": 107466, "score_byte_end": 85, "score_byte_start": 34, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": 13.051695956779408, "worst_text": "i" }, { "bytes": 66, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 71.42079650494595, "gap_bpb": 1.082133280377969, "model_a_bpb": 1.2175624467770494, "model_b_bpb": 0.13542916639908037, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0432\u0438\u043a\u043e\u0440\u0438\u0441\u0442\u043e\u0432\"\"\")\u23ce['\u2420',\u2420'\u0432',\u2420'\u0438',\u2420'\u043a',\u2420'\u043e',\u2420'\u0440',\u2420'\u0438',\u2420'\u0441',\u2420'\u0442',\u2420'\u043e',\u2420'\u0432']\u23ce", "row_index": 106126, "score_byte_end": 111, "score_byte_start": 45, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 8.239717695922506, "worst_text": "\u0432" }, { "bytes": 78, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 70.8959087383064, "gap_bpb": 0.9089219069013641, "model_a_bpb": 0.9638739798975112, "model_b_bpb": 0.05495207299614703, "preview": ">>>\u2420token_chars(\"\"\"\u2420\u0437\u0430\u0445\u0432\u043e\u0440\u044e\u0432\u0430\u043d\u043d\u044f\"\"\")\u23ce['\u2420',\u2420'\u0437',\u2420'\u0430',\u2420'\u0445',\u2420'\u0432',\u2420'\u043e',\u2420'\u0440',\u2420'\u044e',\u2420'\u0432',\u2420'\u0430',\u2420'\u043d',\u2420'\u043d',\u2420'\u044f']\u23ce", "row_index": 123169, "score_byte_end": 127, "score_byte_start": 49, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 4.70118084579565, "worst_text": "\u044e" }, { "bytes": 34, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 70.29837796681858, "gap_bpb": 2.0675993519652525, "model_a_bpb": 2.241948645969957, "model_b_bpb": 0.17434929400470459, "preview": ">>>\u2420token_chars(\"\"\"\u2420\ub4f1\ub85d\ub300\ud589\"\"\")\u23ce['\u2420',\u2420'\ub4f1',\u2420'\ub85d',\u2420'\ub300',\u2420'\ud589']\u23ce", "row_index": 115154, "score_byte_end": 71, "score_byte_start": 37, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/non_ascii_word", "worst_gap_bpb": 7.666060620871764, "worst_text": "\ub4f1" } ], "model_b_worse": [ { "bytes": 46, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -46.11854848781752, "gap_bpb": -1.0025771410395112, "model_a_bpb": 0.5565310419768084, "model_b_bpb": 1.5591081830163196, "preview": ">>>\u2420token_chars(\"\"\"\u2420wannonce\"\"\")\u23ce['\u2420',\u2420'w',\u2420'a',\u2420'n',\u2420'n',\u2420'o',\u2420'n',\u2420'c',\u2420'e']\u23ce", "row_index": 83966, "score_byte_end": 79, "score_byte_start": 33, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -17.551718842837232, "worst_text": "w" }, { "bytes": 61, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -36.98260531953274, "gap_bpb": -0.6062722183529957, "model_a_bpb": 0.5330969873765795, "model_b_bpb": 1.1393692057295752, "preview": ">>>\u2420token_chars(\"\"\"\u2420sexkontakte\"\"\")\u23ce['\u2420',\u2420's',\u2420'e',\u2420'x',\u2420'k',\u2420'o',\u2420'n',\u2420't',\u2420'a',\u2420'k',\u2420't',\u2420'e']\u23ce", "row_index": 86656, "score_byte_end": 97, "score_byte_start": 36, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -7.829941599009052, "worst_text": "n" }, { "bytes": 56, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -33.65525299686297, "gap_bpb": -0.6009866606582673, "model_a_bpb": 0.48952788494551447, "model_b_bpb": 1.090514545603782, "preview": ">>>\u2420token_chars(\"\"\"\u2420vivastreet\"\"\")\u23ce['\u2420',\u2420'v',\u2420'i',\u2420'v',\u2420'a',\u2420's',\u2420't',\u2420'r',\u2420'e',\u2420'e',\u2420't']\u23ce", "row_index": 65287, "score_byte_end": 91, "score_byte_start": 35, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -13.251871809589472, "worst_text": "v" }, { "bytes": 41, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -31.586742683264536, "gap_bpb": -0.7704083581284034, "model_a_bpb": 0.5211139326521513, "model_b_bpb": 1.2915222907805546, "preview": ">>>\u2420token_chars(\"\"\"\u2420sexdate\"\"\")\u23ce['\u2420',\u2420's',\u2420'e',\u2420'x',\u2420'd',\u2420'a',\u2420't',\u2420'e']\u23ce", "row_index": 62411, "score_byte_end": 73, "score_byte_start": 32, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -9.797161281293645, "worst_text": "d" }, { "bytes": 52, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -30.09996958803606, "gap_bpb": -0.5788455690006935, "model_a_bpb": 0.3904587188618888, "model_b_bpb": 0.9693042878625823, "preview": ">>>\u2420token_chars(\"\"\"\u21e5TokenName\"\"\")\u23ce['\\t',\u2420'T',\u2420'o',\u2420'k',\u2420'e',\u2420'n',\u2420'N',\u2420'a',\u2420'm',\u2420'e']\u23ce", "row_index": 40271, "score_byte_end": 86, "score_byte_start": 34, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -11.864086830524382, "worst_text": "N" }, { "bytes": 61, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -28.704538416172873, "gap_bpb": -0.4705662035438176, "model_a_bpb": 0.4451376588325769, "model_b_bpb": 0.9157038623763945, "preview": ">>>\u2420token_chars(\"\"\"\u2420swingerclub\"\"\")\u23ce['\u2420',\u2420's',\u2420'w',\u2420'i',\u2420'n',\u2420'g',\u2420'e',\u2420'r',\u2420'c',\u2420'l',\u2420'u',\u2420'b']\u23ce", "row_index": 46457, "score_byte_end": 97, "score_byte_start": 36, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -10.56935839617078, "worst_text": "s" }, { "bytes": 36, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -28.320285705530345, "gap_bpb": -0.7866746029313985, "model_a_bpb": 0.5760298547433316, "model_b_bpb": 1.36270445767473, "preview": ">>>\u2420token_chars(\"\"\"erdings\"\"\")\u23ce['e',\u2420'r',\u2420'd',\u2420'i',\u2420'n',\u2420'g',\u2420's']\u23ce", "row_index": 69096, "score_byte_end": 67, "score_byte_start": 31, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -15.398426917009221, "worst_text": "i" }, { "bytes": 56, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -28.0963786707817, "gap_bpb": -0.5017210476925303, "model_a_bpb": 0.6098481791157294, "model_b_bpb": 1.1115692268082598, "preview": ">>>\u2420token_chars(\"\"\"\u2420sextreffen\"\"\")\u23ce['\u2420',\u2420's',\u2420'e',\u2420'x',\u2420't',\u2420'r',\u2420'e',\u2420'f',\u2420'f',\u2420'e',\u2420'n']\u23ce", "row_index": 97172, "score_byte_end": 91, "score_byte_start": 35, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -12.100202671807185, "worst_text": "x" }, { "bytes": 41, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -26.988059834690095, "gap_bpb": -0.6582453618217097, "model_a_bpb": 0.3627697631246915, "model_b_bpb": 1.0210151249464012, "preview": ">>>\u2420token_chars(\"\"\"\u2420eskorte\"\"\")\u23ce['\u2420',\u2420'e',\u2420's',\u2420'k',\u2420'o',\u2420'r',\u2420't',\u2420'e']\u23ce", "row_index": 38820, "score_byte_end": 73, "score_byte_start": 32, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -8.980833762170414, "worst_text": "e" }, { "bytes": 16, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -25.579744813639213, "gap_bpb": -1.5987340508524508, "model_a_bpb": 2.3710021730665622, "model_b_bpb": 3.969736223919013, "preview": "(?:['(',\u2420'?',\u2420':']\u23ce", "row_index": 67627, "score_byte_end": 19, "score_byte_start": 3, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/punctuation", "worst_gap_bpb": -3.6986961752942684, "worst_text": "['('," }, { "bytes": 76, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -23.554025044557626, "gap_bpb": -0.3099213821652319, "model_a_bpb": 0.8751303960154092, "model_b_bpb": 1.1850517781806413, "preview": ">>>\u2420token_chars(\"\"\"\u2420thuisontvangst\"\"\")\u23ce['\u2420',\u2420't',\u2420'h',\u2420'u',\u2420'i',\u2420's',\u2420'o',\u2420'n',\u2420't',\u2420'v',\u2420'a',\u2420'n',\u2420'g',\u2420's',\u2420't']\u23ce", "row_index": 78741, "score_byte_end": 115, "score_byte_start": 39, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -10.833787053334536, "worst_text": "t" }, { "bytes": 91, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -21.98111622414017, "gap_bpb": -0.24155072773780406, "model_a_bpb": 0.566592210737716, "model_b_bpb": 0.8081429384755201, "preview": "\u2026oken_chars(\"\"\"MASConstraintMaker\"\"\")\u23ce['M',\u2420'A',\u2420'S',\u2420'C',\u2420'o',\u2420'n',\u2420's',\u2420't',\u2420'r',\u2420'a',\u2420'i',\u2420'n',\u2420't',\u2420'M',\u2420'a',\u2420'k',\u2420\u2026", "row_index": 59493, "score_byte_end": 133, "score_byte_start": 42, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -9.996542488265407, "worst_text": "o" }, { "bytes": 41, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -21.845492292646394, "gap_bpb": -0.5328168851864974, "model_a_bpb": 0.8606758488520652, "model_b_bpb": 1.3934927340385626, "preview": ">>>\u2420token_chars(\"\"\"\u2420davidjl\"\"\")\u23ce['\u2420',\u2420'd',\u2420'a',\u2420'v',\u2420'i',\u2420'd',\u2420'j',\u2420'l']\u23ce", "row_index": 99172, "score_byte_end": 73, "score_byte_start": 32, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -8.041443799248313, "worst_text": "d" }, { "bytes": 66, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -21.329466805441932, "gap_bpb": -0.3231737394763929, "model_a_bpb": 0.5829112801871499, "model_b_bpb": 0.9060850196635428, "preview": ">>>\u2420token_chars(\"\"\"\u2420prostituerte\"\"\")\u23ce['\u2420',\u2420'p',\u2420'r',\u2420'o',\u2420's',\u2420't',\u2420'i',\u2420't',\u2420'u',\u2420'e',\u2420'r',\u2420't',\u2420'e']\u23ce", "row_index": 51183, "score_byte_end": 103, "score_byte_start": 37, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -8.17830716284832, "worst_text": "p" }, { "bytes": 71, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -20.994900175541645, "gap_bpb": -0.295702819373826, "model_a_bpb": 0.49251054805311906, "model_b_bpb": 0.788213367426945, "preview": "\u2026oproject\u23cetoken_chars\u2420raw\u2420block\u2420end\u23cecharacters:\u23ce['.',\u2420'd',\u2420'j',\u2420'a',\u2420'n',\u2420'g',\u2420'o',\u2420'p',\u2420'r',\u2420'o',\u2420'j',\u2420'e',\u2420'c',\u2420't']\u23ce", "row_index": 64019, "score_byte_end": 152, "score_byte_start": 81, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -9.587273998145974, "worst_text": "p" }, { "bytes": 31, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -20.985472355344235, "gap_bpb": -0.6769507211401367, "model_a_bpb": 0.21095722553337073, "model_b_bpb": 0.8879079466735075, "preview": ">>>\u2420token_chars(\"\"\"iosity\"\"\")\u23ce['i',\u2420'o',\u2420's',\u2420'i',\u2420't',\u2420'y']\u23ce", "row_index": 32887, "score_byte_end": 61, "score_byte_start": 30, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -17.438647908497366, "worst_text": "t" }, { "bytes": 56, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -20.091554125639906, "gap_bpb": -0.3587777522435697, "model_a_bpb": 0.950894260785739, "model_b_bpb": 1.3096720130293087, "preview": ">>>\u2420token_chars(\"\"\"useRalative\"\"\")\u23ce['u',\u2420's',\u2420'e',\u2420'R',\u2420'a',\u2420'l',\u2420'a',\u2420't',\u2420'i',\u2420'v',\u2420'e']\u23ce", "row_index": 88747, "score_byte_end": 91, "score_byte_start": 35, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -5.3789699762908265, "worst_text": "u" }, { "bytes": 16, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -20.079761480843256, "gap_bpb": -1.2549850925527035, "model_a_bpb": 3.2724589653326626, "model_b_bpb": 4.527444057885366, "preview": "]+\"[']',\u2420'+',\u2420'\"']\u23ce", "row_index": 53420, "score_byte_end": 19, "score_byte_start": 3, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/punctuation", "worst_gap_bpb": -2.8670464122369, "worst_text": "[']'," }, { "bytes": 57, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -19.703113724783826, "gap_bpb": -0.34566866183831274, "model_a_bpb": 1.069128741357851, "model_b_bpb": 1.4147974031961636, "preview": ">>>\u2420token_chars(\"\"\"\u21e5NdrFcShort\"\"\")\u23ce['\\t',\u2420'N',\u2420'd',\u2420'r',\u2420'F',\u2420'c',\u2420'S',\u2420'h',\u2420'o',\u2420'r',\u2420't']\u23ce", "row_index": 75603, "score_byte_end": 92, "score_byte_start": 35, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -9.22504689062114, "worst_text": "N" }, { "bytes": 17, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -19.340486488872823, "gap_bpb": -1.1376756758160484, "model_a_bpb": 2.675683663679205, "model_b_bpb": 3.813359339495254, "preview": "+'\\['+',\u2420\"'\",\u2420'\\\\']\u23ce", "row_index": 61900, "score_byte_end": 20, "score_byte_start": 3, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/punctuation", "worst_gap_bpb": -1.1268467012014844, "worst_text": "['+'," }, { "bytes": 36, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -19.330324977922505, "gap_bpb": -0.5369534716089585, "model_a_bpb": 0.9607383724647478, "model_b_bpb": 1.4976918440737064, "preview": ">>>\u2420token_chars(\"\"\"\u2420bakeka\"\"\")\u23ce['\u2420',\u2420'b',\u2420'a',\u2420'k',\u2420'e',\u2420'k',\u2420'a']\u23ce", "row_index": 82343, "score_byte_end": 67, "score_byte_start": 31, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -12.894884411497161, "worst_text": "k" }, { "bytes": 33, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -18.96483970408817, "gap_bpb": -0.5746921122450961, "model_a_bpb": 1.0285854853956289, "model_b_bpb": 1.603277597640725, "preview": ">>>\u2420token_chars(\"\"\"\u2420\"\"\"\u21b5\u23ce\"\"\")\u23ce['\u2420',\u2420'\"',\u2420'\"',\u2420'\"',\u2420'\\r',\u2420'\\n']\u23ce", "row_index": 26212, "score_byte_end": 63, "score_byte_start": 30, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/punctuation", "worst_gap_bpb": -1.3190386818776414, "worst_text": "'\"'," }, { "bytes": 56, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -18.94483059755185, "gap_bpb": -0.33830054638485446, "model_a_bpb": 0.6290904155876866, "model_b_bpb": 0.967390961972541, "preview": ">>>\u2420token_chars(\"\"\"\u2420'../../../\"\"\")\u23ce['\u2420',\u2420\"'\",\u2420'.',\u2420'.',\u2420'/',\u2420'.',\u2420'.',\u2420'/',\u2420'.',\u2420'.',\u2420'/']\u23ce", "row_index": 18793, "score_byte_end": 91, "score_byte_start": 35, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/punctuation", "worst_gap_bpb": -1.8868300725675171, "worst_text": "'.'," }, { "bytes": 61, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -18.66268572805092, "gap_bpb": -0.30594566767296594, "model_a_bpb": 0.35170897458976935, "model_b_bpb": 0.6576546422627352, "preview": ">>>\u2420token_chars(\"\"\"\u2420thaimassage\"\"\")\u23ce['\u2420',\u2420't',\u2420'h',\u2420'a',\u2420'i',\u2420'm',\u2420'a',\u2420's',\u2420's',\u2420'a',\u2420'g',\u2420'e']\u23ce", "row_index": 66360, "score_byte_end": 97, "score_byte_start": 36, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -7.122219382581346, "worst_text": "s" }, { "bytes": 56, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -18.50125045338528, "gap_bpb": -0.33037947238188, "model_a_bpb": 0.3705031567903077, "model_b_bpb": 0.7008826291721878, "preview": ">>>\u2420token_chars(\"\"\"\u2420datingside\"\"\")\u23ce['\u2420',\u2420'd',\u2420'a',\u2420't',\u2420'i',\u2420'n',\u2420'g',\u2420's',\u2420'i',\u2420'd',\u2420'e']\u23ce", "row_index": 76462, "score_byte_end": 91, "score_byte_start": 35, "shard": "vocab_mechanics_token_chars_jsonl_gz", "worst_bucket": "text/word", "worst_gap_bpb": -8.319856279762408, "worst_text": "d" } ] }, "top_literals": { "model_a_worse": [ { "bucket": "text/punctuation", "bytes": 203408, "delta_bits": 443750.1394450762, "documents": 101704, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420));\u23ce\"\"\")\u23ce['\u2420',\u2420')',\u2420')',\u2420';',\u2420'\\n']\u23ce", "gap_bpb": 2.1815766314258838, "model_a_bpb": 2.439141114545987, "model_a_token_boundaries": "|['|", "model_b_bpb": 0.2575644831201035, "model_b_token_boundaries": "|['|", "name": "['" }, { "bucket": "text/punctuation", "bytes": 245004, "delta_bits": 85428.96758206986, "documents": 122502, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u0159id\"\"\")\u23ce['\u2420',\u2420'\u0159',\u2420'i',\u2420'd']\u23ce", "gap_bpb": 0.34868397080076186, "model_a_bpb": 0.6167446005513763, "model_a_token_boundaries": "|']\u2026|", "model_b_bpb": 0.2680606297506144, "model_b_token_boundaries": "|']\u2026|", "name": "']" }, { "bucket": "text/punctuation", "bytes": 1132918, "delta_bits": 75119.43201934149, "documents": 566459, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"CompanyName\"\"\")\u23ce['C',\u2420'o',\u2420'm',\u2420'p',\u2420'a',\u2420'n',\u2420'y',\u2420'N',\u2420'a',\u2420'm',\u2420'e']\u23ce", "gap_bpb": 0.06630615103594567, "model_a_bpb": 0.19605052026362926, "model_a_token_boundaries": "|',|", "model_b_bpb": 0.1297443692276836, "model_b_token_boundaries": "|',|", "name": "'," }, { "bucket": "whitespace/single_space", "bytes": 673787, "delta_bits": 64733.577631564025, "documents": 673787, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"')?>',\u2420'<',\u2420'/']\u23ce", "gap_bpb": 0.09607424546861845, "model_a_bpb": 0.23532359319435836, "model_a_token_boundaries": "|\u2420\u2026|", "model_b_bpb": 0.1392493477257399, "model_b_token_boundaries": "|\u2420\u2026|", "name": "\u2420" }, { "bucket": "whitespace/newline", "bytes": 129648, "delta_bits": 44921.619721292955, "documents": 129648, "example_dataset": "ppl_circuit_coverage/python_interpretation/arithmetic_repl", "example_doc_preview": ">>>\u2420(((15\u2420*\u2420-9)\u2420-\u2420(16\u2420*\u242017))\u2420*\u2420((7\u2420+\u2420-2)\u2420+\u2420(7\u2420-\u2420-2)))\u23ce-5698\u23ce", "gap_bpb": 0.3464891068222645, "model_a_bpb": 0.6364287039102098, "model_a_token_boundaries": "|\u23ce|", "model_b_bpb": 0.28993959708794514, "model_b_token_boundaries": "|\u23ce|", "name": "\u23ce" }, { "bucket": "text/punctuation", "bytes": 30680, "delta_bits": 43871.92051601834, "documents": 6136, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "\u2026in\u23ce.LayoutInflater\u23cetoken_chars\u2420raw\u2420block\u2420end\u23cecharacters:\u23ce['.',\u2420'L',\u2420'a',\u2420'y',\u2420'o',\u2420'u',\u2420't',\u2420'I',\u2420'n',\u2420'f',\u2420'l',\u2420'a',\u2420\u2026", "gap_bpb": 1.4299843714477947, "model_a_bpb": 3.264430417557586, "model_a_token_boundaries": "|['|.',|", "model_b_bpb": 1.8344460461097911, "model_b_token_boundaries": "|['|.',|", "name": "['.'," }, { "bucket": "text/punctuation", "bytes": 580524, "delta_bits": 28848.22993266414, "documents": 580524, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "token_chars\u2420raw\u2420block\u2420begin\u23ce(tokens\u23cetoken_chars\u2420raw\u2420block\u2420end\u23cecharacters:\u23ce['(',\u2420't',\u2420'o',\u2420'k',\u2420'e',\u2420'n',\u2420's']\u23ce", "gap_bpb": 0.04969343202462627, "model_a_bpb": 0.06191839227490312, "model_a_token_boundaries": "|\u2026'|", "model_b_bpb": 0.012224960250276843, "model_b_token_boundaries": "|\u2026'|", "name": "'" }, { "bucket": "text/word", "bytes": 59097, "delta_bits": 18340.44972654296, "documents": 59097, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u010dl\u00e1nek\"\"\")\u23ce['\u2420',\u2420'\u010d',\u2420'l',\u2420'\u00e1',\u2420'n',\u2420'e',\u2420'k']\u23ce", "gap_bpb": 0.3103448521336609, "model_a_bpb": 0.40055106547769465, "model_a_token_boundaries": "|e|", "model_b_bpb": 0.09020621334403371, "model_b_token_boundaries": "|e|", "name": "e" }, { "bucket": "text/word", "bytes": 32328, "delta_bits": 16968.21898114583, "documents": 32328, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"cassert\"\"\")\u23ce['c',\u2420'a',\u2420's',\u2420's',\u2420'e',\u2420'r',\u2420't']\u23ce", "gap_bpb": 0.5248768553930286, "model_a_bpb": 0.6924466019064888, "model_a_token_boundaries": "|s|", "model_b_bpb": 0.16756974651346013, "model_b_token_boundaries": "|s|", "name": "s" }, { "bucket": "text/punctuation", "bytes": 13255, "delta_bits": 16285.520083523415, "documents": 2651, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "((((['(',\u2420'(',\u2420'(',\u2420'(']\u23ce", "gap_bpb": 1.2286322205600464, "model_a_bpb": 3.202311490024768, "model_a_token_boundaries": "|['|(',|", "model_b_bpb": 1.9736792694647216, "model_b_token_boundaries": "|['|(',|", "name": "['('," }, { "bucket": "text/word", "bytes": 36782, "delta_bits": 14687.757248876707, "documents": 36782, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420judiciary\"\"\")\u23ce['\u2420',\u2420'j',\u2420'u',\u2420'd',\u2420'i',\u2420'c',\u2420'i',\u2420'a',\u2420'r',\u2420'y']\u23ce", "gap_bpb": 0.39931915743778773, "model_a_bpb": 0.575581611202348, "model_a_token_boundaries": "|r|", "model_b_bpb": 0.17626245376456026, "model_b_token_boundaries": "|r|", "name": "r" }, { "bucket": "text/word", "bytes": 38381, "delta_bits": 14643.936363067933, "documents": 38381, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"erdings\"\"\")\u23ce['e',\u2420'r',\u2420'd',\u2420'i',\u2420'n',\u2420'g',\u2420's']\u23ce", "gap_bpb": 0.38154129290711375, "model_a_bpb": 0.5106702290933508, "model_a_token_boundaries": "|i|", "model_b_bpb": 0.129128936186237, "model_b_token_boundaries": "|i|", "name": "i" }, { "bucket": "text/word", "bytes": 39260, "delta_bits": 14625.337618387304, "documents": 39260, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "\u2026\u2420token_chars(\"\"\"TouchableOpacity\"\"\")\u23ce['T',\u2420'o',\u2420'u',\u2420'c',\u2420'h',\u2420'a',\u2420'b',\u2420'l',\u2420'e',\u2420'O',\u2420'p',\u2420'a',\u2420'c',\u2420'i',\u2420't',\u2420'y']\u23ce", "gap_bpb": 0.3725251558427739, "model_a_bpb": 0.482476325916132, "model_a_token_boundaries": "|a|", "model_b_bpb": 0.10995117007335807, "model_b_token_boundaries": "|a|", "name": "a" }, { "bucket": "text/word", "bytes": 19695, "delta_bits": 14133.31696824524, "documents": 19695, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"syscall\"\"\")\u23ce['s',\u2420'y',\u2420's',\u2420'c',\u2420'a',\u2420'l',\u2420'l']\u23ce", "gap_bpb": 0.7176093916346911, "model_a_bpb": 0.9068497335407162, "model_a_token_boundaries": "|c|", "model_b_bpb": 0.18924034190602512, "model_b_token_boundaries": "|c|", "name": "c" }, { "bucket": "text/punctuation", "bytes": 9570, "delta_bits": 13364.918952050082, "documents": 1914, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "token_chars\u2420raw\u2420block\u2420begin\u23ce-sum\u23cetoken_chars\u2420raw\u2420block\u2420end\u23cecharacters:\u23ce['-',\u2420's',\u2420'u',\u2420'm']\u23ce", "gap_bpb": 1.396543255177647, "model_a_bpb": 3.4469778686490553, "model_a_token_boundaries": "|['|-',|", "model_b_bpb": 2.0504346134714084, "model_b_token_boundaries": "|['|-',|", "name": "['-'," }, { "bucket": "text/word", "bytes": 39968, "delta_bits": 13155.102372264246, "documents": 39968, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"iosity\"\"\")\u23ce['i',\u2420'o',\u2420's',\u2420'i',\u2420't',\u2420'y']\u23ce", "gap_bpb": 0.3291408720042095, "model_a_bpb": 0.4826039313739552, "model_a_token_boundaries": "|t|", "model_b_bpb": 0.1534630593697457, "model_b_token_boundaries": "|t|", "name": "t" }, { "bucket": "text/word", "bytes": 32640, "delta_bits": 11352.752071721312, "documents": 32640, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"AccessorType\"\"\")\u23ce['A',\u2420'c',\u2420'c',\u2420'e',\u2420's',\u2420's',\u2420'o',\u2420'r',\u2420'T',\u2420'y',\u2420'p',\u2420'e']\u23ce", "gap_bpb": 0.3478171590600892, "model_a_bpb": 0.4458797727074253, "model_a_token_boundaries": "|o|", "model_b_bpb": 0.09806261364733608, "model_b_token_boundaries": "|o|", "name": "o" }, { "bucket": "text/word", "bytes": 24477, "delta_bits": 10210.751842432102, "documents": 24477, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u00e6lland\"\"\")\u23ce['\u00e6',\u2420'l',\u2420'l',\u2420'a',\u2420'n',\u2420'd']\u23ce", "gap_bpb": 0.41715699809748347, "model_a_bpb": 0.515081351106472, "model_a_token_boundaries": "|l|", "model_b_bpb": 0.09792435300898858, "model_b_token_boundaries": "|l|", "name": "l" }, { "bucket": "text/word", "bytes": 38393, "delta_bits": 10014.061621741937, "documents": 38393, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u010dn\u011b\"\"\")\u23ce['\u010d',\u2420'n',\u2420'\u011b']\u23ce", "gap_bpb": 0.2608304019415502, "model_a_bpb": 0.36973692225902516, "model_a_token_boundaries": "|n|", "model_b_bpb": 0.10890652031747493, "model_b_token_boundaries": "|n|", "name": "n" }, { "bucket": "text/word", "bytes": 14304, "delta_bits": 9758.243555635205, "documents": 14304, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "\u2026_chars(\"\"\"\u2420sourceMappingURL\"\"\")\u23ce['\u2420',\u2420's',\u2420'o',\u2420'u',\u2420'r',\u2420'c',\u2420'e',\u2420'M',\u2420'a',\u2420'p',\u2420'p',\u2420'i',\u2420'n',\u2420'g',\u2420'U',\u2420'R',\u2420'L']\u23ce", "gap_bpb": 0.6822038279946312, "model_a_bpb": 0.8931681518200193, "model_a_token_boundaries": "|p|", "model_b_bpb": 0.2109643238253881, "model_b_token_boundaries": "|p|", "name": "p" }, { "bucket": "text/word", "bytes": 18275, "delta_bits": 8736.250719904434, "documents": 18275, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420b\u00e5de\"\"\")\u23ce['\u2420',\u2420'b',\u2420'\u00e5',\u2420'd',\u2420'e']\u23ce", "gap_bpb": 0.4780438150426503, "model_a_bpb": 0.6252638383571578, "model_a_token_boundaries": "|d|", "model_b_bpb": 0.14722002331450756, "model_b_token_boundaries": "|d|", "name": "d" }, { "bucket": "text/word", "bytes": 14032, "delta_bits": 8239.438037370499, "documents": 14032, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"m\u0131z\u0131\"\"\")\u23ce['m',\u2420'\u0131',\u2420'z',\u2420'\u0131']\u23ce", "gap_bpb": 0.5871891417738383, "model_a_bpb": 0.7204848951562249, "model_a_token_boundaries": "|m|", "model_b_bpb": 0.13329575338238667, "model_b_token_boundaries": "|m|", "name": "m" }, { "bucket": "text/word", "bytes": 16440, "delta_bits": 7850.063917365917, "documents": 16440, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420astounding\"\"\")\u23ce['\u2420',\u2420'a',\u2420's',\u2420't',\u2420'o',\u2420'u',\u2420'n',\u2420'd',\u2420'i',\u2420'n',\u2420'g']\u23ce", "gap_bpb": 0.47749780519257407, "model_a_bpb": 0.6582107630720855, "model_a_token_boundaries": "|u|", "model_b_bpb": 0.18071295787951136, "model_b_token_boundaries": "|u|", "name": "u" }, { "bucket": "text/punctuation", "bytes": 5013, "delta_bits": 7121.913096070022, "documents": 1671, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "token_chars\u2420raw\u2420block\u2420begin\u23ce\u200c\u062a\u0648\u0627\u0646\u062f\u23cetoken_chars\u2420raw\u2420block\u2420end\u23cecharacters:\u23ce['\\u200c',\u2420'\u062a',\u2420'\u0648',\u2420'\u0627',\u2420'\u0646',\u2420'\u062f']\u23ce", "gap_bpb": 1.420688828260527, "model_a_bpb": 3.5422836366840995, "model_a_token_boundaries": "|['|\\|", "model_b_bpb": 2.1215948084235725, "model_b_token_boundaries": "|['|\\|", "name": "['\\" }, { "bucket": "text/word", "bytes": 12019, "delta_bits": 6518.80610808933, "documents": 12019, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420innings\"\"\")\u23ce['\u2420',\u2420'i',\u2420'n',\u2420'n',\u2420'i',\u2420'n',\u2420'g',\u2420's']\u23ce", "gap_bpb": 0.5423750817946027, "model_a_bpb": 0.6810642354169306, "model_a_token_boundaries": "|g|", "model_b_bpb": 0.13868915362232787, "model_b_token_boundaries": "|g|", "name": "g" }, { "bucket": "text/word", "bytes": 4705, "delta_bits": 5761.948493032326, "documents": 4705, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "\u2026\u2420token_chars(\"\"\"\u2420activeClassName\"\"\")\u23ce['\u2420',\u2420'a',\u2420'c',\u2420't',\u2420'i',\u2420'v',\u2420'e',\u2420'C',\u2420'l',\u2420'a',\u2420's',\u2420's',\u2420'N',\u2420'a',\u2420'm',\u2420'e']\u23ce", "gap_bpb": 1.2246436754585177, "model_a_bpb": 1.803330525069691, "model_a_token_boundaries": "|C|", "model_b_bpb": 0.5786868496111734, "model_b_token_boundaries": "|C|", "name": "C" }, { "bucket": "text/word", "bytes": 5232, "delta_bits": 5707.282418473308, "documents": 5232, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420weakSelf\"\"\")\u23ce['\u2420',\u2420'w',\u2420'e',\u2420'a',\u2420'k',\u2420'S',\u2420'e',\u2420'l',\u2420'f']\u23ce", "gap_bpb": 1.0908414408396996, "model_a_bpb": 1.5605691794733552, "model_a_token_boundaries": "|S|", "model_b_bpb": 0.4697277386336555, "model_b_token_boundaries": "|S|", "name": "S" }, { "bucket": "text/non_ascii_word", "bytes": 3508, "delta_bits": 5598.900059627392, "documents": 1754, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u0438\u0442\u0438\u0441\u044f\"\"\")\u23ce['\u0438',\u2420'\u0442',\u2420'\u0438',\u2420'\u0441',\u2420'\u044f']\u23ce", "gap_bpb": 1.5960376452757674, "model_a_bpb": 1.6870969487952827, "model_a_token_boundaries": "|\u0438|", "model_b_bpb": 0.09105930351951516, "model_b_token_boundaries": "|\u0438|", "name": "\u0438" }, { "bucket": "text/non_ascii_word", "bytes": 5314, "delta_bits": 5383.652082374187, "documents": 2657, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u043e\u0441\u0442\u0430\u043d\u043d\u0456\"\"\")\u23ce['\u2420',\u2420'\u043e',\u2420'\u0441',\u2420'\u0442',\u2420'\u0430',\u2420'\u043d',\u2420'\u043d',\u2420'\u0456']\u23ce", "gap_bpb": 1.0131072793327411, "model_a_bpb": 1.0787996979299042, "model_a_token_boundaries": "|\u043e|", "model_b_bpb": 0.06569241859716292, "model_b_token_boundaries": "|\u043e|", "name": "\u043e" }, { "bucket": "text/non_ascii_word", "bytes": 4172, "delta_bits": 5099.042755005243, "documents": 2086, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u0430\u0434\u0436\u0435\u043d\u043d\u044f\"\"\")\u23ce['\u0430',\u2420'\u0434',\u2420'\u0436',\u2420'\u0435',\u2420'\u043d',\u2420'\u043d',\u2420'\u044f']\u23ce", "gap_bpb": 1.222205837728965, "model_a_bpb": 1.2630164049669717, "model_a_token_boundaries": "|\u0430|", "model_b_bpb": 0.040810567238006336, "model_b_token_boundaries": "|\u0430|", "name": "\u0430" }, { "bucket": "text/punctuation", "bytes": 5780, "delta_bits": 5029.676262070801, "documents": 1156, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"/\uff0f\"\"\")\u23ce['/',\u2420'\uff0f']\u23ce", "gap_bpb": 0.8701862045105193, "model_a_bpb": 3.0090666555038244, "model_a_token_boundaries": "|['|/',|", "model_b_bpb": 2.1388804509933053, "model_b_token_boundaries": "|['|/',|", "name": "['/'," }, { "bucket": "text/word", "bytes": 6170, "delta_bits": 4924.046464756485, "documents": 6170, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"ificador\"\"\")\u23ce['i',\u2420'f',\u2420'i',\u2420'c',\u2420'a',\u2420'd',\u2420'o',\u2420'r']\u23ce", "gap_bpb": 0.798062636103158, "model_a_bpb": 0.9774367239974306, "model_a_token_boundaries": "|f|", "model_b_bpb": 0.1793740878942727, "model_b_token_boundaries": "|f|", "name": "f" }, { "bucket": "text/word", "bytes": 7439, "delta_bits": 4667.356187886373, "documents": 7439, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"ibbean\"\"\")\u23ce['i',\u2420'b',\u2420'b',\u2420'e',\u2420'a',\u2420'n']\u23ce", "gap_bpb": 0.6274171512147295, "model_a_bpb": 0.8212507614659832, "model_a_token_boundaries": "|b|", "model_b_bpb": 0.1938336102512536, "model_b_token_boundaries": "|b|", "name": "b" }, { "bucket": "text/non_ascii_word", "bytes": 3832, "delta_bits": 4491.672456540456, "documents": 1916, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u0435\u0440\u0456\u0433\u0430\"\"\")\u23ce['\u0435',\u2420'\u0440',\u2420'\u0456',\u2420'\u0433',\u2420'\u0430']\u23ce", "gap_bpb": 1.1721483446086784, "model_a_bpb": 1.2358789408771738, "model_a_token_boundaries": "|\u0435|", "model_b_bpb": 0.06373059626849542, "model_b_token_boundaries": "|\u0435|", "name": "\u0435" }, { "bucket": "text/non_ascii_word", "bytes": 2858, "delta_bits": 4329.756795419887, "documents": 1429, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u0441\u043b\u0456\u0434\"\"\")\u23ce['\u0441',\u2420'\u043b',\u2420'\u0456',\u2420'\u0434']\u23ce", "gap_bpb": 1.5149603902798763, "model_a_bpb": 1.6245709339315964, "model_a_token_boundaries": "|\u0441|", "model_b_bpb": 0.10961054365171988, "model_b_token_boundaries": "|\u0441|", "name": "\u0441" }, { "bucket": "text/word", "bytes": 6020, "delta_bits": 4315.635364263937, "documents": 6020, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"avadoc\"\"\")\u23ce['a',\u2420'v',\u2420'a',\u2420'd',\u2420'o',\u2420'c']\u23ce", "gap_bpb": 0.7168829508744081, "model_a_bpb": 0.9080382595379008, "model_a_token_boundaries": "|v|", "model_b_bpb": 0.19115530866349262, "model_b_token_boundaries": "|v|", "name": "v" }, { "bucket": "text/word", "bytes": 9723, "delta_bits": 4226.120881804469, "documents": 9723, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420trhu\"\"\")\u23ce['\u2420',\u2420't',\u2420'r',\u2420'h',\u2420'u']\u23ce", "gap_bpb": 0.43465194711554755, "model_a_bpb": 0.5366479824936785, "model_a_token_boundaries": "|h|", "model_b_bpb": 0.101996035378131, "model_b_token_boundaries": "|h|", "name": "h" }, { "bucket": "text/word", "bytes": 12000, "delta_bits": 4060.13512726207, "documents": 1000, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "\u2026\u2420\u2420oxpkqip\u23ce\u2420\u2420\u2420mmvxzp\u23ce\u2420bveu\u23ceerdgv\u23ce\u2420\u2420\u2420\u2420bwdwd\u23ce\u2420\u2420\u2420\u2420r\u23ceresult:\u23ce{\"line_lengths\":[11,9,5,5,9,5],\"indent_widths\":[4,3,1,0,4,4]}\u23ce", "gap_bpb": 0.33834459393850586, "model_a_bpb": 0.39860733864909487, "model_a_token_boundaries": "|line|_lengths|", "model_b_bpb": 0.06026274471058906, "model_b_token_boundaries": "|line|_lengths|", "name": "line_lengths" }, { "bucket": "text/non_ascii_word", "bytes": 3258, "delta_bits": 4010.765165320531, "documents": 1629, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u043d\u0438\u0446\u0442\u0432\u0430\"\"\")\u23ce['\u043d',\u2420'\u0438',\u2420'\u0446',\u2420'\u0442',\u2420'\u0432',\u2420'\u0430']\u23ce", "gap_bpb": 1.2310513091837112, "model_a_bpb": 1.3367567434225802, "model_a_token_boundaries": "|\u043d|", "model_b_bpb": 0.10570543423886883, "model_b_token_boundaries": "|\u043d|", "name": "\u043d" }, { "bucket": "text/punctuation", "bytes": 27665, "delta_bits": 3893.062989984465, "documents": 5533, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "__(['_',\u2420'_',\u2420'(']\u23ce", "gap_bpb": 0.14072159732457853, "model_a_bpb": 3.5905631938449623, "model_a_token_boundaries": "|['_|',|", "model_b_bpb": 3.449841596520384, "model_b_token_boundaries": "|['_|',|", "name": "['_'," } ], "model_b_worse": [ { "bucket": "text/punctuation", "bytes": 6000, "delta_bits": -3898.879856195666, "documents": 2000, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "line_lengths\u2420and\u2420indent_widths\u2420for\u2420raw\u2420block:\u23ce\u2420v\u23ce\u2420gunsto\u23cej\u23ceresult:\u23ce{\"line_lengths\":[2,7,1],\"indent_widths\":[1,1,0]}\u23ce", "gap_bpb": -0.6498133093659443, "model_a_bpb": 0.32552469127555606, "model_a_token_boundaries": "|\":[|", "model_b_bpb": 0.9753380006415004, "model_b_token_boundaries": "|\":[|", "name": "\":[" }, { "bucket": "text/punctuation", "bytes": 676, "delta_bits": -258.1343836581339, "documents": 169, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "(^)(['(',\u2420'^',\u2420')',\u2420'(']\u23ce", "gap_bpb": -0.381855597127417, "model_a_bpb": 2.2141167305075444, "model_a_token_boundaries": "|\u2026'('|]\u2026|", "model_b_bpb": 2.5959723276349616, "model_b_token_boundaries": "|\u2026'('|]\u2026|", "name": "'(']" }, { "bucket": "text/punctuation", "bytes": 3748, "delta_bits": -232.51456937141245, "documents": 937, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\":\"\",\"\"\"\")\u23ce['\"',\u2420':',\u2420'\"',\u2420'\"',\u2420',',\u2420'\"']\u23ce", "gap_bpb": -0.06203697155053694, "model_a_bpb": 0.8918430549849552, "model_a_token_boundaries": "|\u2026'\"|',|", "model_b_bpb": 0.9538800265354922, "model_b_token_boundaries": "|\u2026'\"|',|", "name": "'\"'," }, { "bucket": "text/punctuation", "bytes": 455, "delta_bits": -184.25406525891407, "documents": 91, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "+\"\\['+',\u2420'\"',\u2420'\\\\']\u23ce", "gap_bpb": -0.4049539895800309, "model_a_bpb": 1.5920655042884215, "model_a_token_boundaries": "|\u2026'\\\\|']\u2026|", "model_b_bpb": 1.9970194938684527, "model_b_token_boundaries": "|\u2026'\\\\|']\u2026|", "name": "'\\\\']" }, { "bucket": "text/punctuation", "bytes": 2000, "delta_bits": -123.58003937390727, "documents": 1000, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "\u2026for\u2420raw\u2420block:\u23ce\u2420s\u23ce\u2420\u2420xsfcmdja\u23ce\u2420\u2420\u2420\u2420injswjh\u23ce\u2420gh\u23ce\u2420\u2420\u2420dk\u23ceresult:\u23ce{\"line_lengths\":[2,10,11,3,5],\"indent_widths\":[1,2,4,1,3]}\u23ce", "gap_bpb": -0.061790019686953634, "model_a_bpb": 0.3314920739868864, "model_a_token_boundaries": "|]}\u2026|", "model_b_bpb": 0.39328209367384, "model_b_token_boundaries": "|]}\u2026|", "name": "]}" }, { "bucket": "text/punctuation", "bytes": 780, "delta_bits": -105.39428651807391, "documents": 195, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"')?>',\u2420'<',\u2420'/']\u23ce", "gap_bpb": -0.1351208801513768, "model_a_bpb": 0.7436571605434057, "model_a_token_boundaries": "|\u2026'<|',|", "model_b_bpb": 0.8787780406947824, "model_b_token_boundaries": "|\u2026'<|',|", "name": "'<'," }, { "bucket": "text/punctuation", "bytes": 556, "delta_bits": -88.47874888446192, "documents": 139, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "\u093e:['\u093e',\u2420':']\u23ce", "gap_bpb": -0.15913444043967973, "model_a_bpb": 1.4944931537309125, "model_a_token_boundaries": "|\u2026':|']\u2026|", "model_b_bpb": 1.6536275941705922, "model_b_token_boundaries": "|\u2026':|']\u2026|", "name": "':']" }, { "bucket": "text/punctuation", "bytes": 428, "delta_bits": -83.77018308214105, "documents": 107, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "#[['#',\u2420'[']\u23ce", "gap_bpb": -0.19572472682743236, "model_a_bpb": 2.2751061373613095, "model_a_token_boundaries": "|\u2026'[|']\u2026|", "model_b_bpb": 2.4708308641887418, "model_b_token_boundaries": "|\u2026'[|']\u2026|", "name": "'[']" }, { "bucket": "text/punctuation", "bytes": 1060, "delta_bits": -78.11621129927626, "documents": 212, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": "]+\"[']',\u2420'+',\u2420'\"']\u23ce", "gap_bpb": -0.07369453896158137, "model_a_bpb": 5.737658388261221, "model_a_token_boundaries": "|[|']|',|", "model_b_bpb": 5.811352927222803, "model_b_token_boundaries": "|[|']|',|", "name": "[']'," }, { "bucket": "text/punctuation", "bytes": 465, "delta_bits": -55.30142847861962, "documents": 93, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420'\\''\"\"\")\u23ce['\u2420',\u2420\"'\",\u2420'\\\\',\u2420\"'\",\u2420\"'\"]\u23ce", "gap_bpb": -0.11892780317982714, "model_a_bpb": 0.9515716059407733, "model_a_token_boundaries": "|\u2026'\\\\|',|", "model_b_bpb": 1.0704994091206006, "model_b_token_boundaries": "|\u2026'\\\\|',|", "name": "'\\\\'," }, { "bucket": "text/punctuation", "bytes": 1284, "delta_bits": -46.18396542768645, "documents": 321, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"','=','\"\"\")\u23ce[\"'\",\u2420',',\u2420\"'\",\u2420'=',\u2420\"'\",\u2420',',\u2420\"'\"]\u23ce", "gap_bpb": -0.03596882042654708, "model_a_bpb": 0.6860598391818765, "model_a_token_boundaries": "|\u2026',',|", "model_b_bpb": 0.7220286596084236, "model_b_token_boundaries": "|\u2026',',|", "name": "','," }, { "bucket": "text/punctuation", "bytes": 356, "delta_bits": -41.47305988094185, "documents": 89, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420!!}\"\"\")\u23ce['\u2420',\u2420'!',\u2420'!',\u2420'}']\u23ce", "gap_bpb": -0.11649735921612878, "model_a_bpb": 0.5214367185337393, "model_a_token_boundaries": "|\u2026'|!',|", "model_b_bpb": 0.6379340777498681, "model_b_token_boundaries": "|\u2026'|!',|", "name": "'!'," }, { "bucket": "text/punctuation", "bytes": 920, "delta_bits": -34.69051310237216, "documents": 230, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"({...\"\"\")\u23ce['(',\u2420'{',\u2420'.',\u2420'.',\u2420'.']\u23ce", "gap_bpb": -0.03770707945910017, "model_a_bpb": 1.2979744328808283, "model_a_token_boundaries": "|\u2026'.|']\u2026|", "model_b_bpb": 1.3356815123399286, "model_b_token_boundaries": "|\u2026'.|']\u2026|", "name": "'.']" }, { "bucket": "text/non_ascii_word", "bytes": 12, "delta_bits": -27.253598811847734, "documents": 4, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u3147\u3147\"\"\")\u23ce['\u2420',\u2420'\u3147',\u2420'\u3147']\u23ce", "gap_bpb": -2.2711332343206445, "model_a_bpb": 0.19640504719858434, "model_a_token_boundaries": "|\u3147|", "model_b_bpb": 2.467538281519229, "model_b_token_boundaries": "|\u3147|", "name": "\u3147" }, { "bucket": "text/punctuation", "bytes": 472, "delta_bits": -25.096936771320912, "documents": 118, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420<->\"\"\")\u23ce['\u2420',\u2420'<',\u2420'-',\u2420'>']\u23ce", "gap_bpb": -0.053171476210425665, "model_a_bpb": 0.8423117653360839, "model_a_token_boundaries": "|\u2026'>|']\u2026|", "model_b_bpb": 0.8954832415465096, "model_b_token_boundaries": "|\u2026'>|']\u2026|", "name": "'>']" }, { "bucket": "text/punctuation", "bytes": 500, "delta_bits": -18.75635367825223, "documents": 125, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\")==\"\"\")\u23ce['\"',\u2420')',\u2420'=',\u2420'=']\u23ce", "gap_bpb": -0.03751270735650446, "model_a_bpb": 1.411456317504864, "model_a_token_boundaries": "|\u2026'=|']\u2026|", "model_b_bpb": 1.4489690248613685, "model_b_token_boundaries": "|\u2026'=|']\u2026|", "name": "'=']" }, { "bucket": "text/number", "bytes": 15, "delta_bits": -17.923975064902553, "documents": 3, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "\u2026dths\u2420for\u2420raw\u2420block:\u23ce\u2420\u2420\u2420\u2420fkyvvqdxwzv\u23ce\u2420\u2420\u2420\u2420gmaguuvs\u23ce\u2420\u2420jvknwnb\u23ceresult:\u23ce{\"line_lengths\":[15,12,9],\"indent_widths\":[4,4,2]}\u23ce", "gap_bpb": -1.1949316709935036, "model_a_bpb": 1.8816233185938662, "model_a_token_boundaries": "|4|,|4|,|2|", "model_b_bpb": 3.0765549895873696, "model_b_token_boundaries": "|4|,|4|,|2|", "name": "4,4,2" }, { "bucket": "text/non_ascii_word", "bytes": 64, "delta_bits": -17.85020857801302, "documents": 32, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"cks\u00e5\"\"\")\u23ce['c',\u2420'k',\u2420's',\u2420'\u00e5']\u23ce", "gap_bpb": -0.27890950903145345, "model_a_bpb": 0.4995915439217331, "model_a_token_boundaries": "|\u00e5|", "model_b_bpb": 0.7785010529531865, "model_b_token_boundaries": "|\u00e5|", "name": "\u00e5" }, { "bucket": "text/non_ascii_word", "bytes": 45, "delta_bits": -16.584141181412672, "documents": 15, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u30b7\u30e3\"\"\")\u23ce['\u30b7',\u2420'\u30e3']\u23ce", "gap_bpb": -0.3685364706980594, "model_a_bpb": 1.4817964048952619, "model_a_token_boundaries": "|\u30b7|", "model_b_bpb": 1.850332875593321, "model_b_token_boundaries": "|\u30b7|", "name": "\u30b7" }, { "bucket": "text/non_ascii", "bytes": 18, "delta_bits": -14.765797564253182, "documents": 3, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\uff0c\u300a\"\"\")\u23ce['\uff0c',\u2420'\u300a']\u23ce", "gap_bpb": -0.8203220869029546, "model_a_bpb": 0.3480528038340802, "model_a_token_boundaries": "|\u2026'|\u300a|']\u2026|", "model_b_bpb": 1.1683748907370348, "model_b_token_boundaries": "|\u2026'|\u300a|']\u2026|", "name": "'\u300a']" }, { "bucket": "text/number", "bytes": 25, "delta_bits": -14.714185739128874, "documents": 5, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "\u2026s\u2420and\u2420indent_widths\u2420for\u2420raw\u2420block:\u23ce\u2420\u2420xqnlpoxpsh\u23ce\u2420lbah\u23ce\u2420shku\u23ceresult:\u23ce{\"line_lengths\":[12,5,5],\"indent_widths\":[2,1,1]}\u23ce", "gap_bpb": -0.588567429565155, "model_a_bpb": 1.7109711541819073, "model_a_token_boundaries": "|2|,|1|,|1|", "model_b_bpb": 2.2995385837470623, "model_b_token_boundaries": "|2|,|1|,|1|", "name": "2,1,1" }, { "bucket": "text/punctuation", "bytes": 1012, "delta_bits": -14.67558368494851, "documents": 253, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"={({\"\"\")\u23ce['=',\u2420'{',\u2420'(',\u2420'{']\u23ce", "gap_bpb": -0.014501564906075603, "model_a_bpb": 0.6341321307935426, "model_a_token_boundaries": "|\u2026'{|',|", "model_b_bpb": 0.6486336956996182, "model_b_token_boundaries": "|\u2026'{|',|", "name": "'{'," }, { "bucket": "text/non_ascii_word", "bytes": 102, "delta_bits": -14.473118635565802, "documents": 34, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u3061\u3087\u3063\u3068\"\"\")\u23ce['\u3061',\u2420'\u3087',\u2420'\u3063',\u2420'\u3068']\u23ce", "gap_bpb": -0.14189331995652746, "model_a_bpb": 0.9211018235440386, "model_a_token_boundaries": "|\u3063|", "model_b_bpb": 1.062995143500566, "model_b_token_boundaries": "|\u3063|", "name": "\u3063" }, { "bucket": "text/non_ascii_word", "bytes": 3, "delta_bits": -14.41686362641653, "documents": 1, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u3161\"\"\")\u23ce['\u2420',\u2420'\u3161']\u23ce", "gap_bpb": -4.80562120880551, "model_a_bpb": 0.4076499040646158, "model_a_token_boundaries": "|\u3161|\u3161|", "model_b_bpb": 5.2132711128701255, "model_b_token_boundaries": "|\u3161|", "name": "\u3161" }, { "bucket": "text/non_ascii", "bytes": 24, "delta_bits": -13.09258604306656, "documents": 4, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u3001\"\"\")\u23ce['\u2420',\u2420'\u3001']\u23ce", "gap_bpb": -0.5455244184611067, "model_a_bpb": 0.8156130957270212, "model_a_token_boundaries": "|\u2026'|\u3001|']\u2026|", "model_b_bpb": 1.361137514188128, "model_b_token_boundaries": "|\u2026'|\u3001|']\u2026|", "name": "'\u3001']" }, { "bucket": "text/non_ascii_word", "bytes": 12, "delta_bits": -13.004038618320765, "documents": 4, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u65cf\u81ea\u6cbb\"\"\")\u23ce['\u65cf',\u2420'\u81ea',\u2420'\u6cbb']\u23ce", "gap_bpb": -1.0836698848600637, "model_a_bpb": 0.535001129389643, "model_a_token_boundaries": "|\u65cf|", "model_b_bpb": 1.6186710142497065, "model_b_token_boundaries": "|\u65cf|", "name": "\u65cf" }, { "bucket": "text/non_ascii", "bytes": 6, "delta_bits": -11.993729151396613, "documents": 1, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u300b\"\"\")\u23ce['\u2420',\u2420'\u300b']\u23ce", "gap_bpb": -1.998954858566102, "model_a_bpb": 0.447952307718889, "model_a_token_boundaries": "|\u2026'|\u300b|']\u2026|", "model_b_bpb": 2.446907166284991, "model_b_token_boundaries": "|\u2026'|\u300b|']\u2026|", "name": "'\u300b']" }, { "bucket": "text/non_ascii", "bytes": 6, "delta_bits": -11.394379910600977, "documents": 1, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u301c\"\"\")\u23ce['\u2420',\u2420'\u301c']\u23ce", "gap_bpb": -1.8990633184334962, "model_a_bpb": 0.46885565596322953, "model_a_token_boundaries": "|\u2026'|\u301c|']\u2026|", "model_b_bpb": 2.3679189743967255, "model_b_token_boundaries": "|\u2026'|\u301c|']\u2026|", "name": "'\u301c']" }, { "bucket": "text/non_ascii", "bytes": 6, "delta_bits": -11.155885545673812, "documents": 1, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u0e3f\"\"\")\u23ce['\u2420',\u2420'\u0e3f']\u23ce", "gap_bpb": -1.8593142576123018, "model_a_bpb": 0.26343832671833545, "model_a_token_boundaries": "|\u2026'|\u0e3f|\u0e3f|']\u2026|", "model_b_bpb": 2.1227525843306374, "model_b_token_boundaries": "|\u2026'|\u0e3f|']\u2026|", "name": "'\u0e3f']" }, { "bucket": "text/number", "bytes": 10, "delta_bits": -10.564167658798102, "documents": 2, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "\u2026\u2420and\u2420indent_widths\u2420for\u2420raw\u2420block:\u23ce\u2420\u2420\u2420yg\u23ce\u2420ktakh\u23ce\u2420\u2420\u2420\u2420mljedgdx\u23ceresult:\u23ce{\"line_lengths\":[5,6,12],\"indent_widths\":[3,1,4]}\u23ce", "gap_bpb": -1.0564167658798103, "model_a_bpb": 2.493211039022008, "model_a_token_boundaries": "|3|,|1|,|4|", "model_b_bpb": 3.549627804901818, "model_b_token_boundaries": "|3|,|1|,|4|", "name": "3,1,4" }, { "bucket": "text/number", "bytes": 15, "delta_bits": -10.356205710677925, "documents": 3, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "\u2026and\u2420indent_widths\u2420for\u2420raw\u2420block:\u23ce\u2420\u2420\u2420\u2420wbaqly\u23ce\u2420\u2420\u2420\u2420yde\u23ce\u2420giipae\u23ceresult:\u23ce{\"line_lengths\":[10,7,7],\"indent_widths\":[4,4,1]}\u23ce", "gap_bpb": -0.690413714045195, "model_a_bpb": 2.259694494079332, "model_a_token_boundaries": "|4|,|4|,|1|", "model_b_bpb": 2.9501082081245267, "model_b_token_boundaries": "|4|,|4|,|1|", "name": "4,4,1" }, { "bucket": "text/number", "bytes": 10, "delta_bits": -10.19387626327648, "documents": 2, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "\u2026ent_widths\u2420for\u2420raw\u2420block:\u23ce\u2420\u2420qbhvtre\u23ce\u2420\u2420\u2420\u2420gcqecjh\u23ce\u2420\u2420\u2420hbbpqry\u23ceresult:\u23ce{\"line_lengths\":[9,11,10],\"indent_widths\":[2,4,3]}\u23ce", "gap_bpb": -1.019387626327648, "model_a_bpb": 2.0464059272438133, "model_a_token_boundaries": "|2|,|4|,|3|", "model_b_bpb": 3.0657935535714613, "model_b_token_boundaries": "|2|,|4|,|3|", "name": "2,4,3" }, { "bucket": "text/number", "bytes": 7, "delta_bits": -10.14757450282083, "documents": 1, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "\u2026\u2420block:\u23ce\u2420\u2420qnujq\u23ce\u2420\u2420\u2420\u2420bspjqvka\u23ce\u2420\u2420\u2420\u2420knspfj\u23ce\u2420\u2420kvwvxeocspu\u23ceresult:\u23ce{\"line_lengths\":[7,12,10,13],\"indent_widths\":[2,4,4,2]}\u23ce", "gap_bpb": -1.4496535004029758, "model_a_bpb": 1.6838000328442444, "model_a_token_boundaries": "|2|,|4|,|4|,|2|", "model_b_bpb": 3.1334535332472204, "model_b_token_boundaries": "|2|,|4|,|4|,|2|", "name": "2,4,4,2" }, { "bucket": "text/non_ascii_word", "bytes": 18, "delta_bits": -9.759982329804068, "documents": 6, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u4e07\"\"\")\u23ce['\u2420',\u2420'\u4e07']\u23ce", "gap_bpb": -0.5422212405446705, "model_a_bpb": 0.735992574091716, "model_a_token_boundaries": "|\u4e07|", "model_b_bpb": 1.2782138146363866, "model_b_token_boundaries": "|\u4e07|", "name": "\u4e07" }, { "bucket": "text/non_ascii_word", "bytes": 6, "delta_bits": -8.994083649558187, "documents": 2, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u53c3\"\"\")\u23ce['\u2420',\u2420'\u53c3']\u23ce", "gap_bpb": -1.499013941593031, "model_a_bpb": 0.28994936337170146, "model_a_token_boundaries": "|\u53c3|", "model_b_bpb": 1.7889633049647324, "model_b_token_boundaries": "|\u53c3|", "name": "\u53c3" }, { "bucket": "text/non_ascii_word", "bytes": 9, "delta_bits": -8.796543430872605, "documents": 3, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\ub9ce\uc774\"\"\")\u23ce['\u2420',\u2420'\ub9ce',\u2420'\uc774']\u23ce", "gap_bpb": -0.9773937145414006, "model_a_bpb": 0.9816063913559147, "model_a_token_boundaries": "|\ub9ce|\ub9ce|", "model_b_bpb": 1.9590001058973152, "model_b_token_boundaries": "|\ub9ce|", "name": "\ub9ce" }, { "bucket": "text/number", "bytes": 15, "delta_bits": -8.66283766874243, "documents": 3, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "\u2026ths\u2420and\u2420indent_widths\u2420for\u2420raw\u2420block:\u23ce\u2420\u2420\u2420homy\u23cebscwglq\u23ce\u2420\u2420oopmv\u23ceresult:\u23ce{\"line_lengths\":[7,7,7],\"indent_widths\":[3,0,2]}\u23ce", "gap_bpb": -0.5775225112494954, "model_a_bpb": 2.1386614075811523, "model_a_token_boundaries": "|3|,|0|,|2|", "model_b_bpb": 2.716183918830647, "model_b_token_boundaries": "|3|,|0|,|2|", "name": "3,0,2" }, { "bucket": "text/non_ascii", "bytes": 42, "delta_bits": -8.54654585944059, "documents": 7, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u2588\u2588\"\"\")\u23ce['\u2420',\u2420'\u2588',\u2420'\u2588']\u23ce", "gap_bpb": -0.20348918712953784, "model_a_bpb": 0.14161296905618306, "model_a_token_boundaries": "|\u2026'|\u2588|',|", "model_b_bpb": 0.34510215618572093, "model_b_token_boundaries": "|\u2026'|\u2588|',|", "name": "'\u2588'," }, { "bucket": "text/number", "bytes": 10, "delta_bits": -8.50900273468782, "documents": 2, "example_dataset": "ppl_circuit_coverage/text_mechanics/line_rhythm", "example_doc_preview": "\u2026and\u2420indent_widths\u2420for\u2420raw\u2420block:\u23ce\u2420\u2420\u2420\u2420banx\u23ce\u2420bybttpq\u23ce\u2420\u2420juehpkg\u23ceresult:\u23ce{\"line_lengths\":[8,8,9],\"indent_widths\":[4,1,2]}\u23ce", "gap_bpb": -0.850900273468782, "model_a_bpb": 2.1314321634307687, "model_a_token_boundaries": "|4|,|1|,|2|", "model_b_bpb": 2.9823324368995507, "model_b_token_boundaries": "|4|,|1|,|2|", "name": "4,1,2" }, { "bucket": "text/non_ascii", "bytes": 6, "delta_bits": -8.461832340024255, "documents": 1, "example_dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "example_doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u25d1\"\"\")\u23ce['\u2420',\u2420'\u25d1']\u23ce", "gap_bpb": -1.4103053900040425, "model_a_bpb": 0.5767172632779228, "model_a_token_boundaries": "|\u2026'|\u25d1|\u25d1|']\u2026|", "model_b_bpb": 1.9870226532819653, "model_b_token_boundaries": "|\u2026'|\u25d1|']\u2026|", "name": "'\u25d1']" } ] }, "top_segments": { "model_a_worse": [ { "bucket": "text/non_ascii", "bytes": 7, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 30.015899104585408, "doc_preview": ">>>\u2420token_chars(\"\"\"\uff0f\uff0f\uff0f\uff0f\uff0f\uff0f\uff0f\uff0f\"\"\")\u23ce['\uff0f',\u2420'\uff0f',\u2420'\uff0f',\u2420'\uff0f',\u2420'\uff0f',\u2420'\uff0f',\u2420'\uff0f',\u2420'\uff0f']\u23ce", "gap_bpb": 4.287985586369344, "text": "['\uff0f'," }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 26.652121135262924, "doc_preview": ">>>\u2420token_chars(\"\"\"\uff86\uff86\uff86\uff86\"\"\")\u23ce['\uff86',\u2420'\uff86',\u2420'\uff86',\u2420'\uff86']\u23ce", "gap_bpb": 8.884040378420975, "text": "\uff86" }, { "bucket": "text/non_ascii", "bytes": 7, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 26.61334575481668, "doc_preview": "token_chars\u2420raw\u2420block\u2420begin\u23ce\uff09\u306f\u23cetoken_chars\u2420raw\u2420block\u2420end\u23cecharacters:\u23ce['\uff09',\u2420'\u306f']\u23ce", "gap_bpb": 3.801906536402383, "text": "['\uff09'," }, { "bucket": "text/non_ascii_word", "bytes": 2, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 25.82697314192486, "doc_preview": ">>>\u2420token_chars(\"\"\"\u0410\u0440\u0445\u0456\u0432\u043e\u0432\u0430\u043d\u043e\"\"\")\u23ce['\u0410',\u2420'\u0440',\u2420'\u0445',\u2420'\u0456',\u2420'\u0432',\u2420'\u043e',\u2420'\u0432',\u2420'\u0430',\u2420'\u043d',\u2420'\u043e']\u23ce", "gap_bpb": 12.91348657096243, "text": "\u0410" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 25.66281363788449, "doc_preview": ">>>\u2420token_chars(\"\"\"\u7f51\u520a\u4e0b\u8f7d\u6b21\u6570\"\"\")\u23ce['\u7f51',\u2420'\u520a',\u2420'\u4e0b',\u2420'\u8f7d',\u2420'\u6b21',\u2420'\u6570']\u23ce", "gap_bpb": 8.554271212628164, "text": "\u7f51" }, { "bucket": "text/non_ascii_word", "bytes": 2, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 25.172112738442735, "doc_preview": ">>>\u2420token_chars(\"\"\"\u045fN\"\"\")\u23ce['\u045f',\u2420'N']\u23ce", "gap_bpb": 12.586056369221367, "text": "\u045f" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 25.15869533995303, "doc_preview": ">>>\u2420token_chars(\"\"\"\u0948.\u23ce\"\"\")\u23ce['\u0948',\u2420'.',\u2420'\\n']\u23ce", "gap_bpb": 8.386231779984344, "text": "\u0948" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 25.04081843105164, "doc_preview": ">>>\u2420token_chars(\"\"\"\u0948\u0902.\u23ce\"\"\")\u23ce['\u0948',\u2420'\u0902',\u2420'.',\u2420'\\n']\u23ce", "gap_bpb": 8.346939477017214, "text": "\u0948" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 25.021570132767085, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\uac24\ub85c\uadf8\ub85c\"\"\")\u23ce['\u2420',\u2420'\uac24',\u2420'\ub85c',\u2420'\uadf8',\u2420'\ub85c']\u23ce", "gap_bpb": 8.340523377589028, "text": "\uac24" }, { "bucket": "text/non_ascii", "bytes": 7, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.953127917029754, "doc_preview": "token_chars\u2420raw\u2420block\u2420begin\u23ce\uff09\u306e\u23cetoken_chars\u2420raw\u2420block\u2420end\u23cecharacters:\u23ce['\uff09',\u2420'\u306e']\u23ce", "gap_bpb": 3.564732559575679, "text": "['\uff09'," }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.949333291821127, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\ub9e4\ub9e4\uac00\"\"\")\u23ce['\u2420',\u2420'\ub9e4',\u2420'\ub9e4',\u2420'\uac00']\u23ce", "gap_bpb": 8.316444430607042, "text": "\ub9e4" }, { "bucket": "text/non_ascii_word", "bytes": 2, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.930140027984805, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u043f\u0440\u043e\u0444\u0435\u0441\u0441\u0438\"\"\")\u23ce['\u2420',\u2420'\u043f',\u2420'\u0440',\u2420'\u043e',\u2420'\u0444',\u2420'\u0435',\u2420'\u0441',\u2420'\u0441',\u2420'\u0438']\u23ce", "gap_bpb": 12.465070013992403, "text": "\u043f" }, { "bucket": "text/non_ascii_word", "bytes": 2, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.750584637039218, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u010ceskosloven\"\"\")\u23ce['\u2420',\u2420'\u010c',\u2420'e',\u2420's',\u2420'k',\u2420'o',\u2420's',\u2420'l',\u2420'o',\u2420'v',\u2420'e',\u2420'n']\u23ce", "gap_bpb": 12.375292318519609, "text": "\u010c" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.662171795883644, "doc_preview": ">>>\u2420token_chars(\"\"\"\u0940)\"\"\")\u23ce['\u0940',\u2420')']\u23ce", "gap_bpb": 8.220723931961215, "text": "\u0940" }, { "bucket": "text/non_ascii", "bytes": 7, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.54252965705382, "doc_preview": ">>>\u2420token_chars(\"\"\"\u30fb\u2501\u30fb\u2501\"\"\")\u23ce['\u30fb',\u2420'\u2501',\u2420'\u30fb',\u2420'\u2501']\u23ce", "gap_bpb": 3.5060756652934026, "text": "['\u30fb'," }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.421225477886065, "doc_preview": ">>>\u2420token_chars(\"\"\"\u0948?\u23ce\"\"\")\u23ce['\u0948',\u2420'?',\u2420'\\n']\u23ce", "gap_bpb": 8.140408492628689, "text": "\u0948" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.31674808127872, "doc_preview": ">>>\u2420token_chars(\"\"\"\u0e4c)\"\"\")\u23ce['\u0e4c',\u2420')']\u23ce", "gap_bpb": 8.105582693759574, "text": "\u0e4c" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.13190112813137, "doc_preview": ">>>\u2420token_chars(\"\"\"\u91cd\u8907\u91cd\u8907\"\"\")\u23ce['\u91cd',\u2420'\u8907',\u2420'\u91cd',\u2420'\u8907']\u23ce", "gap_bpb": 8.043967042710458, "text": "\u91cd" }, { "bucket": "text/non_ascii", "bytes": 7, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.054733544585126, "doc_preview": ">>>\u2420token_chars(\"\"\"\u30fb\u2501\"\"\")\u23ce['\u30fb',\u2420'\u2501']\u23ce", "gap_bpb": 3.4363905063693037, "text": "['\u30fb'," }, { "bucket": "text/non_ascii", "bytes": 6, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 24.011794292178916, "doc_preview": "token_chars\u2420raw\u2420block\u2420begin\u23ce\u00b1\u0638\u23cetoken_chars\u2420raw\u2420block\u2420end\u23cecharacters:\u23ce['\u00b1',\u2420'\u0638']\u23ce", "gap_bpb": 4.001965715363153, "text": "['\u00b1'," }, { "bucket": "text/punctuation", "bytes": 5, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.977094728575533, "doc_preview": ">>>\u2420token_chars(\"\"\"'])){\u23ce\"\"\")\u23ce[\"'\",\u2420']',\u2420')',\u2420')',\u2420'{',\u2420'\\n']\u23ce", "gap_bpb": 4.795418945715107, "text": "[\"'\"," }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.949547235490922, "doc_preview": ">>>\u2420token_chars(\"\"\"\u0947-\"\"\")\u23ce['\u0947',\u2420'-']\u23ce", "gap_bpb": 7.983182411830307, "text": "\u0947" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.880675751057005, "doc_preview": ">>>\u2420token_chars(\"\"\"\uae00\uc0c1\uc704\"\"\")\u23ce['\uae00',\u2420'\uc0c1',\u2420'\uc704']\u23ce", "gap_bpb": 7.960225250352335, "text": "\uae00" }, { "bucket": "text/punctuation", "bytes": 5, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.714394669035613, "doc_preview": ">>>\u2420token_chars(\"\"\"});\u23ce\u23ce\u23ce\"\"\")\u23ce['}',\u2420')',\u2420';',\u2420'\\n',\u2420'\\n',\u2420'\\n']\u23ce", "gap_bpb": 4.742878933807122, "text": "['}'," }, { "bucket": "text/non_ascii_word", "bytes": 2, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.70330385184678, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u043c\u043e\u0432\u0438\"\"\")\u23ce['\u2420',\u2420'\u043c',\u2420'\u043e',\u2420'\u0432',\u2420'\u0438']\u23ce", "gap_bpb": 11.85165192592339, "text": "\u043c" }, { "bucket": "text/punctuation", "bytes": 5, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.688644050687216, "doc_preview": ">>>\u2420token_chars(\"\"\"');\u21b5\u23ce\"\"\")\u23ce[\"'\",\u2420')',\u2420';',\u2420'\\r',\u2420'\\n']\u23ce", "gap_bpb": 4.737728810137443, "text": "[\"'\"," }, { "bucket": "text/non_ascii", "bytes": 7, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.58768903075218, "doc_preview": "token_chars\u2420raw\u2420block\u2420begin\u23ce\u300d\u3068\u23cetoken_chars\u2420raw\u2420block\u2420end\u23cecharacters:\u23ce['\u300d',\u2420'\u3068']\u23ce", "gap_bpb": 3.3696698615360257, "text": "['\u300d'," }, { "bucket": "text/punctuation", "bytes": 5, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.44325094926894, "doc_preview": ">>>\u2420token_chars(\"\"\"})\u23ce\u23ce\u23ce\"\"\")\u23ce['}',\u2420')',\u2420'\\n',\u2420'\\n',\u2420'\\n']\u23ce", "gap_bpb": 4.688650189853788, "text": "['}'," }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.400374986478226, "doc_preview": ">>>\u2420token_chars(\"\"\"\u91b4\u91b4\"\"\")\u23ce['\u91b4',\u2420'\u91b4']\u23ce", "gap_bpb": 7.800124995492742, "text": "\u91b4" }, { "bucket": "text/non_ascii", "bytes": 6, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.304432228919946, "doc_preview": "token_chars\u2420raw\u2420block\u2420begin\u23ce\u00b1\u0637\u23cetoken_chars\u2420raw\u2420block\u2420end\u23cecharacters:\u23ce['\u00b1',\u2420'\u0637']\u23ce", "gap_bpb": 3.8840720381533242, "text": "['\u00b1'," }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.296791899655368, "doc_preview": ">>>\u2420token_chars(\"\"\"\u093e)\"\"\")\u23ce['\u093e',\u2420')']\u23ce", "gap_bpb": 7.765597299885123, "text": "\u093e" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.251272907486186, "doc_preview": ">>>\u2420token_chars(\"\"\"\ub9cc\uc6d0\uc785\ub2c8\ub2e4\"\"\")\u23ce['\ub9cc',\u2420'\uc6d0',\u2420'\uc785',\u2420'\ub2c8',\u2420'\ub2e4']\u23ce", "gap_bpb": 7.750424302495396, "text": "\ub9cc" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.150821280770238, "doc_preview": ">>>\u2420token_chars(\"\"\"\u0948?\"\"\")\u23ce['\u0948',\u2420'?']\u23ce", "gap_bpb": 7.716940426923412, "text": "\u0948" }, { "bucket": "text/punctuation", "bytes": 5, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.14840664435212, "doc_preview": ">>>\u2420token_chars(\"\"\"}))\u23ce\u23ce\"\"\")\u23ce['}',\u2420')',\u2420')',\u2420'\\n',\u2420'\\n']\u23ce", "gap_bpb": 4.629681328870424, "text": "['}'," }, { "bucket": "text/punctuation", "bytes": 5, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.127782484860706, "doc_preview": ">>>\u2420token_chars(\"\"\"}\")\u23ce\u23ce\"\"\")\u23ce['}',\u2420'\"',\u2420')',\u2420'\\n',\u2420'\\n']\u23ce", "gap_bpb": 4.625556496972141, "text": "['}'," }, { "bucket": "text/non_ascii_word", "bytes": 2, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 23.020601520981778, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u0437\u0435\u043c\u043b\u0456\"\"\")\u23ce['\u2420',\u2420'\u0437',\u2420'\u0435',\u2420'\u043c',\u2420'\u043b',\u2420'\u0456']\u23ce", "gap_bpb": 11.510300760490889, "text": "\u0437" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 22.99818186261529, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\ub4f1\ub85d\ub300\ud589\"\"\")\u23ce['\u2420',\u2420'\ub4f1',\u2420'\ub85d',\u2420'\ub300',\u2420'\ud589']\u23ce", "gap_bpb": 7.666060620871764, "text": "\ub4f1" }, { "bucket": "text/non_ascii_word", "bytes": 2, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 22.8909045884519, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u0434\u0438\u0437\u0430\"\"\")\u23ce['\u2420',\u2420'\u0434',\u2420'\u0438',\u2420'\u0437',\u2420'\u0430']\u23ce", "gap_bpb": 11.44545229422595, "text": "\u0434" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 22.755346487180585, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u795e\u9a6c\u6536\u5f55\"\"\")\u23ce['\u2420',\u2420'\u795e',\u2420'\u9a6c',\u2420'\u6536',\u2420'\u5f55']\u23ce", "gap_bpb": 7.585115495726861, "text": "\u795e" }, { "bucket": "text/non_ascii_word", "bytes": 2, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": 22.70859472337008, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u0437\u0434\u0430\u0442\"\"\")\u23ce['\u2420',\u2420'\u0437',\u2420'\u0434',\u2420'\u0430',\u2420'\u0442']\u23ce", "gap_bpb": 11.35429736168504, "text": "\u0437" } ], "model_b_worse": [ { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -22.34676523973404, "doc_preview": ">>>\u2420token_chars(\"\"\"nestjs\"\"\")\u23ce['n',\u2420'e',\u2420's',\u2420't',\u2420'j',\u2420's']\u23ce", "gap_bpb": -22.34676523973404, "text": "j" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -19.706595244827643, "doc_preview": ">>>\u2420token_chars(\"\"\"AccessorType\"\"\")\u23ce['A',\u2420'c',\u2420'c',\u2420'e',\u2420's',\u2420's',\u2420'o',\u2420'r',\u2420'T',\u2420'y',\u2420'p',\u2420'e']\u23ce", "gap_bpb": -19.706595244827643, "text": "o" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -19.0426318963756, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420judiciary\"\"\")\u23ce['\u2420',\u2420'j',\u2420'u',\u2420'd',\u2420'i',\u2420'c',\u2420'i',\u2420'a',\u2420'r',\u2420'y']\u23ce", "gap_bpb": -19.0426318963756, "text": "r" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -18.984533480434127, "doc_preview": ">>>\u2420token_chars(\"\"\"buquerque\"\"\")\u23ce['b',\u2420'u',\u2420'q',\u2420'u',\u2420'e',\u2420'r',\u2420'q',\u2420'u',\u2420'e']\u23ce", "gap_bpb": -18.984533480434127, "text": "q" }, { "bucket": "text/punctuation", "bytes": 5, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -18.49348087647134, "doc_preview": "(?:['(',\u2420'?',\u2420':']\u23ce", "gap_bpb": -3.6986961752942684, "text": "['('," }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -17.850224486408226, "doc_preview": "\u2026_chars(\"\"\"\u2420sourceMappingURL\"\"\")\u23ce['\u2420',\u2420's',\u2420'o',\u2420'u',\u2420'r',\u2420'c',\u2420'e',\u2420'M',\u2420'a',\u2420'p',\u2420'p',\u2420'i',\u2420'n',\u2420'g',\u2420'U',\u2420'R',\u2420'L']\u23ce", "gap_bpb": -17.850224486408226, "text": "p" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -17.551718842837232, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420wannonce\"\"\")\u23ce['\u2420',\u2420'w',\u2420'a',\u2420'n',\u2420'n',\u2420'o',\u2420'n',\u2420'c',\u2420'e']\u23ce", "gap_bpb": -17.551718842837232, "text": "w" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -17.438647908497366, "doc_preview": ">>>\u2420token_chars(\"\"\"iosity\"\"\")\u23ce['i',\u2420'o',\u2420's',\u2420'i',\u2420't',\u2420'y']\u23ce", "gap_bpb": -17.438647908497366, "text": "t" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -17.33860224145431, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420SubLObject\"\"\")\u23ce['\u2420',\u2420'S',\u2420'u',\u2420'b',\u2420'L',\u2420'O',\u2420'b',\u2420'j',\u2420'e',\u2420'c',\u2420't']\u23ce", "gap_bpb": -17.33860224145431, "text": "O" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -16.373938482083343, "doc_preview": ">>>\u2420token_chars(\"\"\"TexCoord\"\"\")\u23ce['T',\u2420'e',\u2420'x',\u2420'C',\u2420'o',\u2420'o',\u2420'r',\u2420'd']\u23ce", "gap_bpb": -16.373938482083343, "text": "C" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -16.178016878116345, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420XCTAssert\"\"\")\u23ce['\u2420',\u2420'X',\u2420'C',\u2420'T',\u2420'A',\u2420's',\u2420's',\u2420'e',\u2420'r',\u2420't']\u23ce", "gap_bpb": -16.178016878116345, "text": "T" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -16.17427956612611, "doc_preview": ">>>\u2420token_chars(\"\"\"ificador\"\"\")\u23ce['i',\u2420'f',\u2420'i',\u2420'c',\u2420'a',\u2420'd',\u2420'o',\u2420'r']\u23ce", "gap_bpb": -16.17427956612611, "text": "f" }, { "bucket": "text/punctuation", "bytes": 4, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -16.15122232507723, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420!!}\"\"\")\u23ce['\u2420',\u2420'!',\u2420'!',\u2420'}']\u23ce", "gap_bpb": -4.037805581269308, "text": "'!'," }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -16.096573289921697, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420innings\"\"\")\u23ce['\u2420',\u2420'i',\u2420'n',\u2420'n',\u2420'i',\u2420'n',\u2420'g',\u2420's']\u23ce", "gap_bpb": -16.096573289921697, "text": "g" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -15.854450438609566, "doc_preview": ">>>\u2420token_chars(\"\"\"toBeInTheDocument\"\"\")\u23ce['t',\u2420'o',\u2420'B',\u2420'e',\u2420'I',\u2420'n',\u2420'T',\u2420'h',\u2420'e',\u2420'D',\u2420'o',\u2420'c',\u2420'u',\u2420'm',\u2420'e',\u2420'\u2026", "gap_bpb": -15.854450438609566, "text": "B" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -15.653688554916677, "doc_preview": ">>>\u2420token_chars(\"\"\"\u21e5RTHOOK\"\"\")\u23ce['\\t',\u2420'R',\u2420'T',\u2420'H',\u2420'O',\u2420'O',\u2420'K']\u23ce", "gap_bpb": -15.653688554916677, "text": "R" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -15.398426917009221, "doc_preview": ">>>\u2420token_chars(\"\"\"erdings\"\"\")\u23ce['e',\u2420'r',\u2420'd',\u2420'i',\u2420'n',\u2420'g',\u2420's']\u23ce", "gap_bpb": -15.398426917009221, "text": "i" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -15.354227203754828, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420astounding\"\"\")\u23ce['\u2420',\u2420'a',\u2420's',\u2420't',\u2420'o',\u2420'u',\u2420'n',\u2420'd',\u2420'i',\u2420'n',\u2420'g']\u23ce", "gap_bpb": -15.354227203754828, "text": "u" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -15.231571875137298, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420everlasting\"\"\")\u23ce['\u2420',\u2420'e',\u2420'v',\u2420'e',\u2420'r',\u2420'l',\u2420'a',\u2420's',\u2420't',\u2420'i',\u2420'n',\u2420'g']\u23ce", "gap_bpb": -15.231571875137298, "text": "l" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -15.199615617546755, "doc_preview": ">>>\u2420token_chars(\"\"\"uento\"\"\")\u23ce['u',\u2420'e',\u2420'n',\u2420't',\u2420'o']\u23ce", "gap_bpb": -15.199615617546755, "text": "o" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -15.157252721999251, "doc_preview": ">>>\u2420token_chars(\"\"\"ibbean\"\"\")\u23ce['i',\u2420'b',\u2420'b',\u2420'e',\u2420'a',\u2420'n']\u23ce", "gap_bpb": -15.157252721999251, "text": "b" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -15.010869473739048, "doc_preview": "\u2026\u2420token_chars(\"\"\"TouchableOpacity\"\"\")\u23ce['T',\u2420'o',\u2420'u',\u2420'c',\u2420'h',\u2420'a',\u2420'b',\u2420'l',\u2420'e',\u2420'O',\u2420'p',\u2420'a',\u2420'c',\u2420'i',\u2420't',\u2420'y']\u23ce", "gap_bpb": -15.010869473739048, "text": "a" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.911586512131363, "doc_preview": ">>>\u2420token_chars(\"\"\"arParams\"\"\")\u23ce['a',\u2420'r',\u2420'P',\u2420'a',\u2420'r',\u2420'a',\u2420'm',\u2420's']\u23ce", "gap_bpb": -14.911586512131363, "text": "P" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.828100629957717, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u3147\u3147\"\"\")\u23ce['\u2420',\u2420'\u3147',\u2420'\u3147']\u23ce", "gap_bpb": -4.942700209985905, "text": "\u3147" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.812444877267035, "doc_preview": ">>>\u2420token_chars(\"\"\"embrance\"\"\")\u23ce['e',\u2420'm',\u2420'b',\u2420'r',\u2420'a',\u2420'n',\u2420'c',\u2420'e']\u23ce", "gap_bpb": -14.812444877267035, "text": "n" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.601341848760468, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u0e1e\u0e24\u0e29\u0e20\u0e32\u0e04\u0e21\"\"\")\u23ce['\u2420',\u2420'\u0e1e',\u2420'\u0e24',\u2420'\u0e29',\u2420'\u0e20',\u2420'\u0e32',\u2420'\u0e04',\u2420'\u0e21']\u23ce", "gap_bpb": -4.867113949586822, "text": "\u0e20" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.587447886345172, "doc_preview": ">>>\u2420token_chars(\"\"\"igInteger\"\"\")\u23ce['i',\u2420'g',\u2420'I',\u2420'n',\u2420't',\u2420'e',\u2420'g',\u2420'e',\u2420'r']\u23ce", "gap_bpb": -14.587447886345172, "text": "g" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.573455033905674, "doc_preview": ">>>\u2420token_chars(\"\"\"tridges\"\"\")\u23ce['t',\u2420'r',\u2420'i',\u2420'd',\u2420'g',\u2420'e',\u2420's']\u23ce", "gap_bpb": -14.573455033905674, "text": "g" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.41686362641653, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420\u3161\"\"\")\u23ce['\u2420',\u2420'\u3161']\u23ce", "gap_bpb": -4.80562120880551, "text": "\u3161" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.412925309703107, "doc_preview": "\u2026_chars(\"\"\"\u2420TouchableOpacity\"\"\")\u23ce['\u2420',\u2420'T',\u2420'o',\u2420'u',\u2420'c',\u2420'h',\u2420'a',\u2420'b',\u2420'l',\u2420'e',\u2420'O',\u2420'p',\u2420'a',\u2420'c',\u2420'i',\u2420't',\u2420'y']\u23ce", "gap_bpb": -14.412925309703107, "text": "a" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.372839335466017, "doc_preview": ">>>\u2420token_chars(\"\"\"ISyntaxException\"\"\")\u23ce['I',\u2420'S',\u2420'y',\u2420'n',\u2420't',\u2420'a',\u2420'x',\u2420'E',\u2420'x',\u2420'c',\u2420'e',\u2420'p',\u2420't',\u2420'i',\u2420'o',\u2420'n\u2026", "gap_bpb": -14.372839335466017, "text": "I" }, { "bucket": "text/non_ascii_word", "bytes": 2, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.341172341945299, "doc_preview": ">>>\u2420token_chars(\"\"\"ket\u00f8y\"\"\")\u23ce['k',\u2420'e',\u2420't',\u2420'\u00f8',\u2420'y']\u23ce", "gap_bpb": -7.1705861709726495, "text": "\u00f8" }, { "bucket": "text/punctuation", "bytes": 5, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.335232061184499, "doc_preview": "]+\"[']',\u2420'+',\u2420'\"']\u23ce", "gap_bpb": -2.8670464122369, "text": "[']'," }, { "bucket": "text/punctuation", "bytes": 4, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.277906975981557, "doc_preview": ">>>\u2420token_chars(\"\"\"')?>',\u2420'<',\u2420'/']\u23ce", "gap_bpb": -3.569476743995389, "text": "'<'," }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.260321749930835, "doc_preview": ">>>\u2420token_chars(\"\"\"m\u0131z\u0131\"\"\")\u23ce['m',\u2420'\u0131',\u2420'z',\u2420'\u0131']\u23ce", "gap_bpb": -14.260321749930835, "text": "m" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.215481401301979, "doc_preview": ">>>\u2420token_chars(\"\"\"\u2420DJs\"\"\")\u23ce['\u2420',\u2420'D',\u2420'J',\u2420's']\u23ce", "gap_bpb": -14.215481401301979, "text": "J" }, { "bucket": "text/non_ascii_word", "bytes": 3, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.210298059183266, "doc_preview": ">>>\u2420token_chars(\"\"\"\u0e15\u0e23\u0e07\"\"\")\u23ce['\u0e15',\u2420'\u0e23',\u2420'\u0e07']\u23ce", "gap_bpb": -4.736766019727756, "text": "\u0e07" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.072371885796622, "doc_preview": ">>>\u2420token_chars(\"\"\"ordial\"\"\")\u23ce['o',\u2420'r',\u2420'd',\u2420'i',\u2420'a',\u2420'l']\u23ce", "gap_bpb": -14.072371885796622, "text": "a" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -14.016524045573268, "doc_preview": ">>>\u2420token_chars(\"\"\"addafi\"\"\")\u23ce['a',\u2420'd',\u2420'd',\u2420'a',\u2420'f',\u2420'i']\u23ce", "gap_bpb": -14.016524045573268, "text": "f" }, { "bucket": "text/word", "bytes": 1, "dataset": "ppl_circuit_coverage/vocab_mechanics/token_chars", "delta_bits": -13.989825372861999, "doc_preview": ">>>\u2420token_chars(\"\"\"caffold\"\"\")\u23ce['c',\u2420'a',\u2420'f',\u2420'f',\u2420'o',\u2420'l',\u2420'd']\u23ce", "gap_bpb": -13.989825372861999, "text": "o" } ] } }