tokenizer.vocab
10.1 KB · 1025 lines · plaintext Raw
1 <unk> 0
2 ▁t -0
3 ▁th -1
4 ▁a -2
5 in -3
6 ▁the -4
7 re -5
8 ▁w -6
9 ▁o -7
10 ▁s -8
11 at -9
12 ou -10
13 er -11
14 nd -12
15 ▁i -13
16 ▁b -14
17 ▁c -15
18 on -16
19 ▁h -17
20 ing -18
21 ▁to -19
22 ▁m -20
23 en -21
24 ▁f -22
25 ▁p -23
26 an -24
27 ▁d -25
28 es -26
29 or -27
30 ll -28
31 ▁of -29
32 ▁and -30
33 ▁y -31
34 ▁l -32
35 ▁I -33
36 it -34
37 ▁in -35
38 is -36
39 ed -37
40 ▁g -38
41 ▁you -39
42 ar -40
43 ▁that -41
44 om -42
45 as -43
46 ▁n -44
47 ve -45
48 us -46
49 ic -47
50 ow -48
51 al -49
52 ▁it -50
53 ▁be -51
54 ▁wh -52
55 le -53
56 ion -54
57 ut -55
58 ot -56
59 ▁we -57
60 ▁is -58
61 ▁e -59
62 et -60
63 ay -61
64 ▁re -62
65 ▁on -63
66 ▁T -64
67 ▁A -65
68 ▁ha -66
69 ent -67
70 ke -68
71 ct -69
72 ▁S -70
73 ig -71
74 ver -72
75 ▁Th -73
76 all -74
77 id -75
78 ▁for -76
79 ro -77
80 ▁he -78
81 se -79
82 ▁this -80
83 ld -81
84 ly -82
85 ▁go -83
86 ▁k -84
87 ▁st -85
88 st -86
89 ch -87
90 ▁li -88
91 ▁u -89
92 am -90
93 ur -91
94 ce -92
95 ith -93
96 im -94
97 ▁so -95
98 ▁have -96
99 ▁do -97
100 ht -98
101 th -99
102 ▁an -100
103 ▁with -101
104 ad -102
105 ▁r -103
106 ir -104
107 ▁was -105
108 ▁as -106
109 ▁W -107
110 ▁are -108
111 ust -109
112 ally -110
113 ▁j -111
114 ▁se -112
115 ation -113
116 od -114
117 ere -115
118 ▁like -116
119 ▁not -117
120 ▁kn -118
121 ight -119
122 ▁B -120
123 ▁they -121
124 ▁And -122
125 ▁know -123
126 ome -124
127 op -125
128 ▁can -126
129 ▁or -127
130 ▁sh -128
131 ▁me -129
132 ill -130
133 ant -131
134 ck -132
135 ▁what -133
136 ▁at -134
137 ▁ab -135
138 ould -136
139 ol -137
140 ▁So -138
141 ▁C -139
142 use -140
143 ter -141
144 il -142
145 ▁but -143
146 ▁just -144
147 ▁ne -145
148 ▁de -146
149 ra -147
150 ore -148
151 ▁there -149
152 ul -150
153 out -151
154 ▁con -152
155 ▁all -153
156 ▁The -154
157 ers -155
158 ▁H -156
159 ▁fr -157
160 ▁pro -158
161 ge -159
162 ea -160
163 ▁Y -161
164 ▁O -162
165 ▁M -163
166 pp -164
167 ▁com -165
168 ess -166
169 ▁ch -167
170 ▁al -168
171 est -169
172 ate -170
173 qu -171
174 ▁lo -172
175 ▁ex -173
176 very -174
177 ▁su -175
178 ain -176
179 ▁one -177
180 ca -178
181 art -179
182 ist -180
183 if -181
184 ive -182
185 ▁if -183
186 ink -184
187 nt -185
188 ab -186
189 ▁about -187
190 ▁going -188
191 ▁v -189
192 ▁wor -190
193 um -191
194 ok -192
195 ▁your -193
196 ▁my -194
197 ind -195
198 ▁get -196
199 cause -197
200 ▁from -198
201 ▁don -199
202 ri -200
203 pe -201
204 un -202
205 ity -203
206 ▁up -204
207 ▁P -205
208 ▁out -206
209 ort -207
210 ▁L -208
211 ment -209
212 el -210
213 ▁N -211
214 ▁some -212
215 ich -213
216 and -214
217 ▁think -215
218 em -216
219 oug -217
220 ▁G -218
221 os -219
222 ▁D -220
223 res -221
224 ▁because -222
225 ▁by -223
226 ake -224
227 ▁int -225
228 ie -226
229 ▁us -227
230 ▁tr -228
231 ▁then -229
232 ack -230
233 ▁pl -231
234 ▁here -232
235 ▁pe -233
236 her -234
237 ▁will -235
238 ▁F -236
239 ▁which -237
240 ard -238
241 ▁right -239
242 ▁thing -240
243 ▁want -241
244 ies -242
245 ople -243
246 ▁It -244
247 ▁them -245
248 ame -246
249 ▁We -247
250 our -248
251 ▁say -249
252 ▁R -250
253 ▁people -251
254 ▁see -252
255 ▁who -253
256 ast -254
257 ure -255
258 ect -256
259 ear -257
260 ▁tim -258
261 ▁E -259
262 ▁You -260
263 ▁would -261
264 ▁when -262
265 ven -263
266 ▁our -264
267 ci -265
268 ▁really -266
269 ▁more -267
270 ound -268
271 ose -269
272 ak -270
273 ▁co -271
274 ide -272
275 ough -273
276 ▁had -274
277 so -275
278 ▁qu -276
279 eah -277
280 ▁were -278
281 ine -279
282 ▁act -280
283 ther -281
284 ▁these -282
285 ▁how -283
286 ▁now -284
287 ▁sa -285
288 ud -286
289 ▁Wh -287
290 ▁man -288
291 ous -289
292 one -290
293 pt -291
294 ff -292
295 ong -293
296 ▁has -294
297 ▁any -295
298 ▁very -296
299 ▁But -297
300 ▁look -298
301 iv -299
302 itt -300
303 ▁time -301
304 ▁mo -302
305 ▁ar -303
306 hing -304
307 ▁le -305
308 ▁work -306
309 ▁their -307
310 are -308
311 ▁his -309
312 per -310
313 ions -311
314 ▁im -312
315 ▁ag -313
316 ▁J -314
317 ▁no -315
318 ▁en -316
319 ▁got -317
320 ag -318
321 ▁sp -319
322 ans -320
323 act -321
324 ▁te -322
325 ▁also -323
326 iz -324
327 ice -325
328 ▁That -326
329 ▁cl -327
330 ▁been -328
331 ▁way -329
332 ▁fe -330
333 ▁did -331
334 ple -332
335 ually -333
336 ▁other -334
337 ▁U -335
338 ite -336
339 age -337
340 omet -338
341 ber -339
342 reat -340
343 ree -341
344 ▁into -342
345 own -343
346 ▁tw -344
347 ▁part -345
348 alk -346
349 ▁where -347
350 ▁need -348
351 ▁every -349
352 pl -350
353 ▁ad -351
354 ry -352
355 ▁over -353
356 ble -354
357 ap -355
358 ue -356
359 ▁kind -357
360 ▁po -358
361 ▁back -359
362 ▁cont -360
363 iff -361
364 ▁somet -362
365 ▁pr -363
366 nder -364
367 ire -365
368 ▁good -366
369 ▁than -367
370 ace -368
371 ▁gu -369
372 ep -370
373 og -371
374 ick -372
375 way -373
376 ▁lot -374
377 ▁un -375
378 ▁things -376
379 ▁In -377
380 ish -378
381 kay -379
382 ▁well -380
383 ▁could -381
384 ▁pre -382
385 ▁two -383
386 irst -384
387 ▁diff -385
388 ach -386
389 cc -387
390 ittle -388
391 int -389
392 ▁He -390
393 ▁those -391
394 ence -392
395 ip -393
396 ase -394
397 ▁him -395
398 ▁make -396
399 ▁little -397
400 ical -398
401 ▁gr -399
402 ▁year -400
403 ass -401
404 ▁thr -402
405 uch -403
406 ated -404
407 ▁This -405
408 ▁off -406
409 ▁res -407
410 ac -408
411 ance -409
412 ▁actually -410
413 ▁talk -411
414 ult -412
415 able -413
416 orm -414
417 ▁dis -415
418 ▁first -416
419 ations -417
420 ▁something -418
421 ▁she -419
422 sel -420
423 ▁let -421
424 ord -422
425 ▁may -423
426 ia -424
427 ▁am -425
428 ▁her -426
429 ▁said -427
430 ▁bo -428
431 be -429
432 ount -430
433 ▁much -431
434 ▁per -432
435 ▁even -433
436 ▁differe -434
437 vel -435
438 ary -436
439 ▁app -437
440 ving -438
441 ▁comm -439
442 ▁imp -440
443 ys -441
444 ▁again -442
445 ress -443
446 ▁yeah -444
447 ▁down -445
448 ang -446
449 ▁mean -447
450 na -448
451 ens -449
452 ▁does -450
453 ▁fo -451
454 ▁comp -452
455 ▁ro -453
456 ▁bl -454
457 ody -455
458 ▁K -456
459 ▁through -457
460 ▁start -458
461 uct -459
462 ▁only -460
463 ▁bet -461
464 ▁under -462
465 ▁br -463
466 ▁take -464
467 ning -465
468 ▁bu -466
469 ▁use -467
470 ▁Ch -468
471 xt -469
472 co -470
473 ory -471
474 ild -472
475 ▁put -473
476 ▁call -474
477 ▁new -475
478 other -476
479 ting -477
480 ▁happ -478
481 ater -479
482 ▁inc -480
483 ition -481
484 ▁different -482
485 ▁should -483
486 ade -484
487 ign -485
488 thing -486
489 ▁day -487
490 fore -488
491 ▁Yeah -489
492 ark -490
493 ile -491
494 ial -492
495 ▁come -493
496 ▁They -494
497 ▁being -495
498 ▁try -496
499 ious -497
500 ▁sc -498
501 ▁bit -499
502 ▁spe -500
503 ub -501
504 fe -502
505 ▁doing -503
506 ▁St -504
507 vers -505
508 av -506
509 ty -507
510 ian -508
511 onna -509
512 red -510
513 wn -511
514 ▁ke -512
515 form -513
516 ors -514
517 ▁fl -515
518 fter -516
519 ail -517
520 ents -518
521 ▁gonna -519
522 ▁point -520
523 ces -521
524 ▁There -522
525 self -523
526 ▁many -524
527 ▁If -525
528 ▁same -526
529 ▁sy -527
530 ▁quest -528
531 ▁most -529
532 ▁great -530
533 ▁What -531
534 ▁fu -532
535 ug -533
536 ▁show -534
537 we -535
538 ual -536
539 ons -537
540 ▁Be -538
541 ically -539
542 ▁ser -540
543 ▁rem -541
544 ▁ind -542
545 ▁pers -543
546 ▁V -544
547 he -545
548 ▁str -546
549 ved -547
550 ▁still -548
551 ank -549
552 ▁rec -550
553 ▁wr -551
554 ought -552
555 day -553
556 ath -554
557 ▁end -555
558 ▁bas -556
559 ft -557
560 erm -558
561 body -559
562 ph -560
563 ject -561
564 ict -562
565 ▁play -563
566 ▁Is -564
567 ates -565
568 ▁ph -566
569 oth -567
570 ▁acc -568
571 get -569
572 ▁years -570
573 ▁em -571
574 ▁id -572
575 ▁Oh -573
576 ves -574
577 ever -575
578 ▁inter -576
579 ▁rel -577
580 ▁before -578
581 ▁feel -579
582 igh -580
583 ▁three -581
584 iss -582
585 ▁des -583
586 ne -584
587 ▁why -585
588 ▁uh -586
589 ▁To -587
590 ▁cons -588
591 ▁hel -589
592 ▁after -590
593 ower -591
594 urn -592
595 ▁okay -593
596 ▁long -594
597 ▁bel -595
598 ▁around -596
599 ful -597
600 te -598
601 ise -599
602 ▁ob -600
603 ▁supp -601
604 ady -602
605 ange -603
606 aking -604
607 ▁pos -605
608 atch -606
609 ▁tra -607
610 gr -608
611 ▁might -609
612 ert -610
613 ▁help -611
614 ost -612
615 ▁too -613
616 cial -614
617 ▁world -615
618 ▁give -616
619 ike -617
620 ▁Okay -618
621 ways -619
622 ▁min -620
623 ward -621
624 ily -622
625 ▁gen -623
626 ▁find -624
627 ▁dec -625
628 ular -626
629 ob -627
630 ▁tell -628
631 ▁Now -629
632 ▁sm -630
633 ▁cour -631
634 ▁real -632
635 cess -633
636 nds -634
637 ▁big -635
638 ▁num -636
639 ction -637
640 ▁add -638
641 ▁set -639
642 ▁um -640
643 ood -641
644 ible -642
645 ▁own -643
646 ▁life -644
647 ities -645
648 ▁its -646
649 ▁God -647
650 pect -648
651 ▁didn -649
652 stem -650
653 les -651
654 uc -652
655 ib -653
656 ating -654
657 olog -655
658 ▁person -656
659 ▁inv -657
660 ably -658
661 ▁sure -659
662 ▁reg -660
663 lic -661
664 ▁stu -662
665 ▁cr -663
666 ▁ev -664
667 ments -665
668 ▁another -666
669 ▁la -667
670 ▁last -668
671 ▁sub -669
672 ▁att -670
673 ▁op -671
674 ▁inst -672
675 ▁sl -673
676 ▁happen -674
677 ▁rep -675
678 ▁import -676
679 ific -677
680 ix -678
681 ▁made -679
682 ▁ear -680
683 ▁ac -681
684 ▁def -682
685 ute -683
686 ▁next -684
687 ative -685
688 ▁form -686
689 ▁guys -687
690 ▁system -688
691 ew -689
692 ▁able -690
693 ied -691
694 ▁always -692
695 ren -693
696 erest -694
697 ▁As -695
698 ▁mod -696
699 ▁done -697
700 ings -698
701 ▁love -699
702 ism -700
703 ▁ask -701
704 old -702
705 ered -703
706 ▁trans -704
707 ▁count -705
708 ility -706
709 ▁high -707
710 ▁fin -708
711 ▁prob -709
712 ▁pol -710
713 ▁exam -711
714 ▁pres -712
715 ▁maybe -713
716 ell -714
717 ▁stud -715
718 ▁prod -716
719 ▁car -717
720 ock -718
721 ▁used -719
722 oy -720
723 stand -721
724 ▁No -722
725 ▁mon -723
726 ks -724
727 ▁interest -725
728 ▁ent -726
729 ited -727
730 ▁sort -728
731 ▁For -729
732 ▁today -730
733 ics -731
734 ▁vide -732
735 ▁bec -733
736 ▁Well -734
737 ▁Al -735
738 ▁important -736
739 ▁such -737
740 ▁run -738
741 ▁keep -739
742 ▁fact -740
743 ata -741
744 ss -742
745 ▁never -743
746 ween -744
747 ▁stuff -745
748 ract -746
749 ▁question -747
750 als -748
751 ▁sim -749
752 vern -750
753 ather -751
754 ▁course -752
755 ▁Of -753
756 oc -754
757 ness -755
758 arch -756
759 ize -757
760 ▁All -758
761 ense -759
762 blem -760
763 ▁probably -761
764 hip -762
765 ▁number -763
766 ention -764
767 ▁saying -765
768 ▁commun -766
769 ▁An -767
770 akes -768
771 ▁belie -769
772 ▁between -770
773 ▁better -771
774 cus -772
775 ▁place -773
776 ▁gener -774
777 ▁ca -775
778 ▁ins -776
779 ▁ass -777
780 cond -778
781 cept -779
782 ull -780
783 ▁understand -781
784 ▁fun -782
785 ▁thought -783
786 gan -784
787 iew -785
788 cy -786
789 ution -787
790 ope -788
791 ason -789
792 ▁problem -790
793 ▁doesn -791
794 ational -792
795 ▁read -793
796 ▁trying -794
797 ▁sch -795
798 ▁el -796
799 ah -797
800 atter -798
801 ▁exper -799
802 ▁four -800
803 ▁ele -801
804 ▁cou -802
805 ont -803
806 ▁called -804
807 ▁partic -805
808 ▁open -806
809 ▁gl -807
810 ▁everything -808
811 ▁eff -809
812 ▁getting -810
813 ▁ty -811
814 ▁Am -812
815 ▁Because -813
816 ave -814
817 ▁met -815
818 ▁Like -816
819 oney -817
820 ▁ -818
821 e -819
822 t -820
823 o -821
824 a -822
825 n -823
826 i -824
827 s -825
828 h -826
829 r -827
830 l -828
831 d -829
832 u -830
833 c -831
834 y -832
835 m -833
836 g -834
837 w -835
838 f -836
839 p -837
840 , -838
841 b -839
842 . -840
843 k -841
844 v -842
845 ' -843
846 I -844
847 T -845
848 A -846
849 S -847
850 j -848
851 x -849
852 W -850
853 B -851
854 C -852
855 ? -853
856 0 -854
857 O -855
858 - -856
859 M -857
860 H -858
861 Y -859
862 q -860
863 1 -861
864 P -862
865 z -863
866 L -864
867 D -865
868 N -866
869 G -867
870 F -868
871 R -869
872 E -870
873 2 -871
874 J -872
875 U -873
876 : -874
877 5 -875
878 9 -876
879 3 -877
880 K -878
881 4 -879
882 V -880
883 8 -881
884 6 -882
885 7 -883
886 ! -884
887 % -885
888 Q -886
889 $ -887
890 Z -888
891 X -889
892 é -890
893 / -891
894 í -892
895 á -893
896 £ -894
897 ó -895
898 ā -896
899 ü -897
900 ñ -898
901 ö -899
902 è -900
903 ç -901
904 à -902
905 ¿ -903
906 μ -904
907 π -905
908 ä -906
909 ú -907
910 θ -908
911 ã -909
912 φ -910
913 ī -911
914 σ -912
915 ê -913
916 ρ -914
917 â -915
918 ô -916
919 ^ -917
920 € -918
921 É -919
922 ū -920
923 Δ -921
924 λ -922
925 α -923
926 τ -924
927 æ -925
928 а -926
929 о -927
930 ν -928
931 î -929
932 γ -930
933 ψ -931
934 ē -932
935 т -933
936 ß -934
937 ω -935
938 ï -936
939 ć -937
940 č -938
941 ε -939
942 е -940
943 и -941
944 ò -942
945 р -943
946 β -944
947 ø -945
948 ł -946
949 δ -947
950 η -948
951 п -949
952 ë -950
953 н -951
954 с -952
955 š -953
956 Ü -954
957 å -955
958 ń -956
959 ś -957
960 я -958
961 đ -959
962 л -960
963 м -961
964 Ö -962
965 û -963
966 ș -964
967 в -965
968 Á -966
969 Ø -967
970 ù -968
971 ο -969
972 ч -970
973 ь -971
974 ž -972
975 Φ -973
976 у -974
977 ę -975
978 ι -976
979 б -977
980 г -978
981 к -979
982 ő -980
983 Ś -981
984 Ω -982
985 κ -983
986 υ -984
987 ì -985
988 Č -986
989 έ -987
990 х -988
991 ы -989
992 Å -990
993 Ç -991
994 ż -992
995 ί -993
996 ζ -994
997 χ -995
998 э -996
999 Æ -997
1000 Í -998
1001 õ -999
1002 ě -1000
1003 ħ -1001
1004 Ł -1002
1005 œ -1003
1006 Ž -1004
1007 ț -1005
1008 Γ -1006
1009 П -1007
1010 д -1008
1011 з -1009
1012 ф -1010
1013 ¡ -1011
1014 À -1012
1015 Î -1013
1016 Ā -1014
1017 ė -1015
1018 Š -1016
1019 ź -1017
1020 Κ -1018
1021 Ψ -1019
1022 ά -1020
1023 ξ -1021
1024 ό -1022
1025