diff --git a/resources/tables/CR_Age_14_0.bin b/resources/tables/CR_Age_14_0.bin new file mode 100644 index 0000000..30faf1f Binary files /dev/null and b/resources/tables/CR_Age_14_0.bin differ diff --git a/resources/tables/CR_Age_15_0.bin b/resources/tables/CR_Age_15_0.bin new file mode 100644 index 0000000..f3aaf07 Binary files /dev/null and b/resources/tables/CR_Age_15_0.bin differ diff --git a/resources/tables/CR_Ahom.bin b/resources/tables/CR_Ahom.bin index 7a42287..9559430 100644 Binary files a/resources/tables/CR_Ahom.bin and b/resources/tables/CR_Ahom.bin differ diff --git a/resources/tables/CR_Alnum.bin b/resources/tables/CR_Alnum.bin index 0d05c34..83f5220 100644 Binary files a/resources/tables/CR_Alnum.bin and b/resources/tables/CR_Alnum.bin differ diff --git a/resources/tables/CR_Alpha.bin b/resources/tables/CR_Alpha.bin index b8a93c2..117fdc9 100644 Binary files a/resources/tables/CR_Alpha.bin and b/resources/tables/CR_Alpha.bin differ diff --git a/resources/tables/CR_Arabic.bin b/resources/tables/CR_Arabic.bin index 26666f5..b7a3b84 100644 Binary files a/resources/tables/CR_Arabic.bin and b/resources/tables/CR_Arabic.bin differ diff --git a/resources/tables/CR_Assigned.bin b/resources/tables/CR_Assigned.bin index 65c8c14..3eeb82e 100644 Binary files a/resources/tables/CR_Assigned.bin and b/resources/tables/CR_Assigned.bin differ diff --git a/resources/tables/CR_Balinese.bin b/resources/tables/CR_Balinese.bin index b4c8cf4..96460d2 100644 Binary files a/resources/tables/CR_Balinese.bin and b/resources/tables/CR_Balinese.bin differ diff --git a/resources/tables/CR_Brahmi.bin b/resources/tables/CR_Brahmi.bin index cedccff..a9b8617 100644 Binary files a/resources/tables/CR_Brahmi.bin and b/resources/tables/CR_Brahmi.bin differ diff --git a/resources/tables/CR_C.bin b/resources/tables/CR_C.bin index ced4d6c..3a41f40 100644 Binary files a/resources/tables/CR_C.bin and b/resources/tables/CR_C.bin differ diff --git a/resources/tables/CR_Canadian_Aboriginal.bin b/resources/tables/CR_Canadian_Aboriginal.bin index 7b9e122..83bef94 100644 Binary files a/resources/tables/CR_Canadian_Aboriginal.bin and b/resources/tables/CR_Canadian_Aboriginal.bin differ diff --git a/resources/tables/CR_Case_Ignorable.bin b/resources/tables/CR_Case_Ignorable.bin index 8731435..03c5a57 100644 Binary files a/resources/tables/CR_Case_Ignorable.bin and b/resources/tables/CR_Case_Ignorable.bin differ diff --git a/resources/tables/CR_Cased.bin b/resources/tables/CR_Cased.bin index 7599c2c..daa2be2 100644 Binary files a/resources/tables/CR_Cased.bin and b/resources/tables/CR_Cased.bin differ diff --git a/resources/tables/CR_Cf.bin b/resources/tables/CR_Cf.bin index bbe8747..d91a2a7 100644 Binary files a/resources/tables/CR_Cf.bin and b/resources/tables/CR_Cf.bin differ diff --git a/resources/tables/CR_Changes_When_Casefolded.bin b/resources/tables/CR_Changes_When_Casefolded.bin index 44b80e2..f2dfd97 100644 Binary files a/resources/tables/CR_Changes_When_Casefolded.bin and b/resources/tables/CR_Changes_When_Casefolded.bin differ diff --git a/resources/tables/CR_Changes_When_Casemapped.bin b/resources/tables/CR_Changes_When_Casemapped.bin index 9393bd6..d785833 100644 Binary files a/resources/tables/CR_Changes_When_Casemapped.bin and b/resources/tables/CR_Changes_When_Casemapped.bin differ diff --git a/resources/tables/CR_Changes_When_Lowercased.bin b/resources/tables/CR_Changes_When_Lowercased.bin index a6eeb60..61fc097 100644 Binary files a/resources/tables/CR_Changes_When_Lowercased.bin and b/resources/tables/CR_Changes_When_Lowercased.bin differ diff --git a/resources/tables/CR_Changes_When_Titlecased.bin b/resources/tables/CR_Changes_When_Titlecased.bin index ddabc63..74f386d 100644 Binary files a/resources/tables/CR_Changes_When_Titlecased.bin and b/resources/tables/CR_Changes_When_Titlecased.bin differ diff --git a/resources/tables/CR_Changes_When_Uppercased.bin b/resources/tables/CR_Changes_When_Uppercased.bin index f4a0cc2..c7b0ed1 100644 Binary files a/resources/tables/CR_Changes_When_Uppercased.bin and b/resources/tables/CR_Changes_When_Uppercased.bin differ diff --git a/resources/tables/CR_Cn.bin b/resources/tables/CR_Cn.bin index 1c64620..c6eeee6 100644 Binary files a/resources/tables/CR_Cn.bin and b/resources/tables/CR_Cn.bin differ diff --git a/resources/tables/CR_Common.bin b/resources/tables/CR_Common.bin index 0ef91f6..a78a02b 100644 Binary files a/resources/tables/CR_Common.bin and b/resources/tables/CR_Common.bin differ diff --git a/resources/tables/CR_Cypro_Minoan.bin b/resources/tables/CR_Cypro_Minoan.bin new file mode 100644 index 0000000..3df632d Binary files /dev/null and b/resources/tables/CR_Cypro_Minoan.bin differ diff --git a/resources/tables/CR_Cyrillic.bin b/resources/tables/CR_Cyrillic.bin index b3ace34..eebdc4c 100644 Binary files a/resources/tables/CR_Cyrillic.bin and b/resources/tables/CR_Cyrillic.bin differ diff --git a/resources/tables/CR_Dash.bin b/resources/tables/CR_Dash.bin index 4deb90c..8c9eded 100644 Binary files a/resources/tables/CR_Dash.bin and b/resources/tables/CR_Dash.bin differ diff --git a/resources/tables/CR_Default_Ignorable_Code_Point.bin b/resources/tables/CR_Default_Ignorable_Code_Point.bin index c887540..7da689e 100644 Binary files a/resources/tables/CR_Default_Ignorable_Code_Point.bin and b/resources/tables/CR_Default_Ignorable_Code_Point.bin differ diff --git a/resources/tables/CR_Devanagari.bin b/resources/tables/CR_Devanagari.bin index e1ec463..76412b5 100644 Binary files a/resources/tables/CR_Devanagari.bin and b/resources/tables/CR_Devanagari.bin differ diff --git a/resources/tables/CR_Diacritic.bin b/resources/tables/CR_Diacritic.bin index e3e422c..55d9214 100644 Binary files a/resources/tables/CR_Diacritic.bin and b/resources/tables/CR_Diacritic.bin differ diff --git a/resources/tables/CR_Digit.bin b/resources/tables/CR_Digit.bin index 18b73ce..a8aac6e 100644 Binary files a/resources/tables/CR_Digit.bin and b/resources/tables/CR_Digit.bin differ diff --git a/resources/tables/CR_Egyptian_Hieroglyphs.bin b/resources/tables/CR_Egyptian_Hieroglyphs.bin index c067f40..b2940db 100644 Binary files a/resources/tables/CR_Egyptian_Hieroglyphs.bin and b/resources/tables/CR_Egyptian_Hieroglyphs.bin differ diff --git a/resources/tables/CR_Emoji.bin b/resources/tables/CR_Emoji.bin index df1327f..e706bc5 100644 Binary files a/resources/tables/CR_Emoji.bin and b/resources/tables/CR_Emoji.bin differ diff --git a/resources/tables/CR_Emoji_Modifier_Base.bin b/resources/tables/CR_Emoji_Modifier_Base.bin index fa63f98..7f93456 100644 Binary files a/resources/tables/CR_Emoji_Modifier_Base.bin and b/resources/tables/CR_Emoji_Modifier_Base.bin differ diff --git a/resources/tables/CR_Emoji_Presentation.bin b/resources/tables/CR_Emoji_Presentation.bin index 9989ea2..b4ef3f6 100644 Binary files a/resources/tables/CR_Emoji_Presentation.bin and b/resources/tables/CR_Emoji_Presentation.bin differ diff --git a/resources/tables/CR_Ethiopic.bin b/resources/tables/CR_Ethiopic.bin index bc89b84..7c3620d 100644 Binary files a/resources/tables/CR_Ethiopic.bin and b/resources/tables/CR_Ethiopic.bin differ diff --git a/resources/tables/CR_Extender.bin b/resources/tables/CR_Extender.bin index 9389687..372b01c 100644 Binary files a/resources/tables/CR_Extender.bin and b/resources/tables/CR_Extender.bin differ diff --git a/resources/tables/CR_Glagolitic.bin b/resources/tables/CR_Glagolitic.bin index 183347a..a530a0a 100644 Binary files a/resources/tables/CR_Glagolitic.bin and b/resources/tables/CR_Glagolitic.bin differ diff --git a/resources/tables/CR_Graph.bin b/resources/tables/CR_Graph.bin index 18017c8..7032a1a 100644 Binary files a/resources/tables/CR_Graph.bin and b/resources/tables/CR_Graph.bin differ diff --git a/resources/tables/CR_Grapheme_Base.bin b/resources/tables/CR_Grapheme_Base.bin index a481b93..7322ed2 100644 Binary files a/resources/tables/CR_Grapheme_Base.bin and b/resources/tables/CR_Grapheme_Base.bin differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_Control.bin b/resources/tables/CR_Grapheme_Cluster_Break_Control.bin index 61e4e8b..9ea27ae 100644 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_Control.bin and b/resources/tables/CR_Grapheme_Cluster_Break_Control.bin differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_Extend.bin b/resources/tables/CR_Grapheme_Cluster_Break_Extend.bin index 25dc8aa..4448efa 100644 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_Extend.bin and b/resources/tables/CR_Grapheme_Cluster_Break_Extend.bin differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_Prepend.bin b/resources/tables/CR_Grapheme_Cluster_Break_Prepend.bin index 7f2f78a..eedc129 100644 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_Prepend.bin and b/resources/tables/CR_Grapheme_Cluster_Break_Prepend.bin differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_SpacingMark.bin b/resources/tables/CR_Grapheme_Cluster_Break_SpacingMark.bin index 3152589..9035da4 100644 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_SpacingMark.bin and b/resources/tables/CR_Grapheme_Cluster_Break_SpacingMark.bin differ diff --git a/resources/tables/CR_Grapheme_Extend.bin b/resources/tables/CR_Grapheme_Extend.bin index 5018ebe..a781c8f 100644 Binary files a/resources/tables/CR_Grapheme_Extend.bin and b/resources/tables/CR_Grapheme_Extend.bin differ diff --git a/resources/tables/CR_Grapheme_Link.bin b/resources/tables/CR_Grapheme_Link.bin index 21d1989..5147819 100644 Binary files a/resources/tables/CR_Grapheme_Link.bin and b/resources/tables/CR_Grapheme_Link.bin differ diff --git a/resources/tables/CR_Han.bin b/resources/tables/CR_Han.bin index fc1a6fa..862a01e 100644 Binary files a/resources/tables/CR_Han.bin and b/resources/tables/CR_Han.bin differ diff --git a/resources/tables/CR_Hiragana.bin b/resources/tables/CR_Hiragana.bin index 0284cf0..31e3262 100644 Binary files a/resources/tables/CR_Hiragana.bin and b/resources/tables/CR_Hiragana.bin differ diff --git a/resources/tables/CR_ID_Continue.bin b/resources/tables/CR_ID_Continue.bin index 9df7742..a83fbd3 100644 Binary files a/resources/tables/CR_ID_Continue.bin and b/resources/tables/CR_ID_Continue.bin differ diff --git a/resources/tables/CR_ID_Start.bin b/resources/tables/CR_ID_Start.bin index 84dec0c..d652c82 100644 Binary files a/resources/tables/CR_ID_Start.bin and b/resources/tables/CR_ID_Start.bin differ diff --git a/resources/tables/CR_Ideographic.bin b/resources/tables/CR_Ideographic.bin index 3b3d7d8..93db510 100644 Binary files a/resources/tables/CR_Ideographic.bin and b/resources/tables/CR_Ideographic.bin differ diff --git a/resources/tables/CR_In_Ahom.bin b/resources/tables/CR_In_Ahom.bin index 7cf8b2f..120d1e4 100644 Binary files a/resources/tables/CR_In_Ahom.bin and b/resources/tables/CR_In_Ahom.bin differ diff --git a/resources/tables/CR_In_Arabic_Extended_B.bin b/resources/tables/CR_In_Arabic_Extended_B.bin new file mode 100644 index 0000000..a059fac Binary files /dev/null and b/resources/tables/CR_In_Arabic_Extended_B.bin differ diff --git a/resources/tables/CR_In_Arabic_Extended_C.bin b/resources/tables/CR_In_Arabic_Extended_C.bin new file mode 100644 index 0000000..01aa369 Binary files /dev/null and b/resources/tables/CR_In_Arabic_Extended_C.bin differ diff --git a/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_H.bin b/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_H.bin new file mode 100644 index 0000000..6f55ca4 Binary files /dev/null and b/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_H.bin differ diff --git a/resources/tables/CR_In_Cypro_Minoan.bin b/resources/tables/CR_In_Cypro_Minoan.bin new file mode 100644 index 0000000..214552a Binary files /dev/null and b/resources/tables/CR_In_Cypro_Minoan.bin differ diff --git a/resources/tables/CR_In_Cyrillic_Extended_D.bin b/resources/tables/CR_In_Cyrillic_Extended_D.bin new file mode 100644 index 0000000..4df6670 Binary files /dev/null and b/resources/tables/CR_In_Cyrillic_Extended_D.bin differ diff --git a/resources/tables/CR_In_Devanagari_Extended_A.bin b/resources/tables/CR_In_Devanagari_Extended_A.bin new file mode 100644 index 0000000..80c9c3b Binary files /dev/null and b/resources/tables/CR_In_Devanagari_Extended_A.bin differ diff --git a/resources/tables/CR_In_Egyptian_Hieroglyph_Format_Controls.bin b/resources/tables/CR_In_Egyptian_Hieroglyph_Format_Controls.bin index 4e410f7..e066989 100644 Binary files a/resources/tables/CR_In_Egyptian_Hieroglyph_Format_Controls.bin and b/resources/tables/CR_In_Egyptian_Hieroglyph_Format_Controls.bin differ diff --git a/resources/tables/CR_In_Ethiopic_Extended_B.bin b/resources/tables/CR_In_Ethiopic_Extended_B.bin new file mode 100644 index 0000000..b6325ef Binary files /dev/null and b/resources/tables/CR_In_Ethiopic_Extended_B.bin differ diff --git a/resources/tables/CR_In_Kaktovik_Numerals.bin b/resources/tables/CR_In_Kaktovik_Numerals.bin new file mode 100644 index 0000000..9beb971 Binary files /dev/null and b/resources/tables/CR_In_Kaktovik_Numerals.bin differ diff --git a/resources/tables/CR_In_Kana_Extended_B.bin b/resources/tables/CR_In_Kana_Extended_B.bin new file mode 100644 index 0000000..2ea6aff Binary files /dev/null and b/resources/tables/CR_In_Kana_Extended_B.bin differ diff --git a/resources/tables/CR_In_Kawi.bin b/resources/tables/CR_In_Kawi.bin new file mode 100644 index 0000000..58df947 Binary files /dev/null and b/resources/tables/CR_In_Kawi.bin differ diff --git a/resources/tables/CR_In_Latin_Extended_F.bin b/resources/tables/CR_In_Latin_Extended_F.bin new file mode 100644 index 0000000..bf7ec02 Binary files /dev/null and b/resources/tables/CR_In_Latin_Extended_F.bin differ diff --git a/resources/tables/CR_In_Latin_Extended_G.bin b/resources/tables/CR_In_Latin_Extended_G.bin new file mode 100644 index 0000000..c949117 Binary files /dev/null and b/resources/tables/CR_In_Latin_Extended_G.bin differ diff --git a/resources/tables/CR_In_Nag_Mundari.bin b/resources/tables/CR_In_Nag_Mundari.bin new file mode 100644 index 0000000..52b0311 Binary files /dev/null and b/resources/tables/CR_In_Nag_Mundari.bin differ diff --git a/resources/tables/CR_In_No_Block.bin b/resources/tables/CR_In_No_Block.bin index 94a03a0..9382881 100644 Binary files a/resources/tables/CR_In_No_Block.bin and b/resources/tables/CR_In_No_Block.bin differ diff --git a/resources/tables/CR_In_Old_Uyghur.bin b/resources/tables/CR_In_Old_Uyghur.bin new file mode 100644 index 0000000..ec970fb Binary files /dev/null and b/resources/tables/CR_In_Old_Uyghur.bin differ diff --git a/resources/tables/CR_In_Tangsa.bin b/resources/tables/CR_In_Tangsa.bin new file mode 100644 index 0000000..aa5a085 Binary files /dev/null and b/resources/tables/CR_In_Tangsa.bin differ diff --git a/resources/tables/CR_In_Tangut_Supplement.bin b/resources/tables/CR_In_Tangut_Supplement.bin index a514d59..74e40bb 100644 Binary files a/resources/tables/CR_In_Tangut_Supplement.bin and b/resources/tables/CR_In_Tangut_Supplement.bin differ diff --git a/resources/tables/CR_In_Toto.bin b/resources/tables/CR_In_Toto.bin new file mode 100644 index 0000000..c81bbe7 Binary files /dev/null and b/resources/tables/CR_In_Toto.bin differ diff --git a/resources/tables/CR_In_Unified_Canadian_Aboriginal_Syllabics_Extended_A.bin b/resources/tables/CR_In_Unified_Canadian_Aboriginal_Syllabics_Extended_A.bin new file mode 100644 index 0000000..64428eb Binary files /dev/null and b/resources/tables/CR_In_Unified_Canadian_Aboriginal_Syllabics_Extended_A.bin differ diff --git a/resources/tables/CR_In_Vithkuqi.bin b/resources/tables/CR_In_Vithkuqi.bin new file mode 100644 index 0000000..d833c46 Binary files /dev/null and b/resources/tables/CR_In_Vithkuqi.bin differ diff --git a/resources/tables/CR_In_Znamenny_Musical_Notation.bin b/resources/tables/CR_In_Znamenny_Musical_Notation.bin new file mode 100644 index 0000000..079a0db Binary files /dev/null and b/resources/tables/CR_In_Znamenny_Musical_Notation.bin differ diff --git a/resources/tables/CR_Inherited.bin b/resources/tables/CR_Inherited.bin index 86bb22f..7dab441 100644 Binary files a/resources/tables/CR_Inherited.bin and b/resources/tables/CR_Inherited.bin differ diff --git a/resources/tables/CR_Kaithi.bin b/resources/tables/CR_Kaithi.bin index 21736cf..e10a54c 100644 Binary files a/resources/tables/CR_Kaithi.bin and b/resources/tables/CR_Kaithi.bin differ diff --git a/resources/tables/CR_Kannada.bin b/resources/tables/CR_Kannada.bin index ca836f7..ab48a13 100644 Binary files a/resources/tables/CR_Kannada.bin and b/resources/tables/CR_Kannada.bin differ diff --git a/resources/tables/CR_Katakana.bin b/resources/tables/CR_Katakana.bin index b8ffc19..858d3e0 100644 Binary files a/resources/tables/CR_Katakana.bin and b/resources/tables/CR_Katakana.bin differ diff --git a/resources/tables/CR_Kawi.bin b/resources/tables/CR_Kawi.bin new file mode 100644 index 0000000..fc7dfda Binary files /dev/null and b/resources/tables/CR_Kawi.bin differ diff --git a/resources/tables/CR_Khojki.bin b/resources/tables/CR_Khojki.bin index 2dd1190..880bcbe 100644 Binary files a/resources/tables/CR_Khojki.bin and b/resources/tables/CR_Khojki.bin differ diff --git a/resources/tables/CR_L.bin b/resources/tables/CR_L.bin index 4701c53..328f5f7 100644 Binary files a/resources/tables/CR_L.bin and b/resources/tables/CR_L.bin differ diff --git a/resources/tables/CR_LC.bin b/resources/tables/CR_LC.bin index 1a3e6a4..32f9d8e 100644 Binary files a/resources/tables/CR_LC.bin and b/resources/tables/CR_LC.bin differ diff --git a/resources/tables/CR_Lao.bin b/resources/tables/CR_Lao.bin index 5a66edd..a6d64e6 100644 Binary files a/resources/tables/CR_Lao.bin and b/resources/tables/CR_Lao.bin differ diff --git a/resources/tables/CR_Latin.bin b/resources/tables/CR_Latin.bin index d45ad2c..8549fcb 100644 Binary files a/resources/tables/CR_Latin.bin and b/resources/tables/CR_Latin.bin differ diff --git a/resources/tables/CR_Ll.bin b/resources/tables/CR_Ll.bin index 29a4077..dc516c0 100644 Binary files a/resources/tables/CR_Ll.bin and b/resources/tables/CR_Ll.bin differ diff --git a/resources/tables/CR_Lm.bin b/resources/tables/CR_Lm.bin index 4b2afa7..700200c 100644 Binary files a/resources/tables/CR_Lm.bin and b/resources/tables/CR_Lm.bin differ diff --git a/resources/tables/CR_Lo.bin b/resources/tables/CR_Lo.bin index 17b98d5..cddfc80 100644 Binary files a/resources/tables/CR_Lo.bin and b/resources/tables/CR_Lo.bin differ diff --git a/resources/tables/CR_Lower.bin b/resources/tables/CR_Lower.bin index 0edc321..e3552a7 100644 Binary files a/resources/tables/CR_Lower.bin and b/resources/tables/CR_Lower.bin differ diff --git a/resources/tables/CR_Lu.bin b/resources/tables/CR_Lu.bin index d006266..449856b 100644 Binary files a/resources/tables/CR_Lu.bin and b/resources/tables/CR_Lu.bin differ diff --git a/resources/tables/CR_M.bin b/resources/tables/CR_M.bin index 7f7b46c..f61ba79 100644 Binary files a/resources/tables/CR_M.bin and b/resources/tables/CR_M.bin differ diff --git a/resources/tables/CR_Mc.bin b/resources/tables/CR_Mc.bin index f7a640f..8c73149 100644 Binary files a/resources/tables/CR_Mc.bin and b/resources/tables/CR_Mc.bin differ diff --git a/resources/tables/CR_Mn.bin b/resources/tables/CR_Mn.bin index e7521a8..ef3e771 100644 Binary files a/resources/tables/CR_Mn.bin and b/resources/tables/CR_Mn.bin differ diff --git a/resources/tables/CR_Mongolian.bin b/resources/tables/CR_Mongolian.bin index 8fdb01e..6d8886c 100644 Binary files a/resources/tables/CR_Mongolian.bin and b/resources/tables/CR_Mongolian.bin differ diff --git a/resources/tables/CR_N.bin b/resources/tables/CR_N.bin index 2262008..9bc3f81 100644 Binary files a/resources/tables/CR_N.bin and b/resources/tables/CR_N.bin differ diff --git a/resources/tables/CR_Nag_Mundari.bin b/resources/tables/CR_Nag_Mundari.bin new file mode 100644 index 0000000..ab46804 Binary files /dev/null and b/resources/tables/CR_Nag_Mundari.bin differ diff --git a/resources/tables/CR_No.bin b/resources/tables/CR_No.bin index 3a8516e..426e624 100644 Binary files a/resources/tables/CR_No.bin and b/resources/tables/CR_No.bin differ diff --git a/resources/tables/CR_Old_Uyghur.bin b/resources/tables/CR_Old_Uyghur.bin new file mode 100644 index 0000000..5856296 Binary files /dev/null and b/resources/tables/CR_Old_Uyghur.bin differ diff --git a/resources/tables/CR_Other_Alphabetic.bin b/resources/tables/CR_Other_Alphabetic.bin index ea2cfee..639aa37 100644 Binary files a/resources/tables/CR_Other_Alphabetic.bin and b/resources/tables/CR_Other_Alphabetic.bin differ diff --git a/resources/tables/CR_Other_Lowercase.bin b/resources/tables/CR_Other_Lowercase.bin index be9ff4a..37409e7 100644 Binary files a/resources/tables/CR_Other_Lowercase.bin and b/resources/tables/CR_Other_Lowercase.bin differ diff --git a/resources/tables/CR_Pd.bin b/resources/tables/CR_Pd.bin index a7bcf21..0ae1042 100644 Binary files a/resources/tables/CR_Pd.bin and b/resources/tables/CR_Pd.bin differ diff --git a/resources/tables/CR_Pe.bin b/resources/tables/CR_Pe.bin index 42920a4..1311265 100644 Binary files a/resources/tables/CR_Pe.bin and b/resources/tables/CR_Pe.bin differ diff --git a/resources/tables/CR_Po.bin b/resources/tables/CR_Po.bin index 9d07d71..71301e3 100644 Binary files a/resources/tables/CR_Po.bin and b/resources/tables/CR_Po.bin differ diff --git a/resources/tables/CR_Prepended_Concatenation_Mark.bin b/resources/tables/CR_Prepended_Concatenation_Mark.bin index b66b647..a90ccf1 100644 Binary files a/resources/tables/CR_Prepended_Concatenation_Mark.bin and b/resources/tables/CR_Prepended_Concatenation_Mark.bin differ diff --git a/resources/tables/CR_Print.bin b/resources/tables/CR_Print.bin index 99bbb9f..868a98a 100644 Binary files a/resources/tables/CR_Print.bin and b/resources/tables/CR_Print.bin differ diff --git a/resources/tables/CR_Ps.bin b/resources/tables/CR_Ps.bin index 9f51448..7998abc 100644 Binary files a/resources/tables/CR_Ps.bin and b/resources/tables/CR_Ps.bin differ diff --git a/resources/tables/CR_Punct.bin b/resources/tables/CR_Punct.bin index f270209..4f8fde2 100644 Binary files a/resources/tables/CR_Punct.bin and b/resources/tables/CR_Punct.bin differ diff --git a/resources/tables/CR_S.bin b/resources/tables/CR_S.bin index dde16c7..c14f8eb 100644 Binary files a/resources/tables/CR_S.bin and b/resources/tables/CR_S.bin differ diff --git a/resources/tables/CR_Sc.bin b/resources/tables/CR_Sc.bin index b3b6ff5..63875fb 100644 Binary files a/resources/tables/CR_Sc.bin and b/resources/tables/CR_Sc.bin differ diff --git a/resources/tables/CR_Sentence_Terminal.bin b/resources/tables/CR_Sentence_Terminal.bin index b9e4ad9..a539433 100644 Binary files a/resources/tables/CR_Sentence_Terminal.bin and b/resources/tables/CR_Sentence_Terminal.bin differ diff --git a/resources/tables/CR_Sk.bin b/resources/tables/CR_Sk.bin index 3339a45..a596356 100644 Binary files a/resources/tables/CR_Sk.bin and b/resources/tables/CR_Sk.bin differ diff --git a/resources/tables/CR_So.bin b/resources/tables/CR_So.bin index 081f079..9011f82 100644 Binary files a/resources/tables/CR_So.bin and b/resources/tables/CR_So.bin differ diff --git a/resources/tables/CR_Soft_Dotted.bin b/resources/tables/CR_Soft_Dotted.bin index 703d83a..a4308cb 100644 Binary files a/resources/tables/CR_Soft_Dotted.bin and b/resources/tables/CR_Soft_Dotted.bin differ diff --git a/resources/tables/CR_Tagalog.bin b/resources/tables/CR_Tagalog.bin index f2b056d..b0036ed 100644 Binary files a/resources/tables/CR_Tagalog.bin and b/resources/tables/CR_Tagalog.bin differ diff --git a/resources/tables/CR_Takri.bin b/resources/tables/CR_Takri.bin index e1cd469..24f19ad 100644 Binary files a/resources/tables/CR_Takri.bin and b/resources/tables/CR_Takri.bin differ diff --git a/resources/tables/CR_Tangsa.bin b/resources/tables/CR_Tangsa.bin new file mode 100644 index 0000000..4e6fea3 Binary files /dev/null and b/resources/tables/CR_Tangsa.bin differ diff --git a/resources/tables/CR_Telugu.bin b/resources/tables/CR_Telugu.bin index 2dfa002..03e903d 100644 Binary files a/resources/tables/CR_Telugu.bin and b/resources/tables/CR_Telugu.bin differ diff --git a/resources/tables/CR_Terminal_Punctuation.bin b/resources/tables/CR_Terminal_Punctuation.bin index 8d556d1..554c2ca 100644 Binary files a/resources/tables/CR_Terminal_Punctuation.bin and b/resources/tables/CR_Terminal_Punctuation.bin differ diff --git a/resources/tables/CR_Toto.bin b/resources/tables/CR_Toto.bin new file mode 100644 index 0000000..835bffc Binary files /dev/null and b/resources/tables/CR_Toto.bin differ diff --git a/resources/tables/CR_Unified_Ideograph.bin b/resources/tables/CR_Unified_Ideograph.bin index 388e0d8..f02be7d 100644 Binary files a/resources/tables/CR_Unified_Ideograph.bin and b/resources/tables/CR_Unified_Ideograph.bin differ diff --git a/resources/tables/CR_Unknown.bin b/resources/tables/CR_Unknown.bin index 83deb7c..314cfce 100644 Binary files a/resources/tables/CR_Unknown.bin and b/resources/tables/CR_Unknown.bin differ diff --git a/resources/tables/CR_Upper.bin b/resources/tables/CR_Upper.bin index d87ae01..21717eb 100644 Binary files a/resources/tables/CR_Upper.bin and b/resources/tables/CR_Upper.bin differ diff --git a/resources/tables/CR_Variation_Selector.bin b/resources/tables/CR_Variation_Selector.bin index d7ffc57..2425c77 100644 Binary files a/resources/tables/CR_Variation_Selector.bin and b/resources/tables/CR_Variation_Selector.bin differ diff --git a/resources/tables/CR_Vithkuqi.bin b/resources/tables/CR_Vithkuqi.bin new file mode 100644 index 0000000..1fb26db Binary files /dev/null and b/resources/tables/CR_Vithkuqi.bin differ diff --git a/resources/tables/CR_Word.bin b/resources/tables/CR_Word.bin index a9e1637..3684663 100644 Binary files a/resources/tables/CR_Word.bin and b/resources/tables/CR_Word.bin differ diff --git a/resources/tables/CR_XID_Continue.bin b/resources/tables/CR_XID_Continue.bin index 17ae376..584b40a 100644 Binary files a/resources/tables/CR_XID_Continue.bin and b/resources/tables/CR_XID_Continue.bin differ diff --git a/resources/tables/CR_XID_Start.bin b/resources/tables/CR_XID_Start.bin index 6dfc537..52e49f8 100644 Binary files a/resources/tables/CR_XID_Start.bin and b/resources/tables/CR_XID_Start.bin differ diff --git a/resources/tables/CR_XPosixPunct.bin b/resources/tables/CR_XPosixPunct.bin index a29a57c..b553250 100644 Binary files a/resources/tables/CR_XPosixPunct.bin and b/resources/tables/CR_XPosixPunct.bin differ diff --git a/resources/tables/CaseFold.bin b/resources/tables/CaseFold.bin index ab9dee6..347026a 100644 Binary files a/resources/tables/CaseFold.bin and b/resources/tables/CaseFold.bin differ diff --git a/resources/tables/CaseUnfold_11.bin b/resources/tables/CaseUnfold_11.bin index db9b182..6724aeb 100644 Binary files a/resources/tables/CaseUnfold_11.bin and b/resources/tables/CaseUnfold_11.bin differ diff --git a/resources/tables/Transcoder_SingleByte_ByteArray.bin b/resources/tables/Transcoder_SingleByte_ByteArray.bin index a6846f6..284b334 100644 Binary files a/resources/tables/Transcoder_SingleByte_ByteArray.bin and b/resources/tables/Transcoder_SingleByte_ByteArray.bin differ diff --git a/resources/tables/Transcoder_SingleByte_WordArray.bin b/resources/tables/Transcoder_SingleByte_WordArray.bin index 61364d9..9ccb607 100644 Binary files a/resources/tables/Transcoder_SingleByte_WordArray.bin and b/resources/tables/Transcoder_SingleByte_WordArray.bin differ diff --git a/scripts/generate.rb b/scripts/generate.rb index ca45ea0..230b71d 100644 --- a/scripts/generate.rb +++ b/scripts/generate.rb @@ -8,7 +8,7 @@ else ;["rdata", ""] end -UNICODE_VERSION = "13.0.0" +UNICODE_VERSION = "15.0.0" SRC_DIR = "../src/org/jcodings" DST_BIN_DIR = "../resources/tables" INDENT = " " * 4 @@ -107,6 +107,7 @@ def generate_transcoder_list src, dst, tree_start, table_info, iul, max_in, max_out, conv, state_size, state_init, state_fini, *funcs = body tree_start = trans_src[/#define\s+#{tree_start}\s+WORDINDEX2INFO\((\d+)\)/, 1].to_i << 2 state_size = "24" if state_size == "sizeof(struct from_utf8_mac_status)" + t_name = "universal_newline" if t_name == "Lf_newline" generic = funcs.all?{|f|f == "NULL" || f == "0"} if generic diff --git a/src/org/jcodings/Config.java b/src/org/jcodings/Config.java index 09bf0c7..28dcd8f 100644 --- a/src/org/jcodings/Config.java +++ b/src/org/jcodings/Config.java @@ -62,12 +62,12 @@ public interface Config { final int ENC_CASE_FOLD_MIN = INTERNAL_ENC_CASE_FOLD_MULTI_CHAR; final int ENC_CASE_FOLD_DEFAULT = ENC_CASE_FOLD_MIN; - final String UNICODE_VERSION_STRING = "13.0.0"; - final int UNICODE_VERSION_MAJOR = 13; + final String UNICODE_VERSION_STRING = "15.0.0"; + final int UNICODE_VERSION_MAJOR = 15; final int UNICODE_VERSION_MINOR = 0; final int UNICODE_VERSION_TEENY = 0; - final String UNICODE_EMOJI_VERSION_STRING = "13.1"; - final int UNICODE_EMOJI_VERSION_MAJOR = 13; - final int UNICODE_EMOJI_VERSION_MINOR = 1; + final String UNICODE_EMOJI_VERSION_STRING = "15.0"; + final int UNICODE_EMOJI_VERSION_MAJOR = 15; + final int UNICODE_EMOJI_VERSION_MINOR = 0; } diff --git a/src/org/jcodings/transcode/EConv.java b/src/org/jcodings/transcode/EConv.java index 1197127..269b4da 100644 --- a/src/org/jcodings/transcode/EConv.java +++ b/src/org/jcodings/transcode/EConv.java @@ -732,6 +732,11 @@ public void binmode() { if (entry.getTranscoder() != null) transcoders[n++] = entry.getTranscoder(); } + if ((flags & LF_NEWLINE_DECORATOR) != 0) { + TranscoderDB.Entry entry = TranscoderDB.getEntry(NULL_STRING, "lf_newline".getBytes()); + if (entry.getTranscoder() != null) transcoders[n++] = entry.getTranscoder(); + } + int nTrans = numTranscoders; int j = 0; for (int i = 0; i < nTrans; i++) { diff --git a/src/org/jcodings/transcode/EConvFlags.java b/src/org/jcodings/transcode/EConvFlags.java index bbd6f7a..51743e2 100644 --- a/src/org/jcodings/transcode/EConvFlags.java +++ b/src/org/jcodings/transcode/EConvFlags.java @@ -29,23 +29,24 @@ public interface EConvFlags { final int UNDEF_REPLACE = 0x00000020; final int UNDEF_HEX_CHARREF = 0x00000030; - final int DECORATOR_MASK = 0x0000ff00; - final int NEWLINE_DECORATOR_MASK = 0x00003f00; + final int DECORATOR_MASK = 0x0001ff00; + final int NEWLINE_DECORATOR_MASK = 0x00007f00; final int NEWLINE_DECORATOR_READ_MASK = 0x00000f00; - final int NEWLINE_DECORATOR_WRITE_MASK = 0x00003000; + final int NEWLINE_DECORATOR_WRITE_MASK = 0x00007000; final int UNIVERSAL_NEWLINE_DECORATOR = 0x00000100; final int CRLF_NEWLINE_DECORATOR = 0x00001000; final int CR_NEWLINE_DECORATOR = 0x00002000; - final int XML_TEXT_DECORATOR = 0x00004000; - final int XML_ATTR_CONTENT_DECORATOR = 0x00008000; + final int LF_NEWLINE_DECORATOR = 0x00004000; + final int XML_TEXT_DECORATOR = 0x00008000; + final int XML_ATTR_CONTENT_DECORATOR = 0x00010000; final int STATEFUL_DECORATOR_MASK = 0x00f00000; final int XML_ATTR_QUOTE_DECORATOR = 0x00100000; - final int PARTIAL_INPUT = 0x00010000; - final int AFTER_OUTPUT = 0x00020000; + final int PARTIAL_INPUT = 0x00020000; + final int AFTER_OUTPUT = 0x00040000; final int MAX_ECFLAGS_DECORATORS = 32; } \ No newline at end of file diff --git a/src/org/jcodings/transcode/TranscoderDB.java b/src/org/jcodings/transcode/TranscoderDB.java index e944b69..e9c72d5 100644 --- a/src/org/jcodings/transcode/TranscoderDB.java +++ b/src/org/jcodings/transcode/TranscoderDB.java @@ -247,6 +247,7 @@ public static int decoratorNames(int ecflags, byte[][] decorators) { case UNIVERSAL_NEWLINE_DECORATOR: case CRLF_NEWLINE_DECORATOR: case CR_NEWLINE_DECORATOR: + case LF_NEWLINE_DECORATOR: case 0: break; default: @@ -263,6 +264,7 @@ public static int decoratorNames(int ecflags, byte[][] decorators) { if ((ecflags & CRLF_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "crlf_newline".getBytes(); if ((ecflags & CR_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "cr_newline".getBytes(); + if ((ecflags & LF_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "lf_newline".getBytes(); if ((ecflags & UNIVERSAL_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "universal_newline".getBytes(); return numDecorators; diff --git a/src/org/jcodings/transcode/TranscoderList.java b/src/org/jcodings/transcode/TranscoderList.java index 9b6f2f7..8d0ed16 100644 --- a/src/org/jcodings/transcode/TranscoderList.java +++ b/src/org/jcodings/transcode/TranscoderList.java @@ -101,6 +101,7 @@ static void load() { TranscoderDB.declare("", "universal_newline", "Universal_newline"); TranscoderDB.declare("", "crlf_newline", null /*Crlf_newline*/); TranscoderDB.declare("", "cr_newline", null /*Cr_newline*/); + TranscoderDB.declare("", "lf_newline", "Universal_newline"); TranscoderDB.declare("US-ASCII", "UTF-8", null /*From_US_ASCII*/); TranscoderDB.declare("UTF-8", "US-ASCII", null /*To_US_ASCII*/); TranscoderDB.declare("ASCII-8BIT", "UTF-8", null /*From_ASCII_8BIT*/); @@ -175,6 +176,8 @@ static void load() { TranscoderDB.declare("UTF-8", "IBM862", null /*To_IBM862*/); TranscoderDB.declare("IBM863", "UTF-8", null /*From_IBM863*/); TranscoderDB.declare("UTF-8", "IBM863", null /*To_IBM863*/); + TranscoderDB.declare("IBM864", "UTF-8", null /*From_IBM864*/); + TranscoderDB.declare("UTF-8", "IBM864", null /*To_IBM864*/); TranscoderDB.declare("IBM865", "UTF-8", null /*From_IBM865*/); TranscoderDB.declare("UTF-8", "IBM865", null /*To_IBM865*/); TranscoderDB.declare("IBM866", "UTF-8", null /*From_IBM866*/); @@ -223,7 +226,7 @@ static void load() { TranscoderDB.declare("UTF-32", "UTF-8", "From_UTF_32"); TranscoderDB.declare("UTF-8", "UTF-16", "To_UTF_16"); TranscoderDB.declare("UTF-8", "UTF-32", "To_UTF_32"); - } + }; static final GenericTranscoderEntry[] GENERIC_LIST = new GenericTranscoderEntry[] { new GenericTranscoderEntry("Big5", "UTF-8", 57100, "Big5", 1, 2, 1, AsciiCompatibility.CONVERTER, 0), @@ -359,36 +362,38 @@ static void load() { new GenericTranscoderEntry("UTF-8", "IBM862", 35228, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("IBM863", "UTF-8", 35752, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("UTF-8", "IBM863", 36132, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM865", "UTF-8", 36656, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM865", 36968, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM866", "UTF-8", 37492, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM866", 37948, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM869", "UTF-8", 38440, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM869", 38956, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACCROATIAN", "UTF-8", 39480, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACCROATIAN", 40312, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACCYRILLIC", "UTF-8", 40836, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACCYRILLIC", 41540, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACGREEK", "UTF-8", 42064, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACGREEK", 42764, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACICELAND", "UTF-8", 43288, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACICELAND", 44004, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACROMAN", "UTF-8", 44528, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACROMAN", 45012, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACROMANIA", "UTF-8", 45536, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACROMANIA", 45912, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACTURKISH", "UTF-8", 46432, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACTURKISH", 46788, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACUKRAINE", "UTF-8", 47312, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACUKRAINE", 47536, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("KOI8-U", "UTF-8", 48060, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "KOI8-U", 48844, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("KOI8-R", "UTF-8", 49368, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "KOI8-R", 49900, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("TIS-620", "UTF-8", 50264, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "TIS-620", 50308, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("CP850", "UTF-8", 50832, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "CP850", 51380, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM864", "UTF-8", 36644, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM864", 37504, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM865", "UTF-8", 38028, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM865", 38340, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM866", "UTF-8", 38864, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM866", 39320, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM869", "UTF-8", 39812, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM869", 40328, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACCROATIAN", "UTF-8", 40852, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACCROATIAN", 41684, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACCYRILLIC", "UTF-8", 42208, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACCYRILLIC", 42912, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACGREEK", "UTF-8", 43436, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACGREEK", 44136, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACICELAND", "UTF-8", 44660, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACICELAND", 45376, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACROMAN", "UTF-8", 45900, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACROMAN", 46384, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACROMANIA", "UTF-8", 46908, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACROMANIA", 47284, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACTURKISH", "UTF-8", 47804, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACTURKISH", 48160, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACUKRAINE", "UTF-8", 48684, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACUKRAINE", 48908, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("KOI8-U", "UTF-8", 49432, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "KOI8-U", 50216, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("KOI8-R", "UTF-8", 50740, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "KOI8-R", 51272, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("TIS-620", "UTF-8", 51636, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "TIS-620", 51680, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("CP850", "UTF-8", 52204, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "CP850", 52752, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("CP852", "UTF-8", 29992, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("UTF-8", "CP852", 30608, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("CP855", "UTF-8", 31132, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), @@ -416,6 +421,7 @@ public static Transcoder getInstance(String name) { case "Eucjp2sjis": return Eucjp2sjis_Transcoder.INSTANCE; case "Sjis2eucjp": return Sjis2eucjp_Transcoder.INSTANCE; case "Universal_newline": return Universal_newline_Transcoder.INSTANCE; + case "Lf_newline": return Universal_newline_Transcoder.INSTANCE; case "From_UTF8_MAC": return From_UTF8_MAC_Transcoder.INSTANCE; case "From_UTF_16BE": return From_UTF_16BE_Transcoder.INSTANCE; case "To_UTF_16BE": return To_UTF_16BE_Transcoder.INSTANCE; diff --git a/src/org/jcodings/unicode/UnicodeCodeRange.java b/src/org/jcodings/unicode/UnicodeCodeRange.java index 6277ffa..8ff37fa 100644 --- a/src/org/jcodings/unicode/UnicodeCodeRange.java +++ b/src/org/jcodings/unicode/UnicodeCodeRange.java @@ -450,6 +450,18 @@ public enum UnicodeCodeRange { KITS("kits", "CR_Khitan_Small_Script"), YEZIDI("yezidi", "CR_Yezidi"), YEZI("yezi", "CR_Yezidi"), + CYPROMINOAN("cyprominoan", "CR_Cypro_Minoan"), + CPMN("cpmn", "CR_Cypro_Minoan"), + OLDUYGHUR("olduyghur", "CR_Old_Uyghur"), + OUGR("ougr", "CR_Old_Uyghur"), + TANGSA("tangsa", "CR_Tangsa"), + TNSA("tnsa", "CR_Tangsa"), + TOTO("toto", "CR_Toto"), + VITHKUQI("vithkuqi", "CR_Vithkuqi"), + VITH("vith", "CR_Vithkuqi"), + KAWI("kawi", "CR_Kawi"), + NAGMUNDARI("nagmundari", "CR_Nag_Mundari"), + NAGM("nagm", "CR_Nag_Mundari"), WHITESPACE("whitespace", "CR_Space"), WSPACE("wspace", "CR_Space"), BIDICONTROL("bidicontrol", "CR_Bidi_Control"), @@ -550,6 +562,8 @@ public enum UnicodeCodeRange { AGE_12_0("age=12.0", "CR_Age_12_0"), AGE_12_1("age=12.1", "CR_Age_12_1"), AGE_13_0("age=13.0", "CR_Age_13_0"), + AGE_14_0("age=14.0", "CR_Age_14_0"), + AGE_15_0("age=15.0", "CR_Age_15_0"), GRAPHEMECLUSTERBREAK_PREPEND("graphemeclusterbreak=prepend", "CR_Grapheme_Cluster_Break_Prepend"), GRAPHEMECLUSTERBREAK_CR("graphemeclusterbreak=cr", "CR_Grapheme_Cluster_Break_CR"), GRAPHEMECLUSTERBREAK_LF("graphemeclusterbreak=lf", "CR_NEWLINE"), @@ -584,6 +598,7 @@ public enum UnicodeCodeRange { INSAMARITAN("insamaritan", "CR_In_Samaritan"), INMANDAIC("inmandaic", "CR_In_Mandaic"), INSYRIACSUPPLEMENT("insyriacsupplement", "CR_In_Syriac_Supplement"), + INARABICEXTENDEDB("inarabicextendedb", "CR_In_Arabic_Extended_B"), INARABICEXTENDEDA("inarabicextendeda", "CR_In_Arabic_Extended_A"), INDEVANAGARI("indevanagari", "CR_In_Devanagari"), INBENGALI("inbengali", "CR_In_Bengali"), @@ -751,7 +766,9 @@ public enum UnicodeCodeRange { INOSAGE("inosage", "CR_In_Osage"), INELBASAN("inelbasan", "CR_In_Elbasan"), INCAUCASIANALBANIAN("incaucasianalbanian", "CR_In_Caucasian_Albanian"), + INVITHKUQI("invithkuqi", "CR_In_Vithkuqi"), INLINEARA("inlineara", "CR_In_Linear_A"), + INLATINEXTENDEDF("inlatinextendedf", "CR_In_Latin_Extended_F"), INCYPRIOTSYLLABARY("incypriotsyllabary", "CR_In_Cypriot_Syllabary"), INIMPERIALARAMAIC("inimperialaramaic", "CR_In_Imperial_Aramaic"), PALMYRENE("palmyrene", "CR_Palmyrene"), @@ -778,8 +795,10 @@ public enum UnicodeCodeRange { INHANIFIROHINGYA("inhanifirohingya", "CR_In_Hanifi_Rohingya"), INRUMINUMERALSYMBOLS("inruminumeralsymbols", "CR_In_Rumi_Numeral_Symbols"), INYEZIDI("inyezidi", "CR_In_Yezidi"), + INARABICEXTENDEDC("inarabicextendedc", "CR_In_Arabic_Extended_C"), INOLDSOGDIAN("inoldsogdian", "CR_In_Old_Sogdian"), INSOGDIAN("insogdian", "CR_In_Sogdian"), + INOLDUYGHUR("inolduyghur", "CR_In_Old_Uyghur"), INCHORASMIAN("inchorasmian", "CR_In_Chorasmian"), INELYMAIC("inelymaic", "CR_In_Elymaic"), INBRAHMI("inbrahmi", "CR_In_Brahmi"), @@ -807,22 +826,27 @@ public enum UnicodeCodeRange { INNANDINAGARI("innandinagari", "CR_In_Nandinagari"), INZANABAZARSQUARE("inzanabazarsquare", "CR_In_Zanabazar_Square"), INSOYOMBO("insoyombo", "CR_In_Soyombo"), + INUNIFIEDCANADIANABORIGINALSYLLABICSEXTENDEDA("inunifiedcanadianaboriginalsyllabicsextendeda", "CR_In_Unified_Canadian_Aboriginal_Syllabics_Extended_A"), INPAUCINHAU("inpaucinhau", "CR_In_Pau_Cin_Hau"), + INDEVANAGARIEXTENDEDA("indevanagariextendeda", "CR_In_Devanagari_Extended_A"), INBHAIKSUKI("inbhaiksuki", "CR_In_Bhaiksuki"), INMARCHEN("inmarchen", "CR_In_Marchen"), INMASARAMGONDI("inmasaramgondi", "CR_In_Masaram_Gondi"), INGUNJALAGONDI("ingunjalagondi", "CR_In_Gunjala_Gondi"), INMAKASAR("inmakasar", "CR_In_Makasar"), + INKAWI("inkawi", "CR_In_Kawi"), INLISUSUPPLEMENT("inlisusupplement", "CR_In_Lisu_Supplement"), INTAMILSUPPLEMENT("intamilsupplement", "CR_In_Tamil_Supplement"), INCUNEIFORM("incuneiform", "CR_In_Cuneiform"), INCUNEIFORMNUMBERSANDPUNCTUATION("incuneiformnumbersandpunctuation", "CR_In_Cuneiform_Numbers_and_Punctuation"), INEARLYDYNASTICCUNEIFORM("inearlydynasticcuneiform", "CR_In_Early_Dynastic_Cuneiform"), + INCYPROMINOAN("incyprominoan", "CR_In_Cypro_Minoan"), INEGYPTIANHIEROGLYPHS("inegyptianhieroglyphs", "CR_In_Egyptian_Hieroglyphs"), INEGYPTIANHIEROGLYPHFORMATCONTROLS("inegyptianhieroglyphformatcontrols", "CR_In_Egyptian_Hieroglyph_Format_Controls"), INANATOLIANHIEROGLYPHS("inanatolianhieroglyphs", "CR_In_Anatolian_Hieroglyphs"), INBAMUMSUPPLEMENT("inbamumsupplement", "CR_In_Bamum_Supplement"), INMRO("inmro", "CR_In_Mro"), + INTANGSA("intangsa", "CR_In_Tangsa"), INBASSAVAH("inbassavah", "CR_In_Bassa_Vah"), INPAHAWHHMONG("inpahawhhmong", "CR_In_Pahawh_Hmong"), INMEDEFAIDRIN("inmedefaidrin", "CR_In_Medefaidrin"), @@ -832,23 +856,31 @@ public enum UnicodeCodeRange { INTANGUTCOMPONENTS("intangutcomponents", "CR_In_Tangut_Components"), INKHITANSMALLSCRIPT("inkhitansmallscript", "CR_In_Khitan_Small_Script"), INTANGUTSUPPLEMENT("intangutsupplement", "CR_In_Tangut_Supplement"), + INKANAEXTENDEDB("inkanaextendedb", "CR_In_Kana_Extended_B"), INKANASUPPLEMENT("inkanasupplement", "CR_In_Kana_Supplement"), INKANAEXTENDEDA("inkanaextendeda", "CR_In_Kana_Extended_A"), INSMALLKANAEXTENSION("insmallkanaextension", "CR_In_Small_Kana_Extension"), INNUSHU("innushu", "CR_In_Nushu"), INDUPLOYAN("induployan", "CR_In_Duployan"), INSHORTHANDFORMATCONTROLS("inshorthandformatcontrols", "CR_In_Shorthand_Format_Controls"), + INZNAMENNYMUSICALNOTATION("inznamennymusicalnotation", "CR_In_Znamenny_Musical_Notation"), INBYZANTINEMUSICALSYMBOLS("inbyzantinemusicalsymbols", "CR_In_Byzantine_Musical_Symbols"), INMUSICALSYMBOLS("inmusicalsymbols", "CR_In_Musical_Symbols"), INANCIENTGREEKMUSICALNOTATION("inancientgreekmusicalnotation", "CR_In_Ancient_Greek_Musical_Notation"), + INKAKTOVIKNUMERALS("inkaktoviknumerals", "CR_In_Kaktovik_Numerals"), INMAYANNUMERALS("inmayannumerals", "CR_In_Mayan_Numerals"), INTAIXUANJINGSYMBOLS("intaixuanjingsymbols", "CR_In_Tai_Xuan_Jing_Symbols"), INCOUNTINGRODNUMERALS("incountingrodnumerals", "CR_In_Counting_Rod_Numerals"), INMATHEMATICALALPHANUMERICSYMBOLS("inmathematicalalphanumericsymbols", "CR_In_Mathematical_Alphanumeric_Symbols"), INSUTTONSIGNWRITING("insuttonsignwriting", "CR_In_Sutton_SignWriting"), + INLATINEXTENDEDG("inlatinextendedg", "CR_In_Latin_Extended_G"), INGLAGOLITICSUPPLEMENT("inglagoliticsupplement", "CR_In_Glagolitic_Supplement"), + INCYRILLICEXTENDEDD("incyrillicextendedd", "CR_In_Cyrillic_Extended_D"), INNYIAKENGPUACHUEHMONG("innyiakengpuachuehmong", "CR_In_Nyiakeng_Puachue_Hmong"), + INTOTO("intoto", "CR_In_Toto"), INWANCHO("inwancho", "CR_In_Wancho"), + INNAGMUNDARI("innagmundari", "CR_In_Nag_Mundari"), + INETHIOPICEXTENDEDB("inethiopicextendedb", "CR_In_Ethiopic_Extended_B"), INMENDEKIKAKUI("inmendekikakui", "CR_In_Mende_Kikakui"), INADLAM("inadlam", "CR_In_Adlam"), ININDICSIYAQNUMBERS("inindicsiyaqnumbers", "CR_In_Indic_Siyaq_Numbers"), @@ -877,6 +909,7 @@ public enum UnicodeCodeRange { INCJKUNIFIEDIDEOGRAPHSEXTENSIONF("incjkunifiedideographsextensionf", "CR_In_CJK_Unified_Ideographs_Extension_F"), INCJKCOMPATIBILITYIDEOGRAPHSSUPPLEMENT("incjkcompatibilityideographssupplement", "CR_In_CJK_Compatibility_Ideographs_Supplement"), INCJKUNIFIEDIDEOGRAPHSEXTENSIONG("incjkunifiedideographsextensiong", "CR_In_CJK_Unified_Ideographs_Extension_G"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIONH("incjkunifiedideographsextensionh", "CR_In_CJK_Unified_Ideographs_Extension_H"), INTAGS("intags", "CR_In_Tags"), INVARIATIONSELECTORSSUPPLEMENT("invariationselectorssupplement", "CR_In_Variation_Selectors_Supplement"), INSUPPLEMENTARYPRIVATEUSEAREAA("insupplementaryprivateuseareaa", "CR_In_Supplementary_Private_Use_Area_A"), @@ -906,5 +939,5 @@ public int getCType() { } static final UnicodeCodeRange[]CodeRangeTable = UnicodeCodeRange.values(); - static final int MAX_WORD_LENGTH = 44; + static final int MAX_WORD_LENGTH = 45; } diff --git a/test/org/jcodings/specific/TestEConv.java b/test/org/jcodings/specific/TestEConv.java index 326c73f..69f3bfe 100644 --- a/test/org/jcodings/specific/TestEConv.java +++ b/test/org/jcodings/specific/TestEConv.java @@ -63,6 +63,18 @@ public void testCrNewline() throws Exception { Assert.assertArrayEquals("foo\rbar".getBytes(), dest); } + @Test + public void testLfNewline() throws Exception { + EConv econv = TranscoderDB.open("".getBytes(), "".getBytes(), EConvFlags.LF_NEWLINE_DECORATOR); + + byte[] src = "foo\nbar\r\nbaz".getBytes(); + byte[] dest = new byte[11]; + + econv.convert(src, new Ptr(0), 12, dest, new Ptr(0), dest.length, 0); + + Assert.assertArrayEquals("foo\nbar\nbaz".getBytes(), dest); + } + @Test public void testXMLWithCharref() throws Exception { EConv econv = TranscoderDB.open("utf-8".getBytes(), "euc-jp".getBytes(), EConvFlags.XML_ATTR_CONTENT_DECORATOR | EConvFlags.XML_ATTR_QUOTE_DECORATOR | EConvFlags.UNDEF_HEX_CHARREF);