update unicode to 15.0

scripts/generate.rb

@@ -8,7 +8,7 @@
 else ;["rdata", ""]
 end
 
-UNICODE_VERSION = "13.0.0"
+UNICODE_VERSION = "15.0.0"
 SRC_DIR = "../src/org/jcodings"
 DST_BIN_DIR =  "../resources/tables"
 INDENT = " " * 4
@@ -107,6 +107,7 @@ def generate_transcoder_list
             src, dst, tree_start, table_info, iul, max_in, max_out, conv, state_size, state_init, state_fini, *funcs = body
             tree_start = trans_src[/#define\s+#{tree_start}\s+WORDINDEX2INFO\((\d+)\)/, 1].to_i << 2
             state_size = "24" if state_size == "sizeof(struct from_utf8_mac_status)"
+            t_name = "universal_newline" if t_name == "Lf_newline"
             generic = funcs.all?{|f|f == "NULL" || f == "0"}
 
             if generic

src/org/jcodings/Config.java

@@ -62,12 +62,12 @@ public interface Config {
     final int ENC_CASE_FOLD_MIN = INTERNAL_ENC_CASE_FOLD_MULTI_CHAR;
     final int ENC_CASE_FOLD_DEFAULT = ENC_CASE_FOLD_MIN;
 
-    final String UNICODE_VERSION_STRING = "13.0.0";
-    final int UNICODE_VERSION_MAJOR = 13;
+    final String UNICODE_VERSION_STRING = "15.0.0";
+    final int UNICODE_VERSION_MAJOR = 15;
     final int UNICODE_VERSION_MINOR = 0;
     final int UNICODE_VERSION_TEENY = 0;
-    final String UNICODE_EMOJI_VERSION_STRING = "13.1";
-    final int UNICODE_EMOJI_VERSION_MAJOR = 13;
-    final int UNICODE_EMOJI_VERSION_MINOR = 1;
+    final String UNICODE_EMOJI_VERSION_STRING = "15.0";
+    final int UNICODE_EMOJI_VERSION_MAJOR = 15;
+    final int UNICODE_EMOJI_VERSION_MINOR = 0;
 
 }

src/org/jcodings/transcode/EConv.java

@@ -732,6 +732,11 @@ public void binmode() {
             if (entry.getTranscoder() != null) transcoders[n++] = entry.getTranscoder();
         }
 
+        if ((flags & LF_NEWLINE_DECORATOR) != 0) {
+            TranscoderDB.Entry entry = TranscoderDB.getEntry(NULL_STRING, "lf_newline".getBytes());
+            if (entry.getTranscoder() != null) transcoders[n++] = entry.getTranscoder();
+        }
+
         int nTrans = numTranscoders;
         int j = 0;
         for (int i = 0; i < nTrans; i++) {

src/org/jcodings/transcode/EConvFlags.java

@@ -29,23 +29,24 @@ public interface EConvFlags {
     final int UNDEF_REPLACE                   = 0x00000020;
     final int UNDEF_HEX_CHARREF               = 0x00000030;
 
-    final int DECORATOR_MASK                  = 0x0000ff00;
-    final int NEWLINE_DECORATOR_MASK          = 0x00003f00;
+    final int DECORATOR_MASK                  = 0x0001ff00;
+    final int NEWLINE_DECORATOR_MASK          = 0x00007f00;
     final int NEWLINE_DECORATOR_READ_MASK      = 0x00000f00;
-    final int NEWLINE_DECORATOR_WRITE_MASK     = 0x00003000;
+    final int NEWLINE_DECORATOR_WRITE_MASK     = 0x00007000;
 
     final int UNIVERSAL_NEWLINE_DECORATOR     = 0x00000100;
     final int CRLF_NEWLINE_DECORATOR          = 0x00001000;
     final int CR_NEWLINE_DECORATOR            = 0x00002000;
-    final int XML_TEXT_DECORATOR              = 0x00004000;
-    final int XML_ATTR_CONTENT_DECORATOR      = 0x00008000;
+    final int LF_NEWLINE_DECORATOR            = 0x00004000;
+    final int XML_TEXT_DECORATOR              = 0x00008000;
+    final int XML_ATTR_CONTENT_DECORATOR      = 0x00010000;
 
     final int STATEFUL_DECORATOR_MASK         = 0x00f00000;
     final int XML_ATTR_QUOTE_DECORATOR        = 0x00100000;
 
 
-    final int PARTIAL_INPUT                    = 0x00010000;
-    final int AFTER_OUTPUT                     = 0x00020000;
+    final int PARTIAL_INPUT                    = 0x00020000;
+    final int AFTER_OUTPUT                     = 0x00040000;
 
     final int  MAX_ECFLAGS_DECORATORS          = 32;
 }

src/org/jcodings/transcode/TranscoderDB.java

@@ -247,6 +247,7 @@ public static int decoratorNames(int ecflags, byte[][] decorators) {
         case UNIVERSAL_NEWLINE_DECORATOR:
         case CRLF_NEWLINE_DECORATOR:
         case CR_NEWLINE_DECORATOR:
+        case LF_NEWLINE_DECORATOR:
         case 0:
             break;
         default:
@@ -263,6 +264,7 @@ public static int decoratorNames(int ecflags, byte[][] decorators) {
 
         if ((ecflags & CRLF_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "crlf_newline".getBytes();
         if ((ecflags & CR_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "cr_newline".getBytes();
+        if ((ecflags & LF_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "lf_newline".getBytes();
         if ((ecflags & UNIVERSAL_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "universal_newline".getBytes();
 
         return numDecorators;

src/org/jcodings/transcode/TranscoderList.java

@@ -101,6 +101,7 @@ static void load() {
         TranscoderDB.declare("", "universal_newline", "Universal_newline");
         TranscoderDB.declare("", "crlf_newline", null /*Crlf_newline*/);
         TranscoderDB.declare("", "cr_newline", null /*Cr_newline*/);
+        TranscoderDB.declare("", "lf_newline", "Universal_newline");
         TranscoderDB.declare("US-ASCII", "UTF-8", null /*From_US_ASCII*/);
         TranscoderDB.declare("UTF-8", "US-ASCII", null /*To_US_ASCII*/);
         TranscoderDB.declare("ASCII-8BIT", "UTF-8", null /*From_ASCII_8BIT*/);
@@ -175,6 +176,8 @@ static void load() {
         TranscoderDB.declare("UTF-8", "IBM862", null /*To_IBM862*/);
         TranscoderDB.declare("IBM863", "UTF-8", null /*From_IBM863*/);
         TranscoderDB.declare("UTF-8", "IBM863", null /*To_IBM863*/);
+        TranscoderDB.declare("IBM864", "UTF-8", null /*From_IBM864*/);
+        TranscoderDB.declare("UTF-8", "IBM864", null /*To_IBM864*/);
         TranscoderDB.declare("IBM865", "UTF-8", null /*From_IBM865*/);
         TranscoderDB.declare("UTF-8", "IBM865", null /*To_IBM865*/);
         TranscoderDB.declare("IBM866", "UTF-8", null /*From_IBM866*/);
@@ -223,7 +226,7 @@ static void load() {
         TranscoderDB.declare("UTF-32", "UTF-8", "From_UTF_32");
         TranscoderDB.declare("UTF-8", "UTF-16", "To_UTF_16");
         TranscoderDB.declare("UTF-8", "UTF-32", "To_UTF_32");
-    }
+    };
 
     static final GenericTranscoderEntry[] GENERIC_LIST = new GenericTranscoderEntry[] {
         new GenericTranscoderEntry("Big5", "UTF-8", 57100, "Big5", 1, 2, 1, AsciiCompatibility.CONVERTER, 0),
@@ -359,36 +362,38 @@ static void load() {
         new GenericTranscoderEntry("UTF-8", "IBM862", 35228, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
         new GenericTranscoderEntry("IBM863", "UTF-8", 35752, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
         new GenericTranscoderEntry("UTF-8", "IBM863", 36132, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("IBM865", "UTF-8", 36656, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "IBM865", 36968, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("IBM866", "UTF-8", 37492, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "IBM866", 37948, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("IBM869", "UTF-8", 38440, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "IBM869", 38956, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("MACCROATIAN", "UTF-8", 39480, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "MACCROATIAN", 40312, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("MACCYRILLIC", "UTF-8", 40836, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "MACCYRILLIC", 41540, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("MACGREEK", "UTF-8", 42064, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "MACGREEK", 42764, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("MACICELAND", "UTF-8", 43288, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "MACICELAND", 44004, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("MACROMAN", "UTF-8", 44528, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "MACROMAN", 45012, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("MACROMANIA", "UTF-8", 45536, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "MACROMANIA", 45912, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("MACTURKISH", "UTF-8", 46432, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "MACTURKISH", 46788, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("MACUKRAINE", "UTF-8", 47312, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "MACUKRAINE", 47536, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("KOI8-U", "UTF-8", 48060, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "KOI8-U", 48844, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("KOI8-R", "UTF-8", 49368, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "KOI8-R", 49900, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("TIS-620", "UTF-8", 50264, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "TIS-620", 50308, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("CP850", "UTF-8", 50832, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
-        new GenericTranscoderEntry("UTF-8", "CP850", 51380, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("IBM864", "UTF-8", 36644, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "IBM864", 37504, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("IBM865", "UTF-8", 38028, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "IBM865", 38340, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("IBM866", "UTF-8", 38864, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "IBM866", 39320, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("IBM869", "UTF-8", 39812, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "IBM869", 40328, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("MACCROATIAN", "UTF-8", 40852, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "MACCROATIAN", 41684, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("MACCYRILLIC", "UTF-8", 42208, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "MACCYRILLIC", 42912, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("MACGREEK", "UTF-8", 43436, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "MACGREEK", 44136, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("MACICELAND", "UTF-8", 44660, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "MACICELAND", 45376, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("MACROMAN", "UTF-8", 45900, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "MACROMAN", 46384, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("MACROMANIA", "UTF-8", 46908, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "MACROMANIA", 47284, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("MACTURKISH", "UTF-8", 47804, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "MACTURKISH", 48160, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("MACUKRAINE", "UTF-8", 48684, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "MACUKRAINE", 48908, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("KOI8-U", "UTF-8", 49432, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "KOI8-U", 50216, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("KOI8-R", "UTF-8", 50740, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "KOI8-R", 51272, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("TIS-620", "UTF-8", 51636, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "TIS-620", 51680, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("CP850", "UTF-8", 52204, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
+        new GenericTranscoderEntry("UTF-8", "CP850", 52752, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
         new GenericTranscoderEntry("CP852", "UTF-8", 29992, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
         new GenericTranscoderEntry("UTF-8", "CP852", 30608, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0),
         new GenericTranscoderEntry("CP855", "UTF-8", 31132, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0),
@@ -416,6 +421,7 @@ public static Transcoder getInstance(String name) {
             case "Eucjp2sjis": return Eucjp2sjis_Transcoder.INSTANCE;
             case "Sjis2eucjp": return Sjis2eucjp_Transcoder.INSTANCE;
             case "Universal_newline": return Universal_newline_Transcoder.INSTANCE;
+            case "Lf_newline": return Universal_newline_Transcoder.INSTANCE;
             case "From_UTF8_MAC": return From_UTF8_MAC_Transcoder.INSTANCE;
             case "From_UTF_16BE": return From_UTF_16BE_Transcoder.INSTANCE;
             case "To_UTF_16BE": return To_UTF_16BE_Transcoder.INSTANCE;