tools: support full-icu by default

Instead of an English-only icudt64l.dat in the repo,
we now have icudt64l.dat.gz with all locales.

- updated READMEs and docs
- shrinker now copies source, and compresses (bzip2) the ICU data file
- configure expects deps/icu-small to be full ICU with a full
compressed data file

Fixes: https://github.com/nodejs/node/issues/19214
Co-Authored-By: Richard Lau <riclau@uk.ibm.com>
Co-Authored-By: Jan Olaf Krems <jan.krems@gmail.com>
Co-Authored-By: James M Snell <jasnell@gmail.com>
PR-URL: https://github.com/nodejs/node/pull/29522

Reviewed-By: Jan Krems <jan.krems@gmail.com>
Reviewed-By: Jiawen Geng <technicalcute@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Michael Dawson <michael_dawson@ca.ibm.com>
Reviewed-By: Michaël Zasso <targos@protonmail.com>

1 files changed, 21 insertions, 16 deletions

diff --git a/tools/icu/icu-generic.gyp b/tools/icu/icu-generic.gyp
index b8f0d13836..d2d0e5a318 100644
--- a/tools/icu/icu-generic.gyp
+++ b/tools/icu/icu-generic.gyp
@@ -212,16 +212,17 @@
           'conditions': [
             [ 'icu_small == "false"', { # and OS=win
               # full data - just build the full data file, then we are done.
-              'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.obj' ],
+              'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ],
               'dependencies': [ 'genccode#host' ],
               'actions': [
                 {
                   'action_name': 'icudata',
                   'msvs_quote_cmd': 0,
                   'inputs': [ '<(icu_data_in)' ],
-                  'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.obj' ],
+                  'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ],
+                  # on Windows, we can go directly to .obj file (-o) option.
                   'action': [ '<(PRODUCT_DIR)/genccode',
-                              '-o',
+                              '<@(icu_asm_opts)', # -o
                               '-d', '<(SHARED_INTERMEDIATE_DIR)',
                               '-n', 'icudata',
                               '-e', 'icudt<(icu_ver_major)',
@@ -256,9 +257,9 @@
                   'action_name': 'genccode',
                   'msvs_quote_cmd': 0,
                   'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ],
-                  'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.obj' ],
+                  'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ],
                   'action': [ '<(PRODUCT_DIR)/genccode',
-                              '-o',
+                              '<@(icu_asm_opts)', # -o
                               '-d', '<(SHARED_INTERMEDIATE_DIR)/',
                               '-n', 'icudata',
                               '-e', 'icusmdt<(icu_ver_major)',
@@ -266,20 +267,20 @@
                 },
               ],
               # This file contains the small ICU data.
-              'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.obj' ],
+              'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ],
             } ] ], #end of OS==win and icu_small == true
         }, { # OS != win
           'conditions': [
             [ 'icu_small == "false"', {
-              # full data - just build the full data file, then we are done.
-              'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.c' ],
+              # full data - no trim needed
+              'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ],
               'dependencies': [ 'genccode#host', 'icupkg#host', 'icu_implementation#host', 'icu_uconfig' ],
               'include_dirs': [
                 '<(icu_path)/source/common',
               ],
               'actions': [
                 {
-                   # Swap endianness (if needed), or at least copy the file
+                   # Copy the .dat file, swapping endianness if needed.
                    'action_name': 'icupkg',
                    'inputs': [ '<(icu_data_in)' ],
                    'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat' ],
@@ -290,7 +291,7 @@
                              ],
                 },
                 {
-                   # Rename without the endianness marker
+                   # Rename without the endianness marker (icudt64l.dat -> icudt64.dat)
                    'action_name': 'copy',
                    'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat' ],
                    'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ],
@@ -300,12 +301,14 @@
                              ],
                 },
                 {
+                  # convert full ICU data file to .c, or .S, etc.
                   'action_name': 'icudata',
                   'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ],
-                  'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.c' ],
+                  'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ],
                   'action': [ '<(PRODUCT_DIR)/genccode',
                               '-e', 'icudt<(icu_ver_major)',
                               '-d', '<(SHARED_INTERMEDIATE_DIR)',
+                              '<@(icu_asm_opts)',
                               '-f', 'icudt<(icu_ver_major)_dat',
                               '<@(_inputs)' ],
                 },
@@ -318,7 +321,8 @@
               'export_dependent_settings': [ 'icustubdata' ],
               'actions': [
                 {
-                  # trim down ICU
+                  # Trim down ICU.
+                  # Note that icupkg is invoked automatically, swapping endianness if needed.
                   'action_name': 'icutrim',
                   'inputs': [ '<(icu_data_in)', 'icu_small.json' ],
                   'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ],
@@ -333,7 +337,7 @@
                               '-v',
                               '-L', '<(icu_locales)'],
                 }, {
-                  # rename to get the final entrypoint name right
+                  # rename to get the final entrypoint name right (icudt64l.dat -> icusmdt64.dat)
                    'action_name': 'rename',
                    'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ],
                    'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ],
@@ -342,17 +346,18 @@
                                '<@(_outputs)',
                              ],
                 }, {
-                  # build final .dat -> .obj
+                  # For icu-small, always use .c, don't try to use .S, etc.
                   'action_name': 'genccode',
                   'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ],
-                  'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.c' ],
+                  'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ],
                   'action': [ '<(PRODUCT_DIR)/genccode',
+                              '<@(icu_asm_opts)',
                               '-d', '<(SHARED_INTERMEDIATE_DIR)',
                               '<@(_inputs)' ],
                 },
               ],
               # This file contains the small ICU data
-              'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.c' ],
+              'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ],
               # for umachine.h
               'include_dirs': [
                 '<(icu_path)/source/common',