Dec 2, 2018 · Dec 2, 2018 · Dec 2, 2018 · Dec 4, 2018 · Dec 5, 2018 · Dec 5, 2018
diff --git a/Lib/test/test_tools/msgfmt_data/.gitattributes b/Lib/test/test_tools/msgfmt_data/.gitattributes
 file1_fr_crlf.po    eol=crlf
 file2_fr_lf.po      eol=lf
diff --git a/Lib/test/test_tools/msgfmt_data/file12_fr.json b/Lib/test/test_tools/msgfmt_data/file12_fr.json
 [
    [
        "",
        "Project-Id-Version: PACKAGE VERSION\nReport-Msgid-Bugs-To: \nPOT-Creation-Date: 2018-11-30 23:57+0100\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME <EMAIL@ADDRESS>\nLanguage-Team: French\nLanguage: fr\nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\nPlural-Forms: nplurals=2; plural=(n > 1);\n"
    ],
    [
        "Bye...",
        "Au revoir ..."
    ],
    [
        "Hello!",
        "Bonjour !"
    ],
    [
        "It's over.",
        "C'est termin\u00e9."
    ],
    [
        [
            "{n} horse",
            0
        ],
        "{n} cheval"
    ],
    [
        [
            "{n} horse",
            1
        ],
        "{n} chevaux"
    ]
 ]
diff --git a/Lib/test/test_tools/msgfmt_data/file12_fr.mo b/Lib/test/test_tools/msgfmt_data/file12_fr.mo
diff --git a/Lib/test/test_tools/msgfmt_data/file1_fr_crlf.json b/Lib/test/test_tools/msgfmt_data/file1_fr_crlf.json
 [
    [
        "",
        "Project-Id-Version: PACKAGE VERSION\nReport-Msgid-Bugs-To: \nPOT-Creation-Date: 2018-11-30 23:46+0100\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME <EMAIL@ADDRESS>\nLanguage-Team: French\nLanguage: fr\nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\nPlural-Forms: nplurals=2; plural=(n > 1);\n"
    ],
    [
        "Hello!",
        "Bonjour !"
    ],
    [
        [
            "{n} horse",
            0
        ],
        "{n} cheval"
    ],
    [
        [
            "{n} horse",
            1
        ],
        "{n} chevaux"
    ]
 ]
diff --git a/Lib/test/test_tools/msgfmt_data/file1_fr_crlf.mo b/Lib/test/test_tools/msgfmt_data/file1_fr_crlf.mo
diff --git a/Lib/test/test_tools/msgfmt_data/file1_fr_crlf.po b/Lib/test/test_tools/msgfmt_data/file1_fr_crlf.po
 # Example of French translations, crlf end of lines
 #
 msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2018-11-30 23:46+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: French\n"
 "Language: fr\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n > 1);\n"

 #: file1.py:6
 msgid "Hello!"
 msgstr "Bonjour !"

 #: file1.py:7
 #, python-brace-format
 msgid "{n} horse"
 msgid_plural "{n} horses"
 msgstr[0] "{n} cheval"
 msgstr[1] "{n} chevaux"
diff --git a/Lib/test/test_tools/msgfmt_data/file2_fr_lf.json b/Lib/test/test_tools/msgfmt_data/file2_fr_lf.json
 [
    [
        "",
        "Project-Id-Version: PACKAGE VERSION\nReport-Msgid-Bugs-To: \nPOT-Creation-Date: 2018-11-30 23:57+0100\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME <EMAIL@ADDRESS>\nLanguage-Team: French\nLanguage: fr\nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\nPlural-Forms: nplurals=2; plural=(n > 1);\n"
    ],
    [
        "Bye...",
        "Au revoir ..."
    ],
    [
        "It's over.",
        "C'est termin\u00e9."
    ]
 ]
diff --git a/Lib/test/test_tools/msgfmt_data/file2_fr_lf.mo b/Lib/test/test_tools/msgfmt_data/file2_fr_lf.mo
diff --git a/Lib/test/test_tools/msgfmt_data/file2_fr_lf.po b/Lib/test/test_tools/msgfmt_data/file2_fr_lf.po
 # Example of French translations, lf end of lines
 #
 msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2018-11-30 23:57+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: French\n"
 "Language: fr\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n > 1);\n"

 #: file2.py:6
 msgid "It's over."
 msgstr "C'est terminé."

 #: file2.py:7
 msgid "Bye..."
 msgstr "Au revoir ..."
diff --git a/Lib/test/test_tools/test_msgfmt.py b/Lib/test/test_tools/test_msgfmt.py
 """Tests for the Tools/i18n/msgfmt.py tool."""
 """Tests for the Tools/i18n/msgfmt.py tool.

 These tests use data files (po and mo) in the msgfmt_data folder.
 The mo files can be generated (if the po file changes, or if msgfmt.py
 slightly changes its output format) by using the --snapshot-update flag
 with this script:

    python test_msgfmt.py --snapshot-update
 """

 import filecmp
 import json
 import os.path
 import shutil
 import struct
 import sys
 import unittest
 from test.support.script_helper import assert_python_failure, assert_python_ok
 from test.test_tools import imports_under_tool, skip_if_missing, toolsdir


 skip_if_missing('i18n')

 data_dir = (Path(__file__).parent / 'msgfmt_data').resolve()
    import msgfmt


 def compile_many_messages(mo_file, *po_files):
    assert_python_ok(msgfmt_py, '-o', mo_file, *po_files)


 def compile_messages(po_file, mo_file):
    assert_python_ok(msgfmt_py, '-o', mo_file, po_file)



 class POParserTest(unittest.TestCase):
    @classmethod
    def tearDownClass(cls):
        # msgfmt uses a global variable to store messages,
        # clear it after the tests.
        msgfmt.MESSAGES.clear()

    def test_strings(self):
        # Test that the PO parser correctly handles and unescape
        # strings in the PO file.
                    # check the result.
                    po = f'msgid {po_string}\nmsgstr "translation"'
                    Path('messages.po').write_text(po)
                    # Reset the global MESSAGES dictionary
                    msgfmt.MESSAGES.clear()
                    msgfmt.make('messages.po', 'messages.mo')
                    msgfmt.make(('messages.po',), 'messages.mo')

                    with open('messages.mo', 'rb') as f:
                        actual = GNUTranslations(f)
                    # Reset the global MESSAGES dictionary
                    msgfmt.MESSAGES.clear()
                    with self.assertRaises(Exception):
                        msgfmt.make('messages.po', 'messages.mo')
                        msgfmt.make(('messages.po',), 'messages.mo')


 class CLITest(unittest.TestCase):
        assert_python_failure(msgfmt_py, 'nonexistent.po')


 class MultiInputTest(unittest.TestCase):
    """Tests for multiple input files
    """

    def test_both_with_outputfile(self):
        """Test script with -o option and 2 input files

        The current behaviour is to merge entries having distinct ids
        and keep last one if the same id occurs in multiple files.

        Here the first file has Windows endings (cflr) while second has
        Unix endings (lf)
        """
        with temp_cwd(None):
            assert_python_ok(msgfmt_py, '-o', 'file12.mo',
                             data_dir / 'file1_fr_crlf.po',
                             data_dir / 'file2_fr_lf.po')
            self.assertTrue(filecmp.cmp(data_dir / 'file12_fr.mo',
                                        'file12.mo'))

    def test_both_without_outputfile(self):
        """Test script without -o option and 2 input files"""

        with temp_cwd(None):
            shutil.copy(data_dir / 'file1_fr_crlf.po', '.')
            shutil.copy(data_dir / 'file2_fr_lf.po', '.')
            assert_python_ok(msgfmt_py, 'file1_fr_crlf.po', 'file2_fr_lf.po')
            self.assertTrue(filecmp.cmp(data_dir / 'file1_fr_crlf.mo',
                                        'file1_fr_crlf.mo'))
            self.assertTrue(filecmp.cmp(data_dir / 'file2_fr_lf.mo',
                                        'file2_fr_lf.mo'))


 class PONamesTest(unittest.TestCase):
    def test_no_extension(self):
        with temp_cwd(None):
            shutil.copy(data_dir / 'file1_fr_crlf.po', 'file1.fr.po')
            assert_python_ok(msgfmt_py, 'file1.fr')
            self.assertTrue(os.path.exists('file1.fr.mo'))

    def test_wrong_extension(self):
        with temp_cwd(None):
            shutil.copy(data_dir / 'file1_fr_crlf.po', 'file1_fr.pox')
            assert_python_failure(msgfmt_py, 'file1_fr.pox')
            self.assertFalse(os.path.exists('file1_fr.mo'))
            self.assertFalse(os.path.exists('file1_fr.pox.mo'))

    @unittest.skipUnless(sys.platform.startswith("win"), "uppercase on Windows")
    def test_MAJ_on_Windows(self):
        with temp_cwd(None):
            shutil.copy(data_dir / 'file1_fr_crlf.po', 'File1.PO')
            assert_python_ok(msgfmt_py, 'FIle1.Po')
            self.assertTrue(os.path.exists('file1.mo'))


 def make_message_files(mo_file, *po_files):
    compile_many_messages(mo_file, *po_files)
    # Create a human-readable JSON file which is
    # easier to review than the binary .mo file.
    with open(mo_file, 'rb') as f:
        translations = GNUTranslations(f)
    catalog_file = mo_file.with_suffix('.json')
    with open(catalog_file, 'w') as f:
        data = translations._catalog.items()
        data = sorted(data, key=lambda x: (isinstance(x[0], tuple), x[0]))
        json.dump(data, f, indent=4)
        f.write('\n')


 def update_catalog_snapshots():
    for po_file in data_dir.glob('*.po'):
        mo_file = po_file.with_suffix('.mo')
        compile_messages(po_file, mo_file)
        # Create a human-readable JSON file which is
        # easier to review than the binary .mo file.
        with open(mo_file, 'rb') as f:
            translations = GNUTranslations(f)
        catalog_file = po_file.with_suffix('.json')
        with open(catalog_file, 'w') as f:
            data = translations._catalog.items()
            data = sorted(data, key=lambda x: (isinstance(x[0], tuple), x[0]))
            json.dump(data, f, indent=4)
            f.write('\n')
        make_message_files(mo_file, po_file)
    # special processing for file12_fr.mo which results from 2 input files
    make_message_files(data_dir / 'file12_fr.mo',
                       data_dir / 'file1_fr_crlf.po',
                       data_dir / 'file2_fr_lf.po')


 if __name__ == '__main__':
diff --git a/Misc/NEWS.d/next/Tools-Demos/2018-12-05-20-46-10.bpo-35335.qtIUBx.rst b/Misc/NEWS.d/next/Tools-Demos/2018-12-05-20-46-10.bpo-35335.qtIUBx.rst
 :program:`msgfmt.py` is now able to merge more than one po file into a compiled mo
 file. When an entry exists in more than on input file, the last file wins.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		file1_fr_crlf.po eol=crlf
		file2_fr_lf.po eol=lf
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,32 @@
		[
		[
		"",
		"Project-Id-Version: PACKAGE VERSION\nReport-Msgid-Bugs-To: \nPOT-Creation-Date: 2018-11-30 23:57+0100\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME <EMAIL@ADDRESS>\nLanguage-Team: French\nLanguage: fr\nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\nPlural-Forms: nplurals=2; plural=(n > 1);\n"
		],
		[
		"Bye...",
		"Au revoir ..."
		],
		[
		"Hello!",
		"Bonjour !"
		],
		[
		"It's over.",
		"C'est termin\u00e9."
		],
		[
		[
		"{n} horse",
		0
		],
		"{n} cheval"
		],
		[
		[
		"{n} horse",
		1
		],
		"{n} chevaux"
		]
		]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,24 @@
		[
		[
		"",
		"Project-Id-Version: PACKAGE VERSION\nReport-Msgid-Bugs-To: \nPOT-Creation-Date: 2018-11-30 23:46+0100\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME <EMAIL@ADDRESS>\nLanguage-Team: French\nLanguage: fr\nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\nPlural-Forms: nplurals=2; plural=(n > 1);\n"
		],
		[
		"Hello!",
		"Bonjour !"
		],
		[
		[
		"{n} horse",
		0
		],
		"{n} cheval"
		],
		[
		[
		"{n} horse",
		1
		],
		"{n} chevaux"
		]
		]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,26 @@
		# Example of French translations, crlf end of lines
		#
		msgid ""
		msgstr ""
		"Project-Id-Version: PACKAGE VERSION\n"
		"Report-Msgid-Bugs-To: \n"
		"POT-Creation-Date: 2018-11-30 23:46+0100\n"
		"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
		"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
		"Language-Team: French\n"
		"Language: fr\n"
		"MIME-Version: 1.0\n"
		"Content-Type: text/plain; charset=UTF-8\n"
		"Content-Transfer-Encoding: 8bit\n"
		"Plural-Forms: nplurals=2; plural=(n > 1);\n"

		#: file1.py:6
		msgid "Hello!"
		msgstr "Bonjour !"

		#: file1.py:7
		#, python-brace-format
		msgid "{n} horse"
		msgid_plural "{n} horses"
		msgstr[0] "{n} cheval"
		msgstr[1] "{n} chevaux"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,14 @@
		[
		[
		"",
		"Project-Id-Version: PACKAGE VERSION\nReport-Msgid-Bugs-To: \nPOT-Creation-Date: 2018-11-30 23:57+0100\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME <EMAIL@ADDRESS>\nLanguage-Team: French\nLanguage: fr\nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\nPlural-Forms: nplurals=2; plural=(n > 1);\n"
		],
		[
		"Bye...",
		"Au revoir ..."
		],
		[
		"It's over.",
		"C'est termin\u00e9."
		]
		]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,23 @@
		# Example of French translations, lf end of lines
		#
		msgid ""
		msgstr ""
		"Project-Id-Version: PACKAGE VERSION\n"
		"Report-Msgid-Bugs-To: \n"
		"POT-Creation-Date: 2018-11-30 23:57+0100\n"
		"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
		"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
		"Language-Team: French\n"
		"Language: fr\n"
		"MIME-Version: 1.0\n"
		"Content-Type: text/plain; charset=UTF-8\n"
		"Content-Transfer-Encoding: 8bit\n"
		"Plural-Forms: nplurals=2; plural=(n > 1);\n"

		#: file2.py:6
		msgid "It's over."
		msgstr "C'est terminé."

		#: file2.py:7
		msgid "Bye..."
		msgstr "Au revoir ..."
Original file line number	Diff line number	Diff line change
		@@ -1,6 +1,17 @@
		"""Tests for the Tools/i18n/msgfmt.py tool."""
		"""Tests for the Tools/i18n/msgfmt.py tool.

		These tests use data files (po and mo) in the msgfmt_data folder.
merwok marked this conversation as resolved. Show resolvedHide resolved
		The mo files can be generated (if the po file changes, or if msgfmt.py
		slightly changes its output format) by using the --snapshot-update flag
		with this script:

		python test_msgfmt.py --snapshot-update
		"""

		import filecmp
		import json
		import os.path
		import shutil
		import struct
		import sys
		import unittest
Expand All		@@ -11,7 +22,6 @@
		from test.support.script_helper import assert_python_failure, assert_python_ok
		from test.test_tools import imports_under_tool, skip_if_missing, toolsdir


		skip_if_missing('i18n')
Copy link Contributor StanFromIrelandMar 1, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. Accidentally removed line here

		data_dir = (Path(__file__).parent / 'msgfmt_data').resolve()
Expand All		@@ -22,6 +32,10 @@
		import msgfmt


		def compile_many_messages(mo_file, *po_files):
		assert_python_ok(msgfmt_py, '-o', mo_file, *po_files)


		def compile_messages(po_file, mo_file):
		assert_python_ok(msgfmt_py, '-o', mo_file, po_file)

Expand DownExpand Up		@@ -145,12 +159,6 @@ def test_generic_syntax_error(self):


		class POParserTest(unittest.TestCase):
merwok marked this conversation as resolved. Show resolvedHide resolved
		@classmethod
		def tearDownClass(cls):
		# msgfmt uses a global variable to store messages,
		# clear it after the tests.
		msgfmt.MESSAGES.clear()

		def test_strings(self):
		# Test that the PO parser correctly handles and unescape
		# strings in the PO file.
Expand DownExpand Up		@@ -202,9 +210,7 @@ def test_strings(self):
		# check the result.
		po = f'msgid {po_string}\nmsgstr "translation"'
		Path('messages.po').write_text(po)
		# Reset the global MESSAGES dictionary
		msgfmt.MESSAGES.clear()
		msgfmt.make('messages.po', 'messages.mo')
		msgfmt.make(('messages.po',), 'messages.mo')

		with open('messages.mo', 'rb') as f:
		actual = GNUTranslations(f)
Expand DownExpand Up		@@ -238,7 +244,7 @@ def test_strings(self):
		# Reset the global MESSAGES dictionary
		msgfmt.MESSAGES.clear()
		with self.assertRaises(Exception):
		msgfmt.make('messages.po', 'messages.mo')
		msgfmt.make(('messages.po',), 'messages.mo')


		class CLITest(unittest.TestCase):
Expand DownExpand Up		@@ -271,20 +277,83 @@ def test_nonexistent_file(self):
		assert_python_failure(msgfmt_py, 'nonexistent.po')


		class MultiInputTest(unittest.TestCase):
		"""Tests for multiple input files
		"""

		def test_both_with_outputfile(self):
		"""Test script with -o option and 2 input files

		The current behaviour is to merge entries having distinct ids
		and keep last one if the same id occurs in multiple files.

		Here the first file has Windows endings (cflr) while second has
		Unix endings (lf)
		"""
		with temp_cwd(None):
		assert_python_ok(msgfmt_py, '-o', 'file12.mo',
		data_dir / 'file1_fr_crlf.po',
		data_dir / 'file2_fr_lf.po')
		self.assertTrue(filecmp.cmp(data_dir / 'file12_fr.mo',
		'file12.mo'))

		def test_both_without_outputfile(self):
		"""Test script without -o option and 2 input files"""

		with temp_cwd(None):
		shutil.copy(data_dir / 'file1_fr_crlf.po', '.')
		shutil.copy(data_dir / 'file2_fr_lf.po', '.')
		assert_python_ok(msgfmt_py, 'file1_fr_crlf.po', 'file2_fr_lf.po')
		self.assertTrue(filecmp.cmp(data_dir / 'file1_fr_crlf.mo',
		'file1_fr_crlf.mo'))
		self.assertTrue(filecmp.cmp(data_dir / 'file2_fr_lf.mo',
		'file2_fr_lf.mo'))


		class PONamesTest(unittest.TestCase):
		def test_no_extension(self):
		with temp_cwd(None):
		shutil.copy(data_dir / 'file1_fr_crlf.po', 'file1.fr.po')
		assert_python_ok(msgfmt_py, 'file1.fr')
		self.assertTrue(os.path.exists('file1.fr.mo'))

		def test_wrong_extension(self):
		with temp_cwd(None):
		shutil.copy(data_dir / 'file1_fr_crlf.po', 'file1_fr.pox')
		assert_python_failure(msgfmt_py, 'file1_fr.pox')
		self.assertFalse(os.path.exists('file1_fr.mo'))
		self.assertFalse(os.path.exists('file1_fr.pox.mo'))

		@unittest.skipUnless(sys.platform.startswith("win"), "uppercase on Windows")
		def test_MAJ_on_Windows(self):
		with temp_cwd(None):
		shutil.copy(data_dir / 'file1_fr_crlf.po', 'File1.PO')
		assert_python_ok(msgfmt_py, 'FIle1.Po')
		self.assertTrue(os.path.exists('file1.mo'))


		def make_message_files(mo_file, *po_files):
		compile_many_messages(mo_file, *po_files)
		# Create a human-readable JSON file which is
		# easier to review than the binary .mo file.
		with open(mo_file, 'rb') as f:
		translations = GNUTranslations(f)
		catalog_file = mo_file.with_suffix('.json')
		with open(catalog_file, 'w') as f:
		data = translations._catalog.items()
		data = sorted(data, key=lambda x: (isinstance(x[0], tuple), x[0]))
		json.dump(data, f, indent=4)
		f.write('\n')


		def update_catalog_snapshots():
		for po_file in data_dir.glob('*.po'):
		mo_file = po_file.with_suffix('.mo')
		compile_messages(po_file, mo_file)
		# Create a human-readable JSON file which is
		# easier to review than the binary .mo file.
		with open(mo_file, 'rb') as f:
		translations = GNUTranslations(f)
		catalog_file = po_file.with_suffix('.json')
		with open(catalog_file, 'w') as f:
		data = translations._catalog.items()
		data = sorted(data, key=lambda x: (isinstance(x[0], tuple), x[0]))
		json.dump(data, f, indent=4)
		f.write('\n')
		make_message_files(mo_file, po_file)
		# special processing for file12_fr.mo which results from 2 input files
		make_message_files(data_dir / 'file12_fr.mo',
		data_dir / 'file1_fr_crlf.po',
		data_dir / 'file2_fr_lf.po')


		if __name__ == '__main__':
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		:program:`msgfmt.py` is now able to merge more than one po file into a compiled mo
		file. When an entry exists in more than on input file, the last file wins.