format-kde: Recognize KUIT markup

Reported by Chusslove Illich in:
https://lists.gnu.org/archive/html/bug-gettext/2015-01/msg00017.html
* gettext-tools/src/format.h (formatstring_kde_kuit): New variable
declaration.
* gettext-tools/src/format.c (formatstring_parsers): Register
formatstring_kde_kuitas a format string parser.
* gettext-tools/src/format-kde-kuit.c: New file.
* gettext-tools/src/message.h (NFORMATS): Increment.
(enum format_type): New enum value format_kde_kuit.
* gettext-tools/src/xgettext.c (xgettext_record_flag): Handle
format_kde_kuit.
* gettext-tools/src/Makefile.am (xgettext_SOURCES): Move
libexpat-compat.c to...
(libgettextsrc_la_SOURCES): ...here.
(xgettext_LDADD): Move @LTLIBEXPAT@ to...
(libgettextsrc_la_LDFLAGS): ...here.
(FORMAT_SOURCE): Add format-kde-kuit.c.
* gettext-tools/libgettextpo/Makefile.am (libgettextpo_la_AUXSOURCES):
Add ../src/format-kde-kuit.c and ../src/libexpat-compat.c.
(libgettextpo_la_LDFLAGS): Add @LTLIBEXPAT@.
* gettext-tools/tests/format-kde-kuit-1: New file.
* gettext-tools/tests/format-kde-kuit-2: New file.
* gettext-tools/tests/Makefile.am (TESTS): Add new tests.
* gettext-tools/doc/gettext.texi (kde-kuit-format): New subsection.
* NEWS: Mention KUIT format string support.

8 files changed, 451 insertions, 7 deletions

diff --git a/gettext-tools/src/ChangeLog b/gettext-tools/src/ChangeLog
index 754a9dc..d1cc42f 100644
--- a/gettext-tools/src/ChangeLog
+++ b/gettext-tools/src/ChangeLog
@@ -1,3 +1,21 @@
+2015-03-06  Daiki Ueno  <ueno@gnu.org>
+
+	format-kde: Recognize KUIT markup
+	Reported by Chusslove Illich in:
+	https://lists.gnu.org/archive/html/bug-gettext/2015-01/msg00017.html
+	* format.h (formatstring_kde_kuit): New variable declaration.
+	* format.c (formatstring_parsers): Register formatstring_kde_kuit
+	as a format string parser.
+	* format-kde-kuit.c: New file.
+	* message.h (NFORMATS): Increment.
+	(enum format_type): New enum value format_kde_kuit.
+	* xgettext.c (xgettext_record_flag): Handle format_kde_kuit.
+	* Makefile.am (xgettext_SOURCES): Move libexpat-compat.c to...
+	(libgettextsrc_la_SOURCES): ...here.
+	(xgettext_LDADD): Move @LTLIBEXPAT@ to...
+	(libgettextsrc_la_LDFLAGS): ...here.
+	(FORMAT_SOURCE): Add format-kde-kuit.c.
+
 2015-03-02  Daiki Ueno  <ueno@gnu.org>
 
 	xgettext: Support message syntax checks
diff --git a/gettext-tools/src/Makefile.am b/gettext-tools/src/Makefile.am
index edb376f..9a23be0 100644
--- a/gettext-tools/src/Makefile.am
+++ b/gettext-tools/src/Makefile.am
@@ -138,6 +138,7 @@ FORMAT_SOURCE += \
   format-qt.c \
   format-qt-plural.c \
   format-kde.c \
+  format-kde-kuit.c \
   format-boost.c \
   format-lua.c \
   format-javascript.c
@@ -148,7 +149,7 @@ $(COMMON_SOURCE) read-catalog.c \
 color.c write-catalog.c write-properties.c write-stringtable.c write-po.c \
 msgl-ascii.c msgl-iconv.c msgl-equal.c msgl-cat.c msgl-header.c msgl-english.c \
 msgl-check.c file-list.c msgl-charset.c po-time.c plural-exp.c plural-eval.c \
-plural-table.c quote.h sentence.h sentence.c \
+plural-table.c quote.h sentence.h sentence.c libexpat-compat.c \
 $(FORMAT_SOURCE) \
 read-desktop.c
 
@@ -180,7 +181,6 @@ xgettext_SOURCES += \
   x-c.c x-po.c x-sh.c x-python.c x-lisp.c x-elisp.c x-librep.c x-scheme.c \
   x-smalltalk.c x-java.c x-csharp.c x-awk.c x-ycp.c x-tcl.c x-perl.c x-php.c \
   x-rst.c x-glade.c x-lua.c x-javascript.c x-vala.c x-gsettings.c \
-  libexpat-compat.c \
   x-desktop.c
 if !WOE32DLL
 msgattrib_SOURCES = msgattrib.c
@@ -247,7 +247,7 @@ urlget_SOURCES = urlget.c
 # use iconv().
 libgettextsrc_la_LDFLAGS = \
   -release @VERSION@ \
-  ../gnulib-lib/libgettextlib.la $(LTLIBUNISTRING) @LTLIBINTL@ @LTLIBICONV@ -lc -no-undefined
+  ../gnulib-lib/libgettextlib.la $(LTLIBUNISTRING) @LTLIBINTL@ @LTLIBICONV@ @LTLIBEXPAT@ -lc -no-undefined
 
 libgettextsrc_la_CPPFLAGS = $(AM_CPPFLAGS)
 
@@ -278,7 +278,7 @@ msgcmp_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ @MSGMERGE_LIBM@ $(WOE32_LDADD
 msgfmt_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
 msgmerge_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ @MSGMERGE_LIBM@ $(WOE32_LDADD) $(OPENMP_CFLAGS)
 msgunfmt_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
-xgettext_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ @LTLIBEXPAT@ @LTLIBICONV@ $(WOE32_LDADD)
+xgettext_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ @LTLIBICONV@ $(WOE32_LDADD)
 msgattrib_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
 msgcat_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
 msgcomm_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
diff --git a/gettext-tools/src/format-kde-kuit.c b/gettext-tools/src/format-kde-kuit.c
new file mode 100644
index 0000000..9584659
--- /dev/null
+++ b/gettext-tools/src/format-kde-kuit.c
@@ -0,0 +1,391 @@
+/* KUIT (KDE UI Text) format strings.
+   Copyright (C) 2015 Free Software Foundation, Inc.
+   Written by Daiki Ueno <ueno@gnu.org>, 2015.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdlib.h>
+
+#include "format.h"
+#include "libexpat-compat.h"
+#include "unistr.h"
+#include "xalloc.h"
+#include "xvasprintf.h"
+#include "gettext.h"
+
+#define _(str) gettext (str)
+
+#define SIZEOF(a) (sizeof(a) / sizeof(a[0]))
+
+
+/* KUIT (KDE User Interface Text) is an XML-like markup which augments
+   translatable strings with semantic information:
+   http://api.kde.org/frameworks-api/frameworks5-apidocs/ki18n/html/prg_guide.html#kuit_markup
+   KUIT can be seen as a fragment of well-formed XML, except that it
+   still allows '&' as a Qt accelerator marker and '%' as a format
+   directive.  */
+
+struct spec
+{
+  /* A format string descriptor returned from formatstring_kde.parse.  */
+  void *base;
+};
+
+#if DYNLOAD_LIBEXPAT || HAVE_LIBEXPAT
+
+/* Callback called when <element> is seen.  */
+static void
+start_element_handler (void *data, const char *name,
+                       const char **attributes)
+{
+  /* Nothing to do for now.  We could check text outside of a
+     structuring tag, etc.  */
+}
+
+/* Callback called when </element> is seen.  */
+static void
+end_element_handler (void *data, const char *name)
+{
+  /* Nothing to do.  */
+}
+
+struct char_range
+{
+  ucs4_t start;
+  ucs4_t end;
+};
+
+/* Character ranges for NameStartChar defined in:
+   http://www.w3.org/TR/REC-xml/#NT-NameStartChar  */
+static const struct char_range name_chars1[] =
+  {
+    { ':', ':' },
+    { 'A', 'Z' },
+    { '_', '_' },
+    { 'a', 'z' },
+    { 0xC0, 0xD6 },
+    { 0xD8, 0xF6 },
+    { 0xF8, 0x2FF },
+    { 0x370, 0x37D },
+    { 0x37F, 0x1FFF },
+    { 0x200C, 0x200D },
+    { 0x2070, 0x218F },
+    { 0x2C00, 0x2FEF },
+    { 0x3001, 0xD7FF },
+    { 0xF900, 0xFDCF },
+    { 0xFDF0, 0xFFFD },
+    { 0x10000, 0xEFFFF }
+  };
+
+/* Character ranges for NameChar, excluding NameStartChar:
+   http://www.w3.org/TR/REC-xml/#NT-NameChar  */
+static const struct char_range name_chars2[] =
+  {
+    { '-', '-' },
+    { '.', '.' },
+    { '0', '9' },
+    { 0xB7, 0xB7 },
+    { 0x0300, 0x036F },
+    { 0x203F, 0x2040 }
+  };
+
+/* Return true if INPUT is an XML reference.  */
+static bool
+is_reference (const char *input)
+{
+  const char *str = input;
+  const char *str_limit = str + strlen (input);
+  ucs4_t uc;
+  int i;
+
+  str += u8_mbtouc (&uc, (const unsigned char *) str, str_limit - str);
+  assert (uc == '&');
+
+  str += u8_mbtouc (&uc, (const unsigned char *) str, str_limit - str);
+
+  /* CharRef */
+  if (uc == '#')
+    {
+      str += u8_mbtouc (&uc, (const unsigned char *) str, str_limit - str);
+      if (uc == 'x')
+        {
+          while (str < str_limit)
+            {
+              str += u8_mbtouc (&uc, (const unsigned char *) str,
+                                str_limit - str);
+              if (!(('0' <= uc && uc <= '9')
+                    || ('A' <= uc && uc <= 'F')
+                    || ('a' <= uc && uc <= 'f')))
+                break;
+            }
+          return uc == ';';
+        }
+      else if ('0' <= uc && uc <= '9')
+        {
+          while (str < str_limit)
+            {
+              str += u8_mbtouc (&uc, (const unsigned char *) str,
+                                str_limit - str);
+              if (!('0' <= uc && uc <= '9'))
+                break;
+            }
+          return uc == ';';
+        }
+    }
+  else
+    {
+      /* EntityRef */
+      for (i = 0; i < SIZEOF (name_chars1); i++)
+        if (name_chars1[i].start <= uc && uc <= name_chars1[i].end)
+          break;
+
+      if (i == SIZEOF (name_chars1))
+        return false;
+
+      while (str < str_limit)
+        {
+          str += u8_mbtouc (&uc, (const unsigned char *) str, str_limit - str);
+          for (i = 0; i < SIZEOF (name_chars1); i++)
+            if (name_chars1[i].start <= uc && uc <= name_chars1[i].end)
+              break;
+          if (i == SIZEOF (name_chars1))
+            {
+              for (i = 0; i < SIZEOF (name_chars2); i++)
+                if (name_chars2[i].start <= uc && uc <= name_chars2[i].end)
+                  break;
+              if (i == SIZEOF (name_chars2))
+                return false;
+            }
+        }
+      return uc == ';';
+    }
+
+  return false;
+}
+
+#endif
+
+
+static void *
+format_parse (const char *format, bool translated, char *fdi,
+              char **invalid_reason)
+{
+  struct spec spec;
+  struct spec *result;
+
+  spec.base = NULL;
+
+#if DYNLOAD_LIBEXPAT || HAVE_LIBEXPAT
+  if (LIBEXPAT_AVAILABLE ())
+    {
+      XML_Parser parser;
+      const char *str = format;
+      const char *str_limit = str + strlen (format);
+      size_t amp_count;
+      char *buffer, *bp;
+
+      for (amp_count = 0; str < str_limit; amp_count++)
+        {
+          const char *amp = strchrnul (str, '&');
+          if (*amp != '&')
+            break;
+          str = amp + 1;
+        }
+
+      buffer = xmalloc (amp_count * 4 + strlen (format) + 16);
+      *buffer = '\0';
+
+      bp = buffer;
+      bp = stpcpy (bp, "<kuit>");
+      str = format;
+      while (str < str_limit)
+        {
+          const char *amp = strchrnul (str, '&');
+
+          bp = stpncpy (bp, str, amp - str);
+          if (*amp != '&')
+            break;
+
+          bp = stpcpy (bp, is_reference (amp) ? "&" : "&amp;");
+          str = amp + 1;
+        }
+      bp = stpcpy (bp, "</kuit>");
+
+      parser = XML_ParserCreate (NULL);
+      if (parser == NULL)
+        {
+          *invalid_reason = xasprintf (_("memory exhausted"));
+          free (buffer);
+          return NULL;
+        }
+
+      XML_SetElementHandler (parser,
+                             start_element_handler,
+                             end_element_handler);
+
+      if (XML_Parse (parser, buffer, strlen (buffer), 0) == 0)
+        {
+          *invalid_reason =
+            xasprintf (_("error while parsing: %s"),
+                       XML_ErrorString (XML_GetErrorCode (parser)));
+          free (buffer);
+          XML_ParserFree (parser);
+          return NULL;
+        }
+
+      if (XML_Parse (parser, NULL, 0, 1) == 0)
+        {
+          *invalid_reason =
+            xasprintf (_("error while parsing: %s"),
+                       XML_ErrorString (XML_GetErrorCode (parser)));
+          free (buffer);
+          XML_ParserFree (parser);
+          return NULL;
+        }
+
+      free (buffer);
+      XML_ParserFree (parser);
+    }
+#endif
+
+  spec.base = formatstring_kde.parse (format, translated, fdi, invalid_reason);
+  if (spec.base == NULL)
+    return NULL;
+
+  result = XMALLOC (struct spec);
+  *result = spec;
+  return result;
+}
+
+static void
+format_free (void *descr)
+{
+  struct spec *spec = descr;
+  formatstring_kde.free (spec->base);
+  free (spec);
+}
+
+static int
+format_get_number_of_directives (void *descr)
+{
+  struct spec *spec = descr;
+  return formatstring_kde.get_number_of_directives (spec->base);
+}
+
+static bool
+format_check (void *msgid_descr, void *msgstr_descr, bool equality,
+              formatstring_error_logger_t error_logger,
+              const char *pretty_msgid, const char *pretty_msgstr)
+{
+  struct spec *msgid_spec = msgid_descr;
+  struct spec *msgstr_spec = msgstr_descr;
+
+  return formatstring_kde.check (msgid_spec->base, msgstr_spec->base, equality,
+                                 error_logger,
+                                 pretty_msgid, pretty_msgstr);
+}
+
+struct formatstring_parser formatstring_kde_kuit =
+{
+  format_parse,
+  format_free,
+  format_get_number_of_directives,
+  NULL,
+  format_check
+};
+
+
+#ifdef TEST
+
+/* Test program: Print the argument list specification returned by
+   format_parse for strings read from standard input.  */
+
+#include <stdio.h>
+
+static void
+format_print (void *descr)
+{
+  struct spec *spec = (struct spec *) descr;
+  unsigned int last;
+  unsigned int i;
+
+  if (spec == NULL)
+    {
+      printf ("INVALID");
+      return;
+    }
+
+  printf ("(");
+  last = 1;
+  for (i = 0; i < spec->numbered_arg_count; i++)
+    {
+      unsigned int number = spec->numbered[i].number;
+
+      if (i > 0)
+        printf (" ");
+      if (number < last)
+        abort ();
+      for (; last < number; last++)
+        printf ("_ ");
+      last = number + 1;
+    }
+  printf (")");
+}
+
+int
+main ()
+{
+  for (;;)
+    {
+      char *line = NULL;
+      size_t line_size = 0;
+      int line_len;
+      char *invalid_reason;
+      void *descr;
+
+      line_len = getline (&line, &line_size, stdin);
+      if (line_len < 0)
+        break;
+      if (line_len > 0 && line[line_len - 1] == '\n')
+        line[--line_len] = '\0';
+
+      invalid_reason = NULL;
+      descr = format_parse (line, false, NULL, &invalid_reason);
+
+      format_print (descr);
+      printf ("\n");
+      if (descr == NULL)
+        printf ("%s\n", invalid_reason);
+
+      free (invalid_reason);
+      free (line);
+    }
+
+  return 0;
+}
+
+/*
+ * For Emacs M-x compile
+ * Local Variables:
+ * compile-command: "/bin/sh ../libtool --tag=CC --mode=link gcc -o a.out -static -O -g -Wall -I.. -I../gnulib-lib -I../intl -DHAVE_CONFIG_H -DTEST format-kde-kuit.c ../gnulib-lib/libgettextlib.la"
+ * End:
+ */
+
+#endif /* TEST */
diff --git a/gettext-tools/src/format.c b/gettext-tools/src/format.c
index c73ad7d..cb5f49b 100644
--- a/gettext-tools/src/format.c
+++ b/gettext-tools/src/format.c
@@ -58,6 +58,7 @@ struct formatstring_parser *formatstring_parsers[NFORMATS] =
   /* format_qt */               &formatstring_qt,
   /* format_qt_plural */        &formatstring_qt_plural,
   /* format_kde */              &formatstring_kde,
+  /* format_kde_kuit */         &formatstring_kde_kuit,
   /* format_boost */            &formatstring_boost,
   /* format_lua */              &formatstring_lua,
   /* format_javascript */       &formatstring_javascript
diff --git a/gettext-tools/src/format.h b/gettext-tools/src/format.h
index d92532d..3214a9d 100644
--- a/gettext-tools/src/format.h
+++ b/gettext-tools/src/format.h
@@ -119,6 +119,7 @@ extern DLL_VARIABLE struct formatstring_parser formatstring_gfc_internal;
 extern DLL_VARIABLE struct formatstring_parser formatstring_qt;
 extern DLL_VARIABLE struct formatstring_parser formatstring_qt_plural;
 extern DLL_VARIABLE struct formatstring_parser formatstring_kde;
+extern DLL_VARIABLE struct formatstring_parser formatstring_kde_kuit;
 extern DLL_VARIABLE struct formatstring_parser formatstring_boost;
 extern DLL_VARIABLE struct formatstring_parser formatstring_lua;
 extern DLL_VARIABLE struct formatstring_parser formatstring_javascript;
diff --git a/gettext-tools/src/message.c b/gettext-tools/src/message.c
index 2596887..00372a2 100644
--- a/gettext-tools/src/message.c
+++ b/gettext-tools/src/message.c
@@ -58,6 +58,7 @@ const char *const format_language[NFORMATS] =
   /* format_qt */               "qt",
   /* format_qt_plursl */        "qt-plural",
   /* format_kde */              "kde",
+  /* format_kde_kuit */         "kde-kuit",
   /* format_boost */            "boost",
   /* format_lua */              "lua",
   /* format_javascript */       "javascript"
@@ -89,6 +90,7 @@ const char *const format_language_pretty[NFORMATS] =
   /* format_qt */               "Qt",
   /* format_qt_plural */        "Qt plural",
   /* format_kde */              "KDE",
+  /* format_kde_kuit */         "KDE KUIT",
   /* format_boost */            "Boost",
   /* format_lua */              "Lua",
   /* format_javascript */       "JavaScript"
diff --git a/gettext-tools/src/message.h b/gettext-tools/src/message.h
index 8b9bc3f..3649e1b 100644
--- a/gettext-tools/src/message.h
+++ b/gettext-tools/src/message.h
@@ -67,11 +67,12 @@ enum format_type
   format_qt,
   format_qt_plural,
   format_kde,
+  format_kde_kuit,
   format_boost,
   format_lua,
   format_javascript
 };
-#define NFORMATS 27     /* Number of format_type enum values.  */
+#define NFORMATS 28     /* Number of format_type enum values.  */
 extern DLL_VARIABLE const char *const format_language[NFORMATS];
 extern DLL_VARIABLE const char *const format_language_pretty[NFORMATS];
 
diff --git a/gettext-tools/src/xgettext.c b/gettext-tools/src/xgettext.c
index 310b349..1c9830e 100644
--- a/gettext-tools/src/xgettext.c
+++ b/gettext-tools/src/xgettext.c
@@ -1864,6 +1864,11 @@ xgettext_record_flag (const char *optionstring)
                                                     name_start, name_end,
                                                     argnum, value, pass);
                     break;
+                  case format_kde_kuit:
+                    flag_context_list_table_insert (&flag_table_cxx_kde, 2,
+                                                    name_start, name_end,
+                                                    argnum, value, pass);
+                    break;
                   case format_boost:
                     flag_context_list_table_insert (&flag_table_cxx_boost, 1,
                                                     name_start, name_end,
@@ -2543,7 +2548,19 @@ meta information, not the empty string.\n")));
                && (possible_format_p (is_format[format_qt])
                    || possible_format_p (is_format[format_qt_plural])
                    || possible_format_p (is_format[format_kde])
-                   || possible_format_p (is_format[format_boost]))))
+                   || possible_format_p (is_format[format_kde_kuit])
+                   || possible_format_p (is_format[format_boost])))
+          /* Avoid flagging a string as kde-format when it's known to
+             be a kde-kuit-format string.  */
+          && !(i == format_kde
+               && possible_format_p (is_format[format_kde_kuit]))
+          /* Avoid flagging a string as kde-kuit-format when it's
+             known to be a kde-format string.  Note that this relies
+             on the fact that format_kde < format_kde_kuit, so a
+             string will be marked as kde-format if both are
+             undecided.  */
+          && !(i == format_kde_kuit
+               && possible_format_p (is_format[format_kde])))
         {
           struct formatstring_parser *parser = formatstring_parsers[i];
           char *invalid_reason = NULL;
@@ -2678,7 +2695,19 @@ remember_a_message_plural (message_ty *mp, char *string,
                  && (possible_format_p (mp->is_format[format_qt])
                      || possible_format_p (mp->is_format[format_qt_plural])
                      || possible_format_p (mp->is_format[format_kde])
-                     || possible_format_p (mp->is_format[format_boost]))))
+                     || possible_format_p (mp->is_format[format_kde_kuit])
+                     || possible_format_p (mp->is_format[format_boost])))
+            /* Avoid flagging a string as kde-format when it's known
+               to be a kde-kuit-format string.  */
+            && !(i == format_kde
+                 && possible_format_p (mp->is_format[format_kde_kuit]))
+            /* Avoid flagging a string as kde-kuit-format when it's
+               known to be a kde-format string.  Note that this relies
+               on the fact that format_kde < format_kde_kuit, so a
+               string will be marked as kde-format if both are
+               undecided.  */
+            && !(i == format_kde_kuit
+                 && possible_format_p (mp->is_format[format_kde])))
           {
             struct formatstring_parser *parser = formatstring_parsers[i];
             char *invalid_reason = NULL;
@@ -3706,6 +3735,7 @@ language_to_extractor (const char *name)
           {
             result.flag_table = &flag_table_cxx_kde;
             result.formatstring_parser2 = &formatstring_kde;
+            result.formatstring_parser3 = &formatstring_kde_kuit;
           }
         /* Likewise for --boost.  */
         if (recognize_format_boost && strcmp (tp->name, "C++") == 0)