xgettext: Add support for generic XML files

* autogen.sh (GNULIB_MODULES_TOOLS_FOR_SRC): Add trim module.
* gettext-tools/src/locating-rule.h: New file.
* gettext-tools/src/locating-rule.c: New file.
* gettext-tools/src/its.h: New file.
* gettext-tools/src/its.c: New file.
* gettext-tools/src/xgettext.c: Include "locating-rule.h" and "its.h".
(its_locators): New variable.
(long_options): Add --itstool option.
(extract_from_xml_file): New function.
(main): Handle --itstool option.
(usage): Document --itstool option.
* gettext-tools/src/Makefile.am (noinst_HEADERS): Add locating-rule.h
and its.h.
(libgettextsrc_la_CPPFLAGS): Add $(INCXML).
(libgettextsrc_la_SOURCES): Add locating-rule.c and its.c.
* gettext-tools/Makefile.am (SUBDIRS): Add its directory.
* gettext-tools/configure.ac: Output its/Makefile.
* gettext-tools/doc/gettext.texi (Preparing ITS Rules): New section.
* gettext-tools/doc/xgettext.texi: Mention --itstool option.
* gettext-tools/tests/Makefile.am (TESTS): Add new tests.
* gettext-tools/tests/xgettext-its-1: New file
* gettext-tools/tests/init-env.in: Set GETTEXTDATADIR for ITS tests.

1 files changed, 202 insertions, 29 deletions

diff --git a/gettext-tools/src/xgettext.c b/gettext-tools/src/xgettext.c
index 89d4d45..bb63a26 100644
--- a/gettext-tools/src/xgettext.c
+++ b/gettext-tools/src/xgettext.c
@@ -29,6 +29,7 @@
 #include <stdlib.h>
 #include <stdbool.h>
 #include <string.h>
+#include <sys/stat.h>
 #include <locale.h>
 #include <limits.h>
 
@@ -71,6 +72,8 @@
 #include "propername.h"
 #include "sentence.h"
 #include "unistr.h"
+#include "its.h"
+#include "locating-rule.h"
 #include "gettext.h"
 
 /* A convenience macro.  I don't like writing gettext() every time.  */
@@ -105,6 +108,10 @@
 #include "x-desktop.h"
 
 
+#define SIZEOF(a) (sizeof(a) / sizeof(a[0]))
+#define ENDOF(a) ((a) + SIZEOF(a))
+
+
 /* If nonzero add all comments immediately preceding one of the keywords. */
 static bool add_all_comments = false;
 
@@ -205,6 +212,17 @@ const char *xgettext_current_source_encoding;
 iconv_t xgettext_current_source_iconv;
 #endif
 
+static locating_rule_list_ty *its_locating_rules;
+
+#define ITS_ROOT_UNTRANSLATABLE \
+  "<its:rules xmlns:its=\"http://www.w3.org/2005/11/its\"" \
+  "           version=\"2.0\">" \
+  "  <its:translateRule selector=\"/*\" translate=\"no\"/>" \
+  "</its:rules>"
+
+/* If nonzero add comments used by itstool. */
+static bool add_itstool_comments = false;
+
 /* Long options.  */
 static const struct option long_options[] =
 {
@@ -228,6 +246,7 @@ static const struct option long_options[] =
   { "from-code", required_argument, NULL, CHAR_MAX + 3 },
   { "help", no_argument, NULL, 'h' },
   { "indent", no_argument, NULL, 'i' },
+  { "itstool", no_argument, NULL, CHAR_MAX + 19 },
   { "join-existing", no_argument, NULL, 'j' },
   { "kde", no_argument, NULL, CHAR_MAX + 10 },
   { "keyword", optional_argument, NULL, 'k' },
@@ -288,6 +307,9 @@ static void usage (int status)
 static void read_exclusion_file (char *file_name);
 static void extract_from_file (const char *file_name, extractor_ty extractor,
                                msgdomain_list_ty *mdlp);
+static void extract_from_xml_file (const char *file_name,
+                                   its_rule_list_ty *rules,
+                                   msgdomain_list_ty *mdlp);
 static message_ty *construct_header (void);
 static void finalize_header (msgdomain_list_ty *mdlp);
 static extractor_ty language_to_extractor (const char *name);
@@ -306,6 +328,7 @@ main (int argc, char *argv[])
   bool some_additional_keywords = false;
   bool sort_by_msgid = false;
   bool sort_by_filepos = false;
+  char *its_dirs[2] = { NULL, NULL };
   const char *file_name;
   const char *files_from = NULL;
   string_list_ty *file_list;
@@ -378,7 +401,6 @@ main (int argc, char *argv[])
         x_tcl_extract_all ();
         x_perl_extract_all ();
         x_php_extract_all ();
-        x_glade_extract_all ();
         x_lua_extract_all ();
         x_javascript_extract_all ();
         x_vala_extract_all ();
@@ -458,7 +480,6 @@ main (int argc, char *argv[])
         x_tcl_keyword (optarg);
         x_perl_keyword (optarg);
         x_php_keyword (optarg);
-        x_glade_keyword (optarg);
         x_lua_keyword (optarg);
         x_javascript_keyword (optarg);
         x_vala_keyword (optarg);
@@ -634,6 +655,10 @@ main (int argc, char *argv[])
           error (EXIT_FAILURE, 0, _("sentence end type '%s' unknown"), optarg);
         break;
 
+      case CHAR_MAX + 19: /* --itstool */
+        add_itstool_comments = true;
+        break;
+
       default:
         usage (EXIT_FAILURE);
         /* NOTREACHED */
@@ -694,6 +719,30 @@ xgettext cannot work without keywords to look for"));
       usage (EXIT_FAILURE);
     }
 
+  {
+    const char *gettextdatadir;
+    char *versioned_gettextdatadir;
+
+    /* Make it possible to override the locator file location.  This
+       is necessary for running the testsuite before "make
+       install".  */
+    gettextdatadir = getenv ("GETTEXTDATADIR");
+    if (gettextdatadir == NULL || gettextdatadir[0] == '\0')
+      gettextdatadir = relocate (GETTEXTDATADIR);
+
+    its_dirs[0] = xconcatenated_filename (gettextdatadir, "its", NULL);
+
+    versioned_gettextdatadir =
+      xasprintf ("%s%s", relocate (GETTEXTDATADIR), PACKAGE_SUFFIX);
+    its_dirs[1] = xconcatenated_filename (versioned_gettextdatadir, "its",
+                                          NULL);
+    free (versioned_gettextdatadir);
+
+    its_locating_rules = locating_rule_list_alloc ();
+    for (i = 0; i < SIZEOF (its_dirs); i++)
+      locating_rule_list_add_from_directory (its_locating_rules, its_dirs[i]);
+  }
+
   /* Determine extractor from language.  */
   if (language != NULL)
     extractor = language_to_extractor (language);
@@ -792,6 +841,7 @@ This version was built without iconv()."),
     {
       const char *filename;
       extractor_ty this_file_extractor;
+      its_rule_list_ty *its_rules = NULL;
 
       filename = file_list->item[i];
 
@@ -799,11 +849,9 @@ This version was built without iconv()."),
         this_file_extractor = extractor;
       else
         {
+          const char *language_from_extension = NULL;
           const char *base;
           char *reduced;
-          const char *extension;
-          const char *language;
-          const char *p;
 
           base = strrchr (filename, '/');
           if (!base)
@@ -815,39 +863,103 @@ This version was built without iconv()."),
                  && memcmp (reduced + strlen (reduced) - 3, ".in", 3) == 0)
             reduced[strlen (reduced) - 3] = '\0';
 
-          /* Work out what the file extension is.  */
-          language = NULL;
-          p = reduced + strlen (reduced);
-          for (; p > reduced && language == NULL; p--)
+          /* If no language is specified with -L, deduce it the extension.  */
+          if (language == NULL)
             {
-              if (*p == '.')
+              const char *p;
+
+              /* Work out what the file extension is.  */
+              p = reduced + strlen (reduced);
+              for (; p > reduced && language_from_extension == NULL; p--)
                 {
-                  extension = p + 1;
+                  if (*p == '.')
+                    {
+                      const char *extension = p + 1;
 
-                  /* Derive the language from the extension, and the extractor
-                     function from the language.  */
-                  language = extension_to_language (extension);
+                      /* Derive the language from the extension, and
+                         the extractor function from the language.  */
+                      language_from_extension =
+                        extension_to_language (extension);
+                    }
                 }
             }
 
-          if (language == NULL)
+          /* If language is not determined from the file name
+             extension, check ITS locating rules.  */
+          if (language_from_extension == NULL
+              && strcmp (filename, "-") != 0)
             {
-              extension = strrchr (reduced, '.');
-              if (extension == NULL)
-                extension = "";
-              else
-                extension++;
-              error (0, 0, _("\
+              const char *its_basename;
+
+              its_basename = locating_rule_list_locate (its_locating_rules,
+                                                          filename,
+                                                          language);
+
+              if (its_basename != NULL)
+                {
+                  size_t j;
+
+                  its_rules = its_rule_list_alloc ();
+
+                  /* If the ITS file is identified by the name,
+                     set the root element untranslatable.  */
+                  if (language != NULL)
+                    its_rule_list_add_from_string (its_rules,
+                                                   ITS_ROOT_UNTRANSLATABLE);
+
+                  for (j = 0; j < SIZEOF (its_dirs); j++)
+                    {
+                      char *its_filename =
+                        xconcatenated_filename (its_dirs[j], its_basename,
+                                                NULL);
+                      struct stat statbuf;
+                      bool ok = false;
+
+                      if (stat (its_filename, &statbuf) == 0)
+                        ok = its_rule_list_add_from_file (its_rules,
+                                                          its_filename);
+                      free (its_filename);
+                      if (ok)
+                        break;
+                    }
+                  if (j == SIZEOF (its_dirs))
+                    {
+                      its_rule_list_free (its_rules);
+                      its_rules = NULL;
+                    }
+                }
+            }
+
+          if (its_rules == NULL)
+            {
+              if (language_from_extension == NULL)
+                {
+                  const char *extension = strrchr (reduced, '.');
+                  if (extension == NULL)
+                    extension = "";
+                  else
+                    extension++;
+                  error (0, 0, _("\
 warning: file '%s' extension '%s' is unknown; will try C"), filename, extension);
-              language = "C";
+                  language_from_extension = "C";
+                }
+
+              this_file_extractor =
+                language_to_extractor (language_from_extension);
             }
-          this_file_extractor = language_to_extractor (language);
 
           free (reduced);
         }
 
-      /* Extract the strings from the file.  */
-      extract_from_file (filename, this_file_extractor, mdlp);
+      if (its_rules != NULL)
+        {
+          /* Extract the strings from the file, using ITS.  */
+          extract_from_xml_file (filename, its_rules, mdlp);
+          its_rule_list_free (its_rules);
+        }
+      else
+        /* Extract the strings from the file.  */
+        extract_from_file (filename, this_file_extractor, mdlp);
     }
   string_list_free (file_list);
 
@@ -889,6 +1001,12 @@ warning: file '%s' extension '%s' is unknown; will try C"), filename, extension)
   /* Write the PO file.  */
   msgdomain_list_print (mdlp, file_name, output_syntax, force_po, do_debug);
 
+  if (its_locating_rules)
+    locating_rule_list_free (its_locating_rules);
+
+  for (i = 0; i < SIZEOF (its_dirs); i++)
+    free (its_dirs[i]);
+
   exit (EXIT_SUCCESS);
 }
 
@@ -1051,6 +1169,8 @@ Output details:\n"));
       printf (_("\
       --stringtable-output    write out a NeXTstep/GNUstep .strings file\n"));
       printf (_("\
+      --itstool               write out itstool comments\n"));
+      printf (_("\
   -w, --width=NUMBER          set output page width\n"));
       printf (_("\
       --no-wrap               do not break long message lines, longer than\n\
@@ -2112,6 +2232,63 @@ extract_from_file (const char *file_name, extractor_ty extractor,
   free (real_file_name);
 }
 
+static message_ty *
+xgettext_its_extract_callback (message_list_ty *mlp,
+                               const char *msgctxt,
+                               const char *msgid,
+                               lex_pos_ty *pos,
+                               const char *extracted_comment,
+                               const char *marker,
+                               enum its_whitespace_type_ty whitespace)
+{
+  message_ty *message;
+
+  message = remember_a_message (mlp,
+                                msgctxt == NULL ? NULL : xstrdup (msgctxt),
+                                xstrdup (msgid),
+                                null_context, pos,
+                                extracted_comment, NULL);
+
+  if (add_itstool_comments)
+    {
+      char *dot = xasprintf ("(itstool) path: %s", marker);
+      message_comment_dot_append (message, dot);
+      free (dot);
+
+      if (whitespace == ITS_WHITESPACE_PRESERVE)
+        message->do_wrap = no;
+    }
+
+  return message;
+}
+
+static void
+extract_from_xml_file (const char *file_name,
+                       its_rule_list_ty *rules,
+                       msgdomain_list_ty *mdlp)
+{
+  char *logical_file_name;
+  char *real_file_name;
+  FILE *fp = xgettext_open (file_name, &logical_file_name, &real_file_name);
+
+  /* Set the default for the source file encoding.  May be overridden by
+     the extractor function.  */
+  xgettext_current_source_encoding = xgettext_global_source_encoding;
+#if HAVE_ICONV
+  xgettext_current_source_iconv = xgettext_global_source_iconv;
+#endif
+
+  its_rule_list_extract (rules, fp, real_file_name, logical_file_name,
+                         NULL,
+                         mdlp,
+                         xgettext_its_extract_callback);
+
+  if (fp != stdin)
+    fclose (fp);
+  free (logical_file_name);
+  free (real_file_name);
+}
+
 
 
 /* Error message about non-ASCII character in a specific lexical context.  */
@@ -3663,10 +3840,6 @@ finalize_header (msgdomain_list_ty *mdlp)
 }
 
 
-#define SIZEOF(a) (sizeof(a) / sizeof(a[0]))
-#define ENDOF(a) ((a) + SIZEOF(a))
-
-
 static extractor_ty
 language_to_extractor (const char *name)
 {