path: root/gettext-tools/src/x-python.c

/* xgettext Python backend.
   Copyright (C) 2002-2003, 2005-2013 Free Software Foundation, Inc.

   This file was written by Bruno Haible <haible@clisp.cons.org>, 2002.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

/* Specification.  */
#include "x-python.h"

#include <assert.h>
#include <errno.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "message.h"
#include "xgettext.h"
#include "error.h"
#include "error-progname.h"
#include "progname.h"
#include "basename.h"
#include "xerror.h"
#include "xvasprintf.h"
#include "xalloc.h"
#include "c-strstr.h"
#include "c-ctype.h"
#include "po-charset.h"
#include "uniname.h"
#include "unistr.h"
#include "gettext.h"

#define _(s) gettext(s)

#define max(a,b) ((a) > (b) ? (a) : (b))

#define SIZEOF(a) (sizeof(a) / sizeof(a[0]))


/* The Python syntax is defined in the Python Reference Manual
   /usr/share/doc/packages/python/html/ref/index.html.
   See also Python-2.0/Parser/tokenizer.c, Python-2.0/Python/compile.c,
   Python-2.0/Objects/unicodeobject.c.  */


/* ====================== Keyword set customization.  ====================== */

/* If true extract all strings.  */
static bool extract_all = false;

static hash_table keywords;
static bool default_keywords = true;


void
x_python_extract_all ()
{
  extract_all = true;
}


void
x_python_keyword (const char *name)
{
  if (name == NULL)
    default_keywords = false;
  else
    {
      const char *end;
      struct callshape shape;
      const char *colon;

      if (keywords.table == NULL)
        hash_init (&keywords, 100);

      split_keywordspec (name, &end, &shape);

      /* The characters between name and end should form a valid C identifier.
         A colon means an invalid parse in split_keywordspec().  */
      colon = strchr (name, ':');
      if (colon == NULL || colon >= end)
        insert_keyword_callshape (&keywords, name, end - name, &shape);
    }
}

/* Finish initializing the keywords hash table.
   Called after argument processing, before each file is processed.  */
static void
init_keywords ()
{
  if (default_keywords)
    {
      /* When adding new keywords here, also update the documentation in
         xgettext.texi!  */
      x_python_keyword ("gettext");
      x_python_keyword ("ugettext");
      x_python_keyword ("dgettext:2");
      x_python_keyword ("ngettext:1,2");
      x_python_keyword ("ungettext:1,2");
      x_python_keyword ("dngettext:2,3");
      x_python_keyword ("_");
      default_keywords = false;
    }
}

void
init_flag_table_python ()
{
  xgettext_record_flag ("gettext:1:pass-python-format");
  xgettext_record_flag ("ugettext:1:pass-python-format");
  xgettext_record_flag ("dgettext:2:pass-python-format");
  xgettext_record_flag ("ngettext:1:pass-python-format");
  xgettext_record_flag ("ngettext:2:pass-python-format");
  xgettext_record_flag ("ungettext:1:pass-python-format");
  xgettext_record_flag ("ungettext:2:pass-python-format");
  xgettext_record_flag ("dngettext:2:pass-python-format");
  xgettext_record_flag ("dngettext:3:pass-python-format");
  xgettext_record_flag ("_:1:pass-python-format");
  /* xgettext_record_flag ("%:1:python-format"); // % is an infix operator! */

  xgettext_record_flag ("gettext:1:pass-python-brace-format");
  xgettext_record_flag ("ugettext:1:pass-python-brace-format");
  xgettext_record_flag ("dgettext:2:pass-python-brace-format");
  xgettext_record_flag ("ngettext:1:pass-python-brace-format");
  xgettext_record_flag ("ngettext:2:pass-python-brace-format");
  xgettext_record_flag ("ungettext:1:pass-python-brace-format");
  xgettext_record_flag ("ungettext:2:pass-python-brace-format");
  xgettext_record_flag ("dngettext:2:pass-python-brace-format");
  xgettext_record_flag ("dngettext:3:pass-python-brace-format");
  xgettext_record_flag ("_:1:pass-python-brace-format");
  /* xgettext_record_flag ("format:1:python-brace-format"); */
}


/* ======================== Reading of characters.  ======================== */

/* Real filename, used in error messages about the input file.  */
static const char *real_file_name;

/* Logical filename and line number, used to label the extracted messages.  */
static char *logical_file_name;
static int line_number;

/* The input file stream.  */
static FILE *fp;


/* 0. Terminate line by \n, regardless whether the external
   representation of a line terminator is CR (Mac), and CR/LF
   (DOS/Windows), as Python treats them equally.  */
static int
phase0_getc ()
{
  int c;

  c = getc (fp);
  if (c == EOF)
    {
      if (ferror (fp))
        error (EXIT_FAILURE, errno, _("error while reading \"%s\""),
               real_file_name);
      return EOF;
    }

  if (c == '\r')
    {
      int c1 = getc (fp);

      if (c1 != EOF && c1 != '\n')
        ungetc (c1, fp);

      /* Seen line terminator CR or CR/LF.  */
      return '\n';
    }

  return c;
}

/* Supports only one pushback character, and not '\n'.  */
static inline void
phase0_ungetc (int c)
{
  if (c != EOF)
    ungetc (c, fp);
}


/* 1. line_number handling.  */

/* Maximum used, roughly a safer MB_LEN_MAX.  */
#define MAX_PHASE1_PUSHBACK 16
static unsigned char phase1_pushback[MAX_PHASE1_PUSHBACK];
static int phase1_pushback_length;

/* Read the next single byte from the input file.  */
static int
phase1_getc ()
{
  int c;

  if (phase1_pushback_length)
    c = phase1_pushback[--phase1_pushback_length];
  else
    c = phase0_getc ();

  if (c == '\n')
    ++line_number;

  return c;
}

/* Supports MAX_PHASE1_PUSHBACK characters of pushback.  */
static void
phase1_ungetc (int c)
{
  if (c != EOF)
    {
      if (c == '\n')
        --line_number;

      if (phase1_pushback_length == SIZEOF (phase1_pushback))
        abort ();
      phase1_pushback[phase1_pushback_length++] = c;
    }
}


/* Phase 2: Conversion to Unicode.
   This is done early because PEP 0263 specifies that conversion to Unicode
   conceptually occurs before tokenization.  A test case where it matters
   is with encodings like BIG5: when a double-byte character ending in 0x5C
   is followed by '\' or 'u0021', the tokenizer must not treat the second
   half of the double-byte character as a backslash.  */

/* End-of-file indicator for functions returning an UCS-4 character.  */
#define UEOF -1

static lexical_context_ty lexical_context;

static int phase2_pushback[max (9, UNINAME_MAX + 3)];
static int phase2_pushback_length;

/* Read the next Unicode UCS-4 character from the input file.  */
static int
phase2_getc ()
{
  if (phase2_pushback_length)
    return phase2_pushback[--phase2_pushback_length];

  if (xgettext_current_source_encoding == po_charset_ascii)
    {
      int c = phase1_getc ();
      if (c == EOF)
        return UEOF;
      if (!c_isascii (c))
        {
          multiline_error (xstrdup (""),
                           xasprintf ("%s\n%s\n",
                                      non_ascii_error_message (lexical_context,
                                                               real_file_name,
                                                               line_number),
                                      _("\
Please specify the source encoding through --from-code or through a comment\n\
as specified in http://www.python.org/peps/pep-0263.html.\n")));
          exit (EXIT_FAILURE);
        }
      return c;
    }
  else if (xgettext_current_source_encoding != po_charset_utf8)
    {
#if HAVE_ICONV
      /* Use iconv on an increasing number of bytes.  Read only as many bytes
         through phase1_getc as needed.  This is needed to give reasonable
         interactive behaviour when fp is connected to an interactive tty.  */
      unsigned char buf[MAX_PHASE1_PUSHBACK];
      size_t bufcount;
      int c = phase1_getc ();
      if (c == EOF)
        return UEOF;
      buf[0] = (unsigned char) c;
      bufcount = 1;

      for (;;)
        {
          unsigned char scratchbuf[6];
          const char *inptr = (const char *) &buf[0];
          size_t insize = bufcount;
          char *outptr = (char *) &scratchbuf[0];
          size_t outsize = sizeof (scratchbuf);

          size_t res = iconv (xgettext_current_source_iconv,
                              (ICONV_CONST char **) &inptr, &insize,
                              &outptr, &outsize);
          /* We expect that a character has been produced if and only if
             some input bytes have been consumed.  */
          if ((insize < bufcount) != (outsize < sizeof (scratchbuf)))
            abort ();
          if (outsize == sizeof (scratchbuf))
            {
              /* No character has been produced.  Must be an error.  */
              if (res != (size_t)(-1))
                abort ();

              if (errno == EILSEQ)
                {
                  /* An invalid multibyte sequence was encountered.  */
                  multiline_error (xstrdup (""),
                                   xasprintf (_("\
%s:%d: Invalid multibyte sequence.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in http://www.python.org/peps/pep-0263.html.\n"),
                                   real_file_name, line_number));
                  exit (EXIT_FAILURE);
                }
              else if (errno == EINVAL)
                {
                  /* An incomplete multibyte character.  */
                  int c;

                  if (bufcount == MAX_PHASE1_PUSHBACK)
                    {
                      /* An overlong incomplete multibyte sequence was
                         encountered.  */
                      multiline_error (xstrdup (""),
                                       xasprintf (_("\
%s:%d: Long incomplete multibyte sequence.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in http://www.python.org/peps/pep-0263.html.\n"),
                                       real_file_name, line_number));
                      exit (EXIT_FAILURE);
                    }

                  /* Read one more byte and retry iconv.  */
                  c = phase1_getc ();
                  if (c == EOF)
                    {
                      multiline_error (xstrdup (""),
                                       xasprintf (_("\
%s:%d: Incomplete multibyte sequence at end of file.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in http://www.python.org/peps/pep-0263.html.\n"),
                                       real_file_name, line_number));
                      exit (EXIT_FAILURE);
                    }
                  if (c == '\n')
                    {
                      multiline_error (xstrdup (""),
                                       xasprintf (_("\
%s:%d: Incomplete multibyte sequence at end of line.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in http://www.python.org/peps/pep-0263.html.\n"),
                                       real_file_name, line_number - 1));
                      exit (EXIT_FAILURE);
                    }
                  buf[bufcount++] = (unsigned char) c;
                }
              else
                error (EXIT_FAILURE, errno, _("%s:%d: iconv failure"),
                       real_file_name, line_number);
            }
          else
            {
              size_t outbytes = sizeof (scratchbuf) - outsize;
              size_t bytes = bufcount - insize;
              ucs4_t uc;

              /* We expect that one character has been produced.  */
              if (bytes == 0)
                abort ();
              if (outbytes == 0)
                abort ();
              /* Push back the unused bytes.  */
              while (insize > 0)
                phase1_ungetc (buf[--insize]);
              /* Convert the character from UTF-8 to UCS-4.  */
              if (u8_mbtoucr (&uc, scratchbuf, outbytes) < (int) outbytes)
                {
                  /* scratchbuf contains an out-of-range Unicode character
                     (> 0x10ffff).  */
                  multiline_error (xstrdup (""),
                                   xasprintf (_("\
%s:%d: Invalid multibyte sequence.\n\
Please specify the source encoding through --from-code or through a comment\n\
as specified in http://www.python.org/peps/pep-0263.html.\n"),
                                   real_file_name, line_number));
                  exit (EXIT_FAILURE);
                }
              return uc;
            }
        }
#else
      /* If we don't have iconv(), the only supported values for
         xgettext_global_source_encoding and thus also for
         xgettext_current_source_encoding are ASCII and UTF-8.  */
      abort ();
#endif
    }
  else
    {
      /* Read an UTF-8 encoded character.  */
      unsigned char buf[6];
      unsigned int count;
      int c;
      ucs4_t uc;

      c = phase1_getc ();
      if (c == EOF)
        return UEOF;
      buf[0] = c;
      count = 1;

      if (buf[0] >= 0xc0)
        {
          c = phase1_getc ();
          if (c == EOF)
            return UEOF;
          buf[1] = c;
          count = 2;
        }

      if (buf[0] >= 0xe0
          && ((buf[1] ^ 0x80) < 0x40))
        {
          c = phase1_getc ();
          if (c == EOF)
            return UEOF;
          buf[2] = c;
          count = 3;
        }

      if (buf[0] >= 0xf0
          && ((buf[1] ^ 0x80) < 0x40)
          && ((buf[2] ^ 0x80) < 0x40))
        {
          c = phase1_getc ();
          if (c == EOF)
            return UEOF;
          buf[3] = c;
          count = 4;
        }

      if (buf[0] >= 0xf8
          && ((buf[1] ^ 0x80) < 0x40)
          && ((buf[2] ^ 0x80) < 0x40)
          && ((buf[3] ^ 0x80) < 0x40))
        {
          c = phase1_getc ();
          if (c == EOF)
            return UEOF;
          buf[4] = c;
          count = 5;
        }

      if (buf[0] >= 0xfc
          && ((buf[1] ^ 0x80) < 0x40)
          && ((buf[2] ^ 0x80) < 0x40)
          && ((buf[3] ^ 0x80) < 0x40)
          && ((buf[4] ^ 0x80) < 0x40))
        {
          c = phase1_getc ();
          if (c == EOF)
            return UEOF;
          buf[5] = c;
          count = 6;
        }

      u8_mbtouc (&uc, buf, count);
      return uc;
    }
}

/* Supports max (9, UNINAME_MAX + 3) pushback characters.  */
static void
phase2_ungetc (int c)
{
  if (c != UEOF)
    {
      if (phase2_pushback_length == SIZEOF (phase2_pushback))
        abort ();
      phase2_pushback[phase2_pushback_length++] = c;
    }
}


/* ========================= Accumulating strings.  ======================== */

/* A string buffer type that allows appending Unicode characters.
   Returns the entire string in UTF-8 encoding.  */

struct unicode_string_buffer
{
  /* The part of the string that has already been converted to UTF-8.  */
  char *utf8_buffer;
  size_t utf8_buflen;
  size_t utf8_allocated;
};

/* Initialize a 'struct unicode_string_buffer' to empty.  */
static inline void
init_unicode_string_buffer (struct unicode_string_buffer *bp)
{
  bp->utf8_buffer = NULL;
  bp->utf8_buflen = 0;
  bp->utf8_allocated = 0;
}

/* Auxiliary function: Ensure count more bytes are available in bp->utf8.  */
static inline void
unicode_string_buffer_append_unicode_grow (struct unicode_string_buffer *bp,
                                           size_t count)
{
  if (bp->utf8_buflen + count > bp->utf8_allocated)
    {
      size_t new_allocated = 2 * bp->utf8_allocated + 10;
      if (new_allocated < bp->utf8_buflen + count)
        new_allocated = bp->utf8_buflen + count;
      bp->utf8_allocated = new_allocated;
      bp->utf8_buffer = xrealloc (bp->utf8_buffer, new_allocated);
    }
}

/* Auxiliary function: Append a Unicode character to bp->utf8.
   uc must be < 0x110000.  */
static inline void
unicode_string_buffer_append_unicode (struct unicode_string_buffer *bp,
                                      unsigned int uc)
{
  unsigned char utf8buf[6];
  int count = u8_uctomb (utf8buf, uc, 6);

  if (count < 0)
    /* The caller should have ensured that uc is not out-of-range.  */
    abort ();

  unicode_string_buffer_append_unicode_grow (bp, count);
  memcpy (bp->utf8_buffer + bp->utf8_buflen, utf8buf, count);
  bp->utf8_buflen += count;
}

/* Return the string buffer's contents.  */
static char *
unicode_string_buffer_result (struct unicode_string_buffer *bp)
{
  /* NUL-terminate it.  */
  unicode_string_buffer_append_unicode_grow (bp, 1);
  bp->utf8_buffer[bp->utf8_buflen] = '\0';
  /* Return it.  */
  return bp->utf8_buffer;
}

/* Free the memory pointed to by a 'struct unicode_string_buffer'.  */
static inline void
free_unicode_string_buffer (struct unicode_string_buffer *bp)
{
  free (bp->utf8_buffer);
}


/* ======================== Accumulating comments.  ======================== */


/* Accumulating a single comment line.  */

static struct unicode_string_buffer comment_buffer;

static inline void
comment_start ()
{
  lexical_context = lc_comment;
  comment_buffer.utf8_buflen = 0;
}

static inline bool
comment_at_start ()
{
  return (comment_buffer.utf8_buflen == 0);
}

static inline void
comment_add (int c)
{
  unicode_string_buffer_append_unicode (&comment_buffer, c);
}

static inline const char *
comment_line_end ()
{
  char *buffer = unicode_string_buffer_result (&comment_buffer);
  size_t buflen = strlen (buffer);

  while (buflen >= 1
         && (buffer[buflen - 1] == ' ' || buffer[buflen - 1] == '\t'))
    --buflen;
  buffer[buflen] = '\0';
  savable_comment_add (buffer);
  lexical_context = lc_outside;
  return buffer;
}


/* These are for tracking whether comments count as immediately before
   keyword.  */
static int last_comment_line;
static int last_non_comment_line;


/* ======================== Recognizing comments.  ======================== */


/* Recognizing the "coding" comment.
   As specified in PEP 0263, it takes the form
     "coding" [":"|"="] {alphanumeric or "-" or "_" or "*"}*
   or
     "set" "fileencoding" "=" {alphanumeric or "-" or "_" or "*"}*
   and is located in a comment in a line that
     - is either the first or second line,
     - is not a continuation line,
     - in the first form, contains no other tokens except this comment.  */

/* Canonicalized encoding name for the current input file.  */
static const char *xgettext_current_file_source_encoding;

#if HAVE_ICONV
/* Converter from xgettext_current_file_source_encoding to UTF-8 (except from
   ASCII or UTF-8, when this conversion is a no-op).  */
static iconv_t xgettext_current_file_source_iconv;
#endif

static inline void
set_current_file_source_encoding (const char *canon_encoding)
{
  xgettext_current_file_source_encoding = canon_encoding;

  if (xgettext_current_file_source_encoding != po_charset_ascii
      && xgettext_current_file_source_encoding != po_charset_utf8)
    {
#if HAVE_ICONV
      iconv_t cd;

      /* Avoid glibc-2.1 bug with EUC-KR.  */
# if ((__GLIBC__ == 2 && __GLIBC_MINOR__ <= 1) && !defined __UCLIBC__) \
     && !defined _LIBICONV_VERSION
      if (strcmp (xgettext_current_file_source_encoding, "EUC-KR") == 0)
        cd = (iconv_t)(-1);
      else
# endif
      cd = iconv_open (po_charset_utf8, xgettext_current_file_source_encoding);
      if (cd == (iconv_t)(-1))
        error_at_line (EXIT_FAILURE, 0, logical_file_name, line_number - 1, _("\
Cannot convert from \"%s\" to \"%s\". %s relies on iconv(), \
and iconv() does not support this conversion."),
               xgettext_current_file_source_encoding, po_charset_utf8,
               basename (program_name));
      xgettext_current_file_source_iconv = cd;
#else
      error_at_line (EXIT_FAILURE, 0, logical_file_name, line_number - 1, _("\
Cannot convert from \"%s\" to \"%s\". %s relies on iconv(). \
This version was built without iconv()."),
             xgettext_global_source_encoding, po_charset_utf8,
             basename (program_name));
#endif
    }

  xgettext_current_source_encoding = xgettext_current_file_source_encoding;
#if HAVE_ICONV
  xgettext_current_source_iconv = xgettext_current_file_source_iconv;
#endif
}

static inline void
try_to_extract_coding (const char *comment)
{
  const char *p = c_strstr (comment, "coding");

  if (p != NULL)
    {
      p += 6;
      if (*p == ':' || *p == '=')
        {
          p++;
          while (*p == ' ' || *p == '\t')
            p++;
          {
            const char *encoding_start = p;

            while (c_isalnum (*p) || *p == '-' || *p == '_' || *p == '.')
              p++;
            {
              const char *encoding_end = p;

              if (encoding_end > encoding_start)
                {
                  /* Extract the encoding string.  */
                  size_t encoding_len = encoding_end - encoding_start;
                  char *encoding = XNMALLOC (encoding_len + 1, char);

                  memcpy (encoding, encoding_start, encoding_len);
                  encoding[encoding_len] = '\0';

                  {
                    /* Canonicalize it.  */
                    const char *canon_encoding = po_charset_canonicalize (encoding);
                    if (canon_encoding == NULL)
                      {
                        error_at_line (0, 0,
                                       logical_file_name, line_number - 1, _("\
Unknown encoding \"%s\". Proceeding with ASCII instead."),
                                       encoding);
                        canon_encoding = po_charset_ascii;
                      }

                    /* Activate it.  */
                    set_current_file_source_encoding (canon_encoding);
                  }

                  free (encoding);
                }
            }
          }
        }
    }
}

/* Tracking whether the current line is a continuation line or contains a
   non-blank character.  */
static bool continuation_or_nonblank_line = false;


/* Phase 3: Outside strings, replace backslash-newline with nothing and a
   comment with nothing.  */

static int
phase3_getc ()
{
  int c;

  for (;;)
    {
      c = phase2_getc ();
      if (c == '\\')
        {
          c = phase2_getc ();
          if (c != '\n')
            {
              phase2_ungetc (c);
              /* This shouldn't happen usually, because "A backslash is
                 illegal elsewhere on a line outside a string literal."  */
              return '\\';
            }
          /* Eat backslash-newline.  */
          continuation_or_nonblank_line = true;
        }
      else if (c == '#')
        {
          /* Eat a comment.  */
          const char *comment;

          last_comment_line = line_number;
          comment_start ();
          for (;;)
            {
              c = phase2_getc ();
              if (c == UEOF || c == '\n')
                break;
              /* We skip all leading white space, but not EOLs.  */
              if (!(comment_at_start () && (c == ' ' || c == '\t')))
                comment_add (c);
            }
          comment = comment_line_end ();
          if (line_number - 1 <= 2 && !continuation_or_nonblank_line)
            try_to_extract_coding (comment);
          continuation_or_nonblank_line = false;
          return c;
        }
      else
        {
          if (c == '\n')
            continuation_or_nonblank_line = false;
          else if (!(c == ' ' || c == '\t' || c == '\f'))
            continuation_or_nonblank_line = true;
          return c;
        }
    }
}

/* Supports only one pushback character.  */
static void
phase3_ungetc (int c)
{
  phase2_ungetc (c);
}


/* ========================= Accumulating strings.  ======================== */

/* Return value of phase7_getuc when EOF is reached.  */
#define P7_EOF (-1)
#define P7_STRING_END (-2)

/* Convert an UTF-16 or UTF-32 code point to a return value that can be
   distinguished from a single-byte return value.  */
#define UNICODE(code) (0x100 + (code))

/* Test a return value of phase7_getuc whether it designates an UTF-16 or
   UTF-32 code point.  */
#define IS_UNICODE(p7_result) ((p7_result) >= 0x100)

/* Extract the UTF-16 or UTF-32 code of a return value that satisfies
   IS_UNICODE.  */
#define UNICODE_VALUE(p7_result) ((p7_result) - 0x100)


/* ========================== Reading of tokens.  ========================== */


enum token_type_ty
{
  token_type_eof,
  token_type_lparen,            /* ( */
  token_type_rparen,            /* ) */
  token_type_comma,             /* , */
  token_type_lbracket,          /* [ */
  token_type_rbracket,          /* ] */
  token_type_string,            /* "abc", 'abc', """abc""", '''abc''' */
  token_type_symbol,            /* symbol, number */
  token_type_plus,              /* + */
  token_type_other              /* misc. operator */
};
typedef enum token_type_ty token_type_ty;

typedef struct token_ty token_ty;
struct token_ty
{
  token_type_ty type;
  char *string;         /* for token_type_string, token_type_symbol */
  refcounted_string_list_ty *comment;   /* for token_type_string */
  int line_number;
};

/* Free the memory pointed to by a 'struct token_ty'.  */
static inline void
free_token (token_ty *tp)
{
  if (tp->type == token_type_string || tp->type == token_type_symbol)
    free (tp->string);
  if (tp->type == token_type_string)
    drop_reference (tp->comment);
}


/* There are two different input syntaxes for strings, "abc" and r"abc",
   and two different input syntaxes for Unicode strings, u"abc" and ur"abc".
   Which escape sequences are understood, i.e. what is interpreted specially
   after backslash?
    "abc"     \<nl> \\ \' \" \a\b\f\n\r\t\v \ooo \xnn
    r"abc"
    u"abc"    \<nl> \\ \' \" \a\b\f\n\r\t\v \ooo \xnn \unnnn \Unnnnnnnn \N{...}
    ur"abc"                                           \unnnn
   The \unnnn values are UTF-16 values; a single \Unnnnnnnn can expand to two
   \unnnn items.  The \ooo and \xnn values are in the current source encoding
   for byte strings, and Unicode code points for Unicode strings.
 */

static int
phase7_getuc (int quote_char,
              bool triple, bool interpret_ansic, bool interpret_unicode,
              unsigned int *backslash_counter)
{
  int c;

  for (;;)
    {
      /* Use phase 2, because phase 3 elides comments.  */
      c = phase2_getc ();

      if (c == UEOF)
        return P7_EOF;

      if (c == quote_char && (interpret_ansic || (*backslash_counter & 1) == 0))
        {
          if (triple)
            {
              int c1 = phase2_getc ();
              if (c1 == quote_char)
                {
                  int c2 = phase2_getc ();
                  if (c2 == quote_char)
                    return P7_STRING_END;
                  phase2_ungetc (c2);
                }
              phase2_ungetc (c1);
              return UNICODE (c);
            }
          else
            return P7_STRING_END;
        }

      if (c == '\n')
        {
          if (triple)
            {
              *backslash_counter = 0;
              return UNICODE ('\n');
            }
          /* In r"..." and ur"..." strings, newline is only allowed
             immediately after an odd number of backslashes (although the
             backslashes are not interpreted!).  */
          if (!(interpret_ansic || (*backslash_counter & 1) == 0))
            {
              *backslash_counter = 0;
              return UNICODE ('\n');
            }
          phase2_ungetc (c);
          error_with_progname = false;
          error (0, 0, _("%s:%d: warning: unterminated string"),
                 logical_file_name, line_number);
          error_with_progname = true;
          return P7_STRING_END;
        }

      if (c != '\\')
        {
          *backslash_counter = 0;
          return UNICODE (c);
        }

      /* Backslash handling.  */

      if (!interpret_ansic && !interpret_unicode)
        {
          ++*backslash_counter;
          return UNICODE ('\\');
        }

      /* Dispatch according to the character following the backslash.  */
      c = phase2_getc ();
      if (c == UEOF)
        {
          ++*backslash_counter;
          return UNICODE ('\\');
        }

      if (interpret_ansic)
        switch (c)
          {
          case '\n':
            continue;
          case '\\':
            ++*backslash_counter;
            return UNICODE (c);
          case '\'': case '"':
            *backslash_counter = 0;
            return UNICODE (c);
          case 'a':
            *backslash_counter = 0;
            return UNICODE ('\a');
          case 'b':
            *backslash_counter = 0;
            return UNICODE ('\b');
          case 'f':
            *backslash_counter = 0;
            return UNICODE ('\f');
          case 'n':
            *backslash_counter = 0;
            return UNICODE ('\n');
          case 'r':
            *backslash_counter = 0;
            return UNICODE ('\r');
          case 't':
            *backslash_counter = 0;
            return UNICODE ('\t');
          case 'v':
            *backslash_counter = 0;
            return UNICODE ('\v');
          case '0': case '1': case '2': case '3': case '4':
          case '5': case '6': case '7':
            {
              int n = c - '0';

              c = phase2_getc ();
              if (c != UEOF)
                {
                  if (c >= '0' && c <= '7')
                    {
                      n = (n << 3) + (c - '0');
                      c = phase2_getc ();
                      if (c != UEOF)
                        {
                          if (c >= '0' && c <= '7')
                            n = (n << 3) + (c - '0');
                          else
                            phase2_ungetc (c);
                        }
                    }
                  else
                    phase2_ungetc (c);
                }
              *backslash_counter = 0;
              if (interpret_unicode)
                return UNICODE (n);
              else
                return (unsigned char) n;
            }
          case 'x':
            {
              int c1 = phase2_getc ();
              int n1;

              if (c1 >= '0' && c1 <= '9')
                n1 = c1 - '0';
              else if (c1 >= 'A' && c1 <= 'F')
                n1 = c1 - 'A' + 10;
              else if (c1 >= 'a' && c1 <= 'f')
                n1 = c1 - 'a' + 10;
              else
                n1 = -1;

              if (n1 >= 0)
                {
                  int c2 = phase2_getc ();
                  int n2;

                  if (c2 >= '0' && c2 <= '9')
                    n2 = c2 - '0';
                  else if (c2 >= 'A' && c2 <= 'F')
                    n2 = c2 - 'A' + 10;
                  else if (c2 >= 'a' && c2 <= 'f')
                    n2 = c2 - 'a' + 10;
                  else
                    n2 = -1;

                  if (n2 >= 0)
                    {
                      int n = (n1 << 4) + n2;
                      *backslash_counter = 0;
                      if (interpret_unicode)
                        return UNICODE (n);
                      else
                        return (unsigned char) n;
                    }

                  phase2_ungetc (c2);
                }
              phase2_ungetc (c1);
              phase2_ungetc (c);
              ++*backslash_counter;
              return UNICODE ('\\');
            }
          }

      if (interpret_unicode)
        {
          if (c == 'u')
            {
              unsigned char buf[4];
              unsigned int n = 0;
              int i;

              for (i = 0; i < 4; i++)
                {
                  int c1 = phase2_getc ();

                  if (c1 >= '0' && c1 <= '9')
                    n = (n << 4) + (c1 - '0');
                  else if (c1 >= 'A' && c1 <= 'F')
                    n = (n << 4) + (c1 - 'A' + 10);
                  else if (c1 >= 'a' && c1 <= 'f')
                    n = (n << 4) + (c1 - 'a' + 10);
                  else
                    {
                      phase2_ungetc (c1);
                      while (--i >= 0)
                        phase2_ungetc (buf[i]);
                      phase2_ungetc (c);
                      ++*backslash_counter;
                      return UNICODE ('\\');
                    }

                  buf[i] = c1;
                }
              *backslash_counter = 0;
              return UNICODE (n);
            }

          if (interpret_ansic)
            {
              if (c == 'U')
                {
                  unsigned char buf[8];
                  unsigned int n = 0;
                  int i;

                  for (i = 0; i < 8; i++)
                    {
                      int c1 = phase2_getc ();

                      if (c1 >= '0' && c1 <= '9')
                        n = (n << 4) + (c1 - '0');
                      else if (c1 >= 'A' && c1 <= 'F')
                        n = (n << 4) + (c1 - 'A' + 10);
                      else if (c1 >= 'a' && c1 <= 'f')
                        n = (n << 4) + (c1 - 'a' + 10);
                      else
                        {
                          phase2_ungetc (c1);
                          while (--i >= 0)
                            phase2_ungetc (buf[i]);
                          phase2_ungetc (c);
                          ++*backslash_counter;
                          return UNICODE ('\\');
                        }

                      buf[i] = c1;
                    }
                  if (n < 0x110000)
                    {
                      *backslash_counter = 0;
                      return UNICODE (n);
                    }

                  error_with_progname = false;
                  error (0, 0, _("%s:%d: warning: invalid Unicode character"),
                         logical_file_name, line_number);
                  error_with_progname = true;

                  while (--i >= 0)
                    phase2_ungetc (buf[i]);
                  phase2_ungetc (c);
                  ++*backslash_counter;
                  return UNICODE ('\\');
                }

              if (c == 'N')
                {
                  int c1 = phase2_getc ();
                  if (c1 == '{')
                    {
                      unsigned char buf[UNINAME_MAX + 1];
                      int i;
                      unsigned int n;

                      for (i = 0; i < UNINAME_MAX; i++)
                        {
                          int c2 = phase2_getc ();
                          if (!(c2 >= ' ' && c2 <= '~'))
                            {
                              phase2_ungetc (c2);
                              while (--i >= 0)
                                phase2_ungetc (buf[i]);
                              phase2_ungetc (c1);
                              phase2_ungetc (c);
                              ++*backslash_counter;
                              return UNICODE ('\\');
                            }
                          if (c2 == '}')
                            break;
                          buf[i] = c2;
                        }
                      buf[i] = '\0';

                      n = unicode_name_character ((char *) buf);
                      if (n != UNINAME_INVALID)
                        {
                          *backslash_counter = 0;
                          return UNICODE (n);
                        }

                      phase2_ungetc ('}');
                      while (--i >= 0)
                        phase2_ungetc (buf[i]);
                    }
                  phase2_ungetc (c1);
                  phase2_ungetc (c);
                  ++*backslash_counter;
                  return UNICODE ('\\');
                }
            }
        }

      phase2_ungetc (c);
      ++*backslash_counter;
      return UNICODE ('\\');
    }
}


/* Combine characters into tokens.  Discard whitespace except newlines at
   the end of logical lines.  */

/* Number of pending open parentheses/braces/brackets.  */
static int open_pbb;

static token_ty phase5_pushback[2];
static int phase5_pushback_length;

static void
phase5_get (token_ty *tp)
{
  int c;

  if (phase5_pushback_length)
    {
      *tp = phase5_pushback[--phase5_pushback_length];
      return;
    }

  for (;;)
    {
      tp->line_number = line_number;
      c = phase3_getc ();

      switch (c)
        {
        case UEOF:
          tp->type = token_type_eof;
          return;

        case ' ':
        case '\t':
        case '\f':
          /* Ignore whitespace and comments.  */
          continue;

        case '\n':
          if (last_non_comment_line > last_comment_line)
            savable_comment_reset ();
          /* Ignore newline if and only if it is used for implicit line
             joining.  */
          if (open_pbb > 0)
            continue;
          tp->type = token_type_other;
          return;
        }

      last_non_comment_line = tp->line_number;

      switch (c)
        {
        case '.':
          {
            int c1 = phase3_getc ();
            phase3_ungetc (c1);
            if (!(c1 >= '0' && c1 <= '9'))
              {

                tp->type = token_type_other;
                return;
              }
          }
          /* FALLTHROUGH */
        case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
        case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
        case 'M': case 'N': case 'O': case 'P': case 'Q':
        case 'S': case 'T':           case 'V': case 'W': case 'X':
        case 'Y': case 'Z':
        case '_':
        case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
        case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
        case 'm': case 'n': case 'o': case 'p': case 'q':
        case 's': case 't':           case 'v': case 'w': case 'x':
        case 'y': case 'z':
        case '0': case '1': case '2': case '3': case '4':
        case '5': case '6': case '7': case '8': case '9':
        symbol:
          /* Symbol, or part of a number.  */
          {
            static char *buffer;
            static int bufmax;
            int bufpos;

            bufpos = 0;
            for (;;)
              {
                if (bufpos >= bufmax)
                  {
                    bufmax = 2 * bufmax + 10;
                    buffer = xrealloc (buffer, bufmax);
                  }
                buffer[bufpos++] = c;
                c = phase3_getc ();
                switch (c)
                  {
                  case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
                  case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
                  case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
                  case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
                  case 'Y': case 'Z':
                  case '_':
                  case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
                  case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
                  case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
                  case 's': case 't': case 'u': case 'v': case 'w': case 'x':
                  case 'y': case 'z':
                  case '0': case '1': case '2': case '3': case '4':
                  case '5': case '6': case '7': case '8': case '9':
                    continue;
                  default:
                    phase3_ungetc (c);
                    break;
                  }
                break;
              }
            if (bufpos >= bufmax)
              {
                bufmax = 2 * bufmax + 10;
                buffer = xrealloc (buffer, bufmax);
              }
            buffer[bufpos] = '\0';
            tp->string = xstrdup (buffer);
            tp->type = token_type_symbol;
            return;
          }

        /* Strings.  */
          {
            struct mixed_string_buffer *bp;
            int quote_char;
            bool interpret_ansic;
            bool interpret_unicode;
            bool triple;
            unsigned int backslash_counter;

            case 'R': case 'r':
              {
                int c1 = phase2_getc ();
                if (c1 == '"' || c1 == '\'')
                  {
                    quote_char = c1;
                    interpret_ansic = false;
                    interpret_unicode = false;
                    goto string;
                  }
                phase2_ungetc (c1);
                goto symbol;
              }

            case 'U': case 'u':
              {
                int c1 = phase2_getc ();
                if (c1 == '"' || c1 == '\'')
                  {
                    quote_char = c1;
                    interpret_ansic = true;
                    interpret_unicode = true;
                    goto string;
                  }
                if (c1 == 'R' || c1 == 'r')
                  {
                    int c2 = phase2_getc ();
                    if (c2 == '"' || c2 == '\'')
                      {
                        quote_char = c2;
                        interpret_ansic = false;
                        interpret_unicode = true;
                        goto string;
                      }
                    phase2_ungetc (c2);
                  }
                phase2_ungetc (c1);
                goto symbol;
              }

            case '"': case '\'':
              quote_char = c;
              interpret_ansic = true;
              interpret_unicode = false;
            string:
              triple = false;
              lexical_context = lc_string;
              {
                int c1 = phase2_getc ();
                if (c1 == quote_char)
                  {
                    int c2 = phase2_getc ();
                    if (c2 == quote_char)
                      triple = true;
                    else
                      {
                        phase2_ungetc (c2);
                        phase2_ungetc (c1);
                      }
                  }
                else
                  phase2_ungetc (c1);
              }
              backslash_counter = 0;
              /* Start accumulating the string.  */
              bp = mixed_string_buffer_alloc (lexical_context,
                                              logical_file_name,
                                              line_number);
              for (;;)
                {
                  int uc = phase7_getuc (quote_char, triple, interpret_ansic,
                                         interpret_unicode, &backslash_counter);

                  bp->line_number = line_number;

                  if (uc == P7_EOF || uc == P7_STRING_END)
                    break;

                  if (IS_UNICODE (uc))
                    {
                      assert (UNICODE_VALUE (uc) >= 0
                              && UNICODE_VALUE (uc) < 0x110000);
                      mixed_string_buffer_append_unicode (bp,
                                                          UNICODE_VALUE (uc));
                    }
                  else
                    mixed_string_buffer_append_char (bp, uc);
                }
              tp->string = xstrdup (mixed_string_buffer_done (bp));
              tp->comment = add_reference (savable_comment);
              lexical_context = lc_outside;
              tp->type = token_type_string;
              return;
          }

        case '(':
          open_pbb++;
          tp->type = token_type_lparen;
          return;

        case ')':
          if (open_pbb > 0)
            open_pbb--;
          tp->type = token_type_rparen;
          return;

        case ',':
          tp->type = token_type_comma;
          return;

        case '[': case '{':
          open_pbb++;
          tp->type = (c == '[' ? token_type_lbracket : token_type_other);
          return;

        case ']': case '}':
          if (open_pbb > 0)
            open_pbb--;
          tp->type = (c == ']' ? token_type_rbracket : token_type_other);
          return;

        case '+':
          tp->type = token_type_plus;
          return;

        default:
          /* We could carefully recognize each of the 2 and 3 character
             operators, but it is not necessary, as we only need to recognize
             gettext invocations.  Don't bother.  */
          tp->type = token_type_other;
          return;
        }
    }
}

/* Supports only one pushback token.  */
static void
phase5_unget (token_ty *tp)
{
  if (tp->type != token_type_eof)
    {
      if (phase5_pushback_length == SIZEOF (phase5_pushback))
        abort ();
      phase5_pushback[phase5_pushback_length++] = *tp;
    }
}


/* Combine adjacent strings to form a single string.  Note that the end
   of a logical line appears as a token of its own, therefore strings that
   belong to different logical lines will not be concatenated.  */

static void
x_python_lex (token_ty *tp)
{
  phase5_get (tp);
  if (tp->type == token_type_string)
    {
      char *sum = tp->string;
      size_t sum_len = strlen (sum);

      for (;;)
        {
          token_ty token2, *tp2 = NULL;

          phase5_get (&token2);
          switch (token2.type)
            {
            case token_type_plus:
              {
                token_ty token3;

                phase5_get (&token3);
                if (token3.type == token_type_string)
                  {
                    free_token (&token2);
                    tp2 = &token3;
                  }
                else
                  phase5_unget (&token3);
              }
              break;
            case token_type_string:
              tp2 = &token2;
              break;
            default:
              break;
            }

          if (tp2)
            {
              char *addend = tp2->string;
              size_t addend_len = strlen (addend);

              sum = (char *) xrealloc (sum, sum_len + addend_len + 1);
              memcpy (sum + sum_len, addend, addend_len + 1);
              sum_len += addend_len;

              free_token (tp2);
              continue;
            }
          phase5_unget (&token2);
          break;
        }
      tp->string = sum;
    }
}


/* ========================= Extracting strings.  ========================== */


/* Context lookup table.  */
static flag_context_list_table_ty *flag_context_list_table;


/* The file is broken into tokens.  Scan the token stream, looking for
   a keyword, followed by a left paren, followed by a string.  When we
   see this sequence, we have something to remember.  We assume we are
   looking at a valid C or C++ program, and leave the complaints about
   the grammar to the compiler.

     Normal handling: Look for
       keyword ( ... msgid ... )
     Plural handling: Look for
       keyword ( ... msgid ... msgid_plural ... )

   We use recursion because the arguments before msgid or between msgid
   and msgid_plural can contain subexpressions of the same form.  */


/* Extract messages until the next balanced closing parenthesis or bracket.
   Extracted messages are added to MLP.
   DELIM can be either token_type_rparen or token_type_rbracket, or
   token_type_eof to accept both.
   Return true upon eof, false upon closing parenthesis or bracket.  */
static bool
extract_balanced (message_list_ty *mlp,
                  token_type_ty delim,
                  flag_context_ty outer_context,
                  flag_context_list_iterator_ty context_iter,
                  struct arglist_parser *argparser)
{
  /* Current argument number.  */
  int arg = 1;
  /* 0 when no keyword has been seen.  1 right after a keyword is seen.  */
  int state;
  /* Parameters of the keyword just seen.  Defined only in state 1.  */
  const struct callshapes *next_shapes = NULL;
  /* Context iterator that will be used if the next token is a '('.  */
  flag_context_list_iterator_ty next_context_iter =
    passthrough_context_list_iterator;
  /* Current context.  */
  flag_context_ty inner_context =
    inherited_context (outer_context,
                       flag_context_list_iterator_advance (&context_iter));

  /* Start state is 0.  */
  state = 0;

  for (;;)
    {
      token_ty token;

      x_python_lex (&token);
      switch (token.type)
        {
        case token_type_symbol:
          {
            void *keyword_value;

            if (hash_find_entry (&keywords, token.string, strlen (token.string),
                                 &keyword_value)
                == 0)
              {
                next_shapes = (const struct callshapes *) keyword_value;
                state = 1;
              }
            else
              state = 0;
          }
          next_context_iter =
            flag_context_list_iterator (
              flag_context_list_table_lookup (
                flag_context_list_table,
                token.string, strlen (token.string)));
          free (token.string);
          continue;

        case token_type_lparen:
          if (extract_balanced (mlp, token_type_rparen,
                                inner_context, next_context_iter,
                                arglist_parser_alloc (mlp,
                                                      state ? next_shapes : NULL)))
            {
              xgettext_current_source_encoding = po_charset_utf8;
              arglist_parser_done (argparser, arg);
              xgettext_current_source_encoding = xgettext_current_file_source_encoding;
              return true;
            }
          next_context_iter = null_context_list_iterator;
          state = 0;
          continue;

        case token_type_rparen:
          if (delim == token_type_rparen || delim == token_type_eof)
            {
              xgettext_current_source_encoding = po_charset_utf8;
              arglist_parser_done (argparser, arg);
              xgettext_current_source_encoding = xgettext_current_file_source_encoding;
              return false;
            }
          next_context_iter = null_context_list_iterator;
          state = 0;
          continue;

        case token_type_comma:
          arg++;
          inner_context =
            inherited_context (outer_context,
                               flag_context_list_iterator_advance (
                                 &context_iter));
          next_context_iter = passthrough_context_list_iterator;
          state = 0;
          continue;

        case token_type_lbracket:
          if (extract_balanced (mlp, token_type_rbracket,
                                null_context, null_context_list_iterator,
                                arglist_parser_alloc (mlp, NULL)))
            {
              xgettext_current_source_encoding = po_charset_utf8;
              arglist_parser_done (argparser, arg);
              xgettext_current_source_encoding = xgettext_current_file_source_encoding;
              return true;
            }
          next_context_iter = null_context_list_iterator;
          state = 0;
          continue;

        case token_type_rbracket:
          if (delim == token_type_rbracket || delim == token_type_eof)
            {
              xgettext_current_source_encoding = po_charset_utf8;
              arglist_parser_done (argparser, arg);
              xgettext_current_source_encoding = xgettext_current_file_source_encoding;
              return false;
            }
          next_context_iter = null_context_list_iterator;
          state = 0;
          continue;

        case token_type_string:
          {
            lex_pos_ty pos;
            pos.file_name = logical_file_name;
            pos.line_number = token.line_number;

            xgettext_current_source_encoding = po_charset_utf8;
            if (extract_all)
              remember_a_message (mlp, NULL, token.string, inner_context,
                                  &pos, NULL, token.comment);
            else
              arglist_parser_remember (argparser, arg, token.string,
                                       inner_context,
                                       pos.file_name, pos.line_number,
                                       token.comment);
            xgettext_current_source_encoding = xgettext_current_file_source_encoding;
          }
          drop_reference (token.comment);
          next_context_iter = null_context_list_iterator;
          state = 0;
          continue;

        case token_type_eof:
          xgettext_current_source_encoding = po_charset_utf8;
          arglist_parser_done (argparser, arg);
          xgettext_current_source_encoding = xgettext_current_file_source_encoding;
          return true;

        case token_type_plus:
        case token_type_other:
          next_context_iter = null_context_list_iterator;
          state = 0;
          continue;

        default:
          abort ();
        }
    }
}


void
extract_python (FILE *f,
                const char *real_filename, const char *logical_filename,
                flag_context_list_table_ty *flag_table,
                msgdomain_list_ty *mdlp)
{
  message_list_ty *mlp = mdlp->item[0]->messages;

  fp = f;
  real_file_name = real_filename;
  logical_file_name = xstrdup (logical_filename);
  line_number = 1;

  lexical_context = lc_outside;

  last_comment_line = -1;
  last_non_comment_line = -1;

  xgettext_current_file_source_encoding = xgettext_global_source_encoding;
#if HAVE_ICONV
  xgettext_current_file_source_iconv = xgettext_global_source_iconv;
#endif

  xgettext_current_source_encoding = xgettext_current_file_source_encoding;
#if HAVE_ICONV
  xgettext_current_source_iconv = xgettext_current_file_source_iconv;
#endif

  continuation_or_nonblank_line = false;

  open_pbb = 0;

  flag_context_list_table = flag_table;

  init_keywords ();

  /* Eat tokens until eof is seen.  When extract_balanced returns
     due to an unbalanced closing parenthesis, just restart it.  */
  while (!extract_balanced (mlp, token_type_eof,
                            null_context, null_context_list_iterator,
                            arglist_parser_alloc (mlp, NULL)))
    ;

  fp = NULL;
  real_file_name = NULL;
  logical_file_name = NULL;
  line_number = 0;
}