[NaCl SDK] Fixup GCC-generated .d files (second attempt)

The change adds a script that will post-procress
gcc-generated .d files such that they do not have
file-not-found issue when headers are removed or
renamed.

Also, fix an issue in common.mk where it was checking
for IGNORE_DEPS incorrectly.  This was causing dependencies
to always be built which was leading to duplicate builds
of the same project causing race conditions in the build
system.

R=binji@chromium.org
BUG=None

Review URL: https://codereview.chromium.org/17769006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@209229 0039d316-1c4b-4281-b951-d872f2087c98

7 files changed, 207 insertions, 4 deletions

diff --git a/native_client_sdk/src/build_tools/sdk_files.list b/native_client_sdk/src/build_tools/sdk_files.list
index 4008e28..ffa9720 100644
--- a/native_client_sdk/src/build_tools/sdk_files.list
+++ b/native_client_sdk/src/build_tools/sdk_files.list
@@ -1071,6 +1071,7 @@ tools/create_html.py
 tools/create_nmf.py
 tools/decode_dump.py
 [linux,mac]tools/dump_syms
+tools/fix_deps.py
 [linux,mac]tools/minidump_stackwalk
 [linux,mac]tools/minidump_dump
 tools/genhttpfs.py
diff --git a/native_client_sdk/src/tools/common.mk b/native_client_sdk/src/tools/common.mk
index 058af1b..4a5ab62 100644
--- a/native_client_sdk/src/tools/common.mk
+++ b/native_client_sdk/src/tools/common.mk
@@ -28,6 +28,7 @@ TOP_MAKE := $(word 1,$(MAKEFILE_LIST))
 # Figure out which OS we are running on.
 #
 GETOS = python $(NACL_SDK_ROOT)/tools/getos.py
+FIXDEPS = python $(NACL_SDK_ROOT)/tools/fix_deps.py
 OSNAME := $(shell $(GETOS))
 
 
@@ -370,7 +371,7 @@ endif
 #
 # Assign a sensible default to CHROME_PATH.
 #
-CHROME_PATH ?= $(shell python $(NACL_SDK_ROOT)/tools/getos.py --chrome 2> $(DEV_NULL))
+CHROME_PATH ?= $(shell $(GETOS) --chrome 2> $(DEV_NULL))
 
 #
 # Verify we can find the Chrome executable if we need to launch it.
@@ -423,7 +424,7 @@ run_package: check_for_chrome all
 	$(CHROME_PATH) --load-and-launch-app=$(CURDIR) $(CHROME_ARGS)
 
 
-SYSARCH = $(shell python $(NACL_SDK_ROOT)/tools/getos.py --nacl-arch)
+SYSARCH = $(shell $(GETOS) --nacl-arch)
 GDB_ARGS += -D $(TC_PATH)/$(OSNAME)_x86_$(TOOLCHAIN)/bin/$(SYSARCH)-nacl-gdb
 GDB_ARGS += -D $(abspath $(OUTDIR))/$(TARGET)_$(SYSARCH).nexe
 
diff --git a/native_client_sdk/src/tools/fix_deps.py b/native_client_sdk/src/tools/fix_deps.py
new file mode 100755
index 0000000..a10feac
--- /dev/null
+++ b/native_client_sdk/src/tools/fix_deps.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Fixup GCC-generated dependency files.
+
+Modify GCC generated dependency files in-place so they are more suitable
+for including in a GNU Makefile.  Without the fixups, deleting or renaming
+headers can cause the build to be broken. See:
+http://mad-scientist.net/make/autodep.html for more details of the problem.
+"""
+
+import os
+import optparse
+import sys
+
+TAG_LINE = '# Updated by fix_deps.py\n'
+
+
+class Error(Exception):
+  pass
+
+
+def ParseLine(line, new_target):
+  """Parse one line of a GCC-generated deps file.
+
+  Each line contains an optional target and then a list
+  of space seperated dependencies.  Spaces within filenames
+  are escaped with a backslash.
+  """
+  filenames = []
+
+  if new_target and ':' in line:
+    line = line.split(':', 1)[1]
+
+  line = line.strip()
+  line = line.rstrip('\\')
+
+  while True:
+    # Find the next non-escaped space
+    line = line.strip()
+    pos = line.find(' ')
+    while pos > 0 and line[pos-1] == '\\':
+      pos = line.find(' ', pos+1)
+
+    if pos == -1:
+      filenames.append(line)
+      break
+    filenames.append(line[:pos])
+    line = line[pos+1:]
+
+  return filenames
+
+
+def FixupDepFile(filename):
+  if not os.path.exists(filename):
+    raise Error('File not found: %s' % filename)
+
+  outlines = [TAG_LINE]
+  deps = []
+  new_target = True
+  with open(filename) as infile:
+    for line in infile:
+      if line == TAG_LINE:
+        raise Error('Already processed: %s' % filename)
+      outlines.append(line)
+      deps += ParseLine(line, new_target)
+      new_target = line.endswith('\\')
+
+  # For every depenency found output a dummy target with no rules
+  for dep in deps:
+    outlines.append('%s:\n' % dep)
+
+  with open(filename, 'w') as outfile:
+    for line in outlines:
+      outfile.write(line)
+
+
+def main(argv):
+  usage = "usage: %prog [options] <dep_file ...>"
+  parser = optparse.OptionParser(usage=usage, description=__doc__)
+  args = parser.parse_args(argv)[1]
+  if not args:
+    raise parser.error('expected one or more files as arguments')
+  for arg in args:
+    FixupDepFile(arg)
+
+
+if __name__ == '__main__':
+  try:
+    sys.exit(main(sys.argv[1:]))
+  except Error as e:
+    sys.stderr.write('%s: %s\n' % (os.path.basename(__file__), e))
+    sys.exit(1)
diff --git a/native_client_sdk/src/tools/host_gcc.mk b/native_client_sdk/src/tools/host_gcc.mk
index 61b4a7e..562e277 100644
--- a/native_client_sdk/src/tools/host_gcc.mk
+++ b/native_client_sdk/src/tools/host_gcc.mk
@@ -41,15 +41,19 @@ define C_COMPILER_RULE
 -include $(call SRC_TO_DEP,$(1))
 $(call SRC_TO_OBJ,$(1)): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CC  ,$$@,$(HOST_CC) -o $$@ -c $$< -fPIC $(POSIX_FLAGS) $(2) $(LINUX_FLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1))
 endef
 
 define CXX_COMPILER_RULE
 -include $(call SRC_TO_DEP,$(1))
 $(call SRC_TO_OBJ,$(1)): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CXX ,$$@,$(HOST_CXX) -o $$@ -c $$< -fPIC $(POSIX_FLAGS) $(2) $(LINUX_FLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1))
 endef
 
-
+#
+# Compile Macro
+#
 # $1 = Source Name
 # $2 = POSIX Compile Flags
 # $3 = VC Flags (unused)
diff --git a/native_client_sdk/src/tools/nacl_gcc.mk b/native_client_sdk/src/tools/nacl_gcc.mk
index 108039d..d7edbbe 100644
--- a/native_client_sdk/src/tools/nacl_gcc.mk
+++ b/native_client_sdk/src/tools/nacl_gcc.mk
@@ -43,7 +43,6 @@ ARM_LIB ?= $(TC_PATH)/$(OSNAME)_arm_$(TOOLCHAIN)/bin/arm-nacl-ar
 ARM_STRIP ?= $(TC_PATH)/$(OSNAME)_arm_$(TOOLCHAIN)/bin/arm-nacl-strip
 ARM_NM ?= $(TC_PATH)/$(OSNAME)_arm_$(TOOLCHAIN)/bin/arm-nacl-nm
 
-
 # Architecture-specific flags
 X86_32_CFLAGS ?=
 X86_64_CFLAGS ?=
@@ -67,52 +66,64 @@ define C_COMPILER_RULE
 -include $(call SRC_TO_DEP,$(1),_x86_32)
 $(call SRC_TO_OBJ,$(1),_x86_32): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CC  ,$$@,$(X86_32_CC) -o $$@ -c $$< $(POSIX_FLAGS) $(2) $(NACL_CFLAGS) $(X86_32_CFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_x86_32)
 
 -include $(call SRC_TO_DEP,$(1),_x86_64)
 $(call SRC_TO_OBJ,$(1),_x86_64): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CC  ,$$@,$(X86_64_CC) -o $$@ -c $$< $(POSIX_FLAGS) $(2) $(NACL_CFLAGS) $(X86_64_CFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_x86_64)
 
 -include $(call SRC_TO_DEP,$(1),_arm)
 $(call SRC_TO_OBJ,$(1),_arm): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CC  ,$$@,$(ARM_CC) -o $$@ -c $$< $(POSIX_FLAGS) $(2) $(NACL_CFLAGS) $(ARM_CFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_arm)
 
 -include $(call SRC_TO_DEP,$(1),_x86_32_pic)
 $(call SRC_TO_OBJ,$(1),_x86_32_pic): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CC  ,$$@,$(X86_32_CC) -o $$@ -c $$< -fPIC $(POSIX_FLAGS) $(2) $(NACL_CFLAGS) $(X86_32_CFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_x86_32_pic)
 
 -include $(call SRC_TO_DEP,$(1),_x86_64_pic)
 $(call SRC_TO_OBJ,$(1),_x86_64_pic): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CC  ,$$@,$(X86_64_CC) -o $$@ -c $$< -fPIC $(POSIX_FLAGS) $(2) $(NACL_CFLAGS) $(X86_64_CFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_x86_64_pic)
 
 -include $(call SRC_TO_DEP,$(1),_arm_pic)
 $(call SRC_TO_OBJ,$(1),_arm_pic): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CC  ,$$@,$(ARM_CC) -o $$@ -c $$< -fPIC $(POSIX_FLAGS) $(2) $(NACL_CFLAGS) $(ARM_CFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_arm_pic)
 endef
 
 define CXX_COMPILER_RULE
 -include $(call SRC_TO_DEP,$(1),_x86_32)
 $(call SRC_TO_OBJ,$(1),_x86_32): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CXX ,$$@,$(X86_32_CXX) -o $$@ -c $$< $(POSIX_FLAGS) $(2) $(NACL_CXXFLAGS) $(X86_32_CXXFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_x86_32)
 
 -include $(call SRC_TO_DEP,$(1),_x86_64)
 $(call SRC_TO_OBJ,$(1),_x86_64): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CXX ,$$@,$(X86_64_CXX) -o $$@ -c $$< $(POSIX_FLAGS) $(2) $(NACL_CXXFLAGS) $(X86_64_CXXFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_x86_64)
 
 -include $(call SRC_TO_DEP,$(1),_arm)
 $(call SRC_TO_OBJ,$(1),_arm): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CXX ,$$@,$(ARM_CXX) -o $$@ -c $$< $(POSIX_FLAGS) $(2) $(NACL_CXXFLAGS) $(ARM_CXXFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_arm)
 
 -include $(call SRC_TO_DEP,$(1),_x86_32_pic)
 $(call SRC_TO_OBJ,$(1),_x86_32_pic): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CXX ,$$@,$(X86_32_CXX) -o $$@ -c $$< -fPIC $(POSIX_FLAGS) $(2) $(NACL_CXXFLAGS) $(X86_32_CXXFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_x86_32_pic)
 
 -include $(call SRC_TO_DEP,$(1),_x86_64_pic)
 $(call SRC_TO_OBJ,$(1),_x86_64_pic): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CXX ,$$@,$(X86_64_CXX) -o $$@ -c $$< -fPIC $(POSIX_FLAGS) $(2) $(NACL_CXXFLAGS) $(X86_64_CXXFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_x86_64_pic)
 
 -include $(call SRC_TO_DEP,$(1),_arm_pic)
 $(call SRC_TO_OBJ,$(1),_arm_pic): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CXX ,$$@,$(ARM_CXX) -o $$@ -c $$< -fPIC $(POSIX_FLAGS) $(2) $(NACL_CXXFLAGS) $(ARM_CXXFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_arm_pic)
 endef
 
 
diff --git a/native_client_sdk/src/tools/nacl_llvm.mk b/native_client_sdk/src/tools/nacl_llvm.mk
index 784ff62..0b4b56b 100644
--- a/native_client_sdk/src/tools/nacl_llvm.mk
+++ b/native_client_sdk/src/tools/nacl_llvm.mk
@@ -28,12 +28,14 @@ define C_COMPILER_RULE
 -include $(call SRC_TO_DEP,$(1),_pnacl)
 $(call SRC_TO_OBJ,$(1),_pnacl): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CC  ,$$@,$(PNACL_CC) -o $$@ -c $$< $(POSIX_FLAGS) $(2) $(NACL_CFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_pnacl)
 endef
 
 define CXX_COMPILER_RULE
 -include $(call SRC_TO_DEP,$(1),_pnacl)
 $(call SRC_TO_OBJ,$(1),_pnacl): $(1) $(TOP_MAKE) | $(dir $(call SRC_TO_OBJ,$(1)))dir.stamp
 	$(call LOG,CXX ,$$@,$(PNACL_CXX) -o $$@ -c $$< $(POSIX_FLAGS) $(2) $(NACL_CFLAGS))
+	@$(FIXDEPS) $(call SRC_TO_DEP,$(1),_pnacl)
 endef
 
 
diff --git a/native_client_sdk/src/tools/tests/fix_deps_test.py b/native_client_sdk/src/tools/tests/fix_deps_test.py
new file mode 100755
index 0000000..d02f8ac
--- /dev/null
+++ b/native_client_sdk/src/tools/tests/fix_deps_test.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import os
+import sys
+import tempfile
+import unittest
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+PARENT_DIR = os.path.dirname(SCRIPT_DIR)
+DATA_DIR = os.path.join(SCRIPT_DIR, 'data')
+CHROME_SRC = os.path.dirname(os.path.dirname(os.path.dirname(PARENT_DIR)))
+MOCK_DIR = os.path.join(CHROME_SRC, "third_party", "pymock")
+
+# For the mock library
+sys.path.append(MOCK_DIR)
+sys.path.append(PARENT_DIR)
+
+import fix_deps
+import mock
+
+
+class TestFixDeps(unittest.TestCase):
+  def setUp(self):
+    self.tempfile = None
+
+  def tearDown(self):
+    if self.tempfile:
+      os.remove(self.tempfile)
+
+  def testRequiresFile(self):
+    with mock.patch('sys.stderr'):
+      self.assertRaises(SystemExit, fix_deps.main, [])
+
+  def testInvalidOption(self):
+    with mock.patch('sys.stderr'):
+      self.assertRaises(SystemExit, fix_deps.main, ['--foo', 'bar'])
+
+  def testMissingFile(self):
+    with mock.patch('sys.stderr'):
+      self.assertRaises(fix_deps.Error, fix_deps.main, ['nonexistent.file'])
+
+  def testAddsDeps(self):
+    self.tempfile = tempfile.mktemp("_sdktest")
+    with open(self.tempfile, 'w') as out:
+      out.write('foo.o: foo.c foo.h bar.h\n')
+    fix_deps.FixupDepFile(self.tempfile)
+    with open(self.tempfile) as infile:
+      contents = infile.read()
+    lines = contents.splitlines()
+    self.assertEqual(len(lines), 5)
+    self.assertTrue('foo.c:' in lines)
+    self.assertTrue('foo.h:' in lines)
+    self.assertTrue('bar.h:' in lines)
+
+  def testSpacesInFilenames(self):
+    self.tempfile = tempfile.mktemp("_sdktest")
+    with open(self.tempfile, 'w') as out:
+      out.write('foo.o: foo\\ bar.h\n')
+    fix_deps.FixupDepFile(self.tempfile)
+    with open(self.tempfile) as infile:
+      contents = infile.read()
+    lines = contents.splitlines()
+    self.assertEqual(len(lines), 3)
+    self.assertEqual(lines[2], 'foo\\ bar.h:')
+
+  def testColonInFilename(self):
+    self.tempfile = tempfile.mktemp("_sdktest")
+    with open(self.tempfile, 'w') as out:
+      out.write('foo.o: c:foo.c\\\n c:bar.h\n')
+    fix_deps.FixupDepFile(self.tempfile)
+    with open(self.tempfile) as infile:
+      contents = infile.read()
+    lines = contents.splitlines()
+    self.assertEqual(len(lines), 5)
+    self.assertEqual(lines[3], 'c:foo.c:')
+    self.assertEqual(lines[4], 'c:bar.h:')
+
+  def testDoubleInvoke(self):
+    self.tempfile = tempfile.mktemp("_sdktest")
+    with open(self.tempfile, 'w') as out:
+      out.write('foo.o: foo\\ bar.h\n')
+    fix_deps.FixupDepFile(self.tempfile)
+    self.assertRaises(fix_deps.Error, fix_deps.FixupDepFile, self.tempfile)
+
+
+if __name__ == '__main__':
+  unittest.main()