diff options
author | initial.commit <initial.commit@0039d316-1c4b-4281-b951-d872f2087c98> | 2008-07-27 00:12:16 +0000 |
---|---|---|
committer | initial.commit <initial.commit@0039d316-1c4b-4281-b951-d872f2087c98> | 2008-07-27 00:12:16 +0000 |
commit | 920c091ac3ee15079194c82ae8a7a18215f3f23c (patch) | |
tree | d28515d1e7732e2b6d077df1b4855ace3f4ac84f /tools | |
parent | ae2c20f398933a9e86c387dcc465ec0f71065ffc (diff) | |
download | chromium_src-920c091ac3ee15079194c82ae8a7a18215f3f23c.zip chromium_src-920c091ac3ee15079194c82ae8a7a18215f3f23c.tar.gz chromium_src-920c091ac3ee15079194c82ae8a7a18215f3f23c.tar.bz2 |
Add tools to the repository.
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@17 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'tools')
195 files changed, 33753 insertions, 0 deletions
diff --git a/tools/channel_changer/channel_changer.cc b/tools/channel_changer/channel_changer.cc new file mode 100644 index 0000000..89f1e37a --- /dev/null +++ b/tools/channel_changer/channel_changer.cc @@ -0,0 +1,245 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <Windowsx.h> + +#include "base/registry.h" + +#include "Resource.h" + +// This enum needs to be in sync with the strings below. +enum Branch { + UNKNOWN_BRANCH = 0, + DEV_BRANCH, + BETA_BRANCH, +}; + +// This vector of strings needs to be in sync with the Branch enum above. +static const wchar_t* const kBranchStrings[] = { + L"", + L"1.1-dev", + L"1.1-beta" +}; + +// This vector of strings needs to be in sync with the Branch enum above. +static const wchar_t* const kBranchStringsReadable[] = { + L"", + L"Dev", + L"Beta" +}; + +// The root key for Google Update. +static const HKEY kGoogleUpdateRoot = HKEY_CURRENT_USER; + +// The Google Update key to read to find out which branch you are on. +static const wchar_t* const kGoogleUpdateKey = + L"Software\\Google\\Update\\ClientState\\" + L"{8A69D345-D564-463C-AFF1-A69D9E530F96}"; + +// The Google Update value that defines which branch you are on. +static const wchar_t* const kBranchKey = L"ap"; + +// The suffix Google Update sometimes adds to the channel name (channel names +// are defined in kBranchStrings), indicating that a full install is needed. We +// strip this out (if present) for the purpose of determining which channel you +// are on. +static const wchar_t* const kChannelSuffix = L"-full"; + +// The icon to use. +static HICON dlg_icon = NULL; + +void SetMainLabel(HWND dialog, Branch branch) { + std::wstring main_label = L"You are currently on "; + if (branch == DEV_BRANCH || branch == BETA_BRANCH) + main_label += std::wstring(L"the ") + kBranchStringsReadable[branch] + + std::wstring(L" channel"); + else + main_label += L"NO UPDATE CHANNEL"; + + main_label += L". Choose a different channel and click Update, " + L"or click Close to stay on this channel."; + + SetWindowText(GetDlgItem(dialog, IDC_LABEL_MAIN), main_label.c_str()); +} + +void OnInitDialog(HWND dialog) { + SendMessage(dialog, WM_SETICON, (WPARAM) false, (LPARAM) dlg_icon); + + std::wstring branch_string; + RegKey google_update(kGoogleUpdateRoot, kGoogleUpdateKey, KEY_READ); + if (google_update.Valid() && + !google_update.ReadValue(kBranchKey, &branch_string)) { + // If the 'ap' value is missing, we create it, unless the key is missing. + RegKey write_default(kGoogleUpdateRoot, kGoogleUpdateKey, KEY_WRITE); + branch_string = kBranchStrings[BETA_BRANCH]; + if (!write_default.WriteValue(kBranchKey, branch_string.c_str())) + branch_string = L""; // Error, show disabled UI. + } + + // We look for '1.1-beta' or '1.1-dev', but Google Update might have added + // '-full' to the channel name, which we need to strip out to determine what + // channel you are on. + std::wstring suffix = kChannelSuffix; + if (branch_string.length() > suffix.length()) { + size_t index = branch_string.rfind(suffix); + if (index != std::wstring::npos && + index == branch_string.length() - suffix.length()) { + branch_string = branch_string.substr(0, index); + } + } + + Branch branch = UNKNOWN_BRANCH; + if (branch_string == kBranchStrings[DEV_BRANCH]) { + branch = DEV_BRANCH; + } else if (branch_string == kBranchStrings[BETA_BRANCH]) { + branch = BETA_BRANCH; + } else { + // Hide the controls we can't use. + EnableWindow(GetDlgItem(dialog, IDOK), false); + EnableWindow(GetDlgItem(dialog, IDC_STABLE), false); + EnableWindow(GetDlgItem(dialog, IDC_CUTTING_EDGE), false); + + MessageBox(dialog, L"KEY NOT FOUND\n\nChrome is not installed, or is not " + L"using GoogleUpdate for updates.", + L"Chrome Channel Changer", + MB_ICONEXCLAMATION | MB_OK); + } + + SetMainLabel(dialog, branch); + + CheckDlgButton(dialog, IDC_STABLE, + branch == BETA_BRANCH ? BST_CHECKED : BST_UNCHECKED); + CheckDlgButton(dialog, IDC_CUTTING_EDGE, + branch == DEV_BRANCH ? BST_CHECKED : BST_UNCHECKED); +} + +INT_PTR OnCtlColorStatic(HWND dialog, WPARAM wparam, LPARAM lparam) { + HDC hdc = reinterpret_cast<HDC>(wparam); + HWND control_wnd = reinterpret_cast<HWND>(lparam); + + if (GetDlgItem(dialog, IDC_STABLE) == control_wnd || + GetDlgItem(dialog, IDC_CUTTING_EDGE) == control_wnd || + GetDlgItem(dialog, IDC_LABEL_MAIN) == control_wnd || + GetDlgItem(dialog, IDC_SECONDARY_LABEL) == control_wnd) { + SetBkMode(hdc, TRANSPARENT); + SetTextColor(hdc, RGB(0, 0, 0)); + return reinterpret_cast<INT_PTR>(GetSysColorBrush(COLOR_WINDOW)); + } + + return static_cast<INT_PTR>(FALSE); +} + +void SaveChanges(HWND dialog) { + Branch branch = UNKNOWN_BRANCH; + if (IsDlgButtonChecked(dialog, IDC_STABLE)) + branch = BETA_BRANCH; + else if (IsDlgButtonChecked(dialog, IDC_CUTTING_EDGE)) + branch = DEV_BRANCH; + + if (branch != UNKNOWN_BRANCH) { + RegKey google_update(kGoogleUpdateRoot, kGoogleUpdateKey, KEY_WRITE); + if (!google_update.WriteValue(kBranchKey, kBranchStrings[branch])) { + MessageBox(dialog, L"Unable to change value, please make sure you\n" + L"have permission to change registry keys under HKLM", + L"Unable to update branch info", MB_OK); + } else { + std::wstring save_msg = L"Your changes have been saved.\nYou are now " + L"on the " + + std::wstring(kBranchStringsReadable[branch]) + + L" branch."; + MessageBox(dialog, save_msg.c_str(), L"Changes were saved", MB_OK); + + SetMainLabel(dialog, branch); + } + } +} + +INT_PTR CALLBACK DialogWndProc(HWND dialog, + UINT message_id, + WPARAM wparam, + LPARAM lparam) { + UNREFERENCED_PARAMETER(lparam); + + switch (message_id) { + case WM_INITDIALOG: + OnInitDialog(dialog); + return static_cast<INT_PTR>(TRUE); + case WM_CTLCOLORSTATIC: + return OnCtlColorStatic(dialog, wparam, lparam); + case WM_COMMAND: + // If the user presses the OK button. + if (LOWORD(wparam) == IDOK) { + SaveChanges(dialog); + return static_cast<INT_PTR>(TRUE); + } + // If the user presses the Cancel button. + if (LOWORD(wparam) == IDCANCEL) { + ::EndDialog(dialog, LOWORD(wparam)); + return static_cast<INT_PTR>(TRUE); + } + break; + case WM_ERASEBKGND: + PAINTSTRUCT paint; + HDC hdc = BeginPaint(dialog, &paint); + if (!hdc) + return static_cast<INT_PTR>(FALSE); // We didn't handle it. + + // Fill the background with White. + HBRUSH brush = GetStockBrush(WHITE_BRUSH); + HGDIOBJ old_brush = SelectObject(hdc, brush); + RECT rc; + GetClientRect(dialog, &rc); + FillRect(hdc, &rc, brush); + + // Clean up. + SelectObject(hdc, old_brush); + EndPaint(dialog, &paint); + return static_cast<INT_PTR>(TRUE); + } + + return static_cast<INT_PTR>(FALSE); +} + +int APIENTRY _tWinMain(HINSTANCE instance, + HINSTANCE previous_instance, + LPTSTR cmd_line, + int cmd_show) { + UNREFERENCED_PARAMETER(previous_instance); + UNREFERENCED_PARAMETER(cmd_line); + UNREFERENCED_PARAMETER(cmd_show); + + dlg_icon = ::LoadIcon(instance, MAKEINTRESOURCE(IDI_BRANCH_SWITCHER)); + + ::DialogBox(instance, + MAKEINTRESOURCE(IDD_MAIN_DIALOG), + GetDesktopWindow(), + DialogWndProc); + + return TRUE; +} diff --git a/tools/channel_changer/channel_changer.ico b/tools/channel_changer/channel_changer.ico Binary files differnew file mode 100644 index 0000000..401329f --- /dev/null +++ b/tools/channel_changer/channel_changer.ico diff --git a/tools/channel_changer/channel_changer.rc b/tools/channel_changer/channel_changer.rc new file mode 100644 index 0000000..ba89a09 --- /dev/null +++ b/tools/channel_changer/channel_changer.rc @@ -0,0 +1,124 @@ +// Microsoft Visual C++ generated resource script. +// +#include "resource.h" + +#define APSTUDIO_READONLY_SYMBOLS +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 2 resource. +// +#define APSTUDIO_HIDDEN_SYMBOLS +#include "windows.h" +#undef APSTUDIO_HIDDEN_SYMBOLS + +///////////////////////////////////////////////////////////////////////////// +#undef APSTUDIO_READONLY_SYMBOLS + +///////////////////////////////////////////////////////////////////////////// +// English (U.S.) resources + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +#ifdef _WIN32 +LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US +#pragma code_page(1252) +#endif //_WIN32 + +///////////////////////////////////////////////////////////////////////////// +// +// Icon +// + +// Icon with lowest ID value placed first to ensure application icon +// remains consistent on all systems. +IDI_BRANCH_SWITCHER ICON "channel_changer.ico" + +///////////////////////////////////////////////////////////////////////////// +// +// Dialog +// + +IDD_MAIN_DIALOG DIALOGEX 0, 0, 419, 147 +STYLE DS_SETFONT | DS_MODALFRAME | DS_FIXEDSYS | DS_CENTER | WS_POPUP | WS_CAPTION | WS_SYSMENU +CAPTION "Chrome Channel Switcher v1.1" +FONT 8, "MS Shell Dlg", 400, 0, 0x1 +BEGIN + DEFPUSHBUTTON "Update",IDOK,156,127,50,14 + PUSHBUTTON "Close",IDCANCEL,212,127,50,14 + LTEXT "This label will be populated programmatically.",IDC_LABEL_MAIN,47,13,322,18 + CONTROL "Beta: More stable releases, less frequently. You won't get bug fixes and new features as quickly, but what you get will have had more testing.",IDC_STABLE, + "Button",BS_AUTORADIOBUTTON | BS_MULTILINE,47,39,340,19 + CONTROL "Dev: Get the latest features and bugfixes every week. You'll be helping us improve, iterate, and make Chrome great faster. We need Dev users, and we'll give your feedback and issues higher priority.",IDC_CUTTING_EDGE, + "Button",BS_AUTORADIOBUTTON | BS_MULTILINE,47,63,355,21 + CONTROL 129,IDC_IMAGEVIEW,"Static",SS_BITMAP,7,7,35,23 + LTEXT "Note: If you switch from Dev to Beta, you'll stay on the current Chrome version until the Beta channel gets a later update. Changing channels does not change the version of Chrome you are currently using.",IDC_SECONDARY_LABEL,47,97,340,21 +END + + +#ifdef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// TEXTINCLUDE +// + +1 TEXTINCLUDE +BEGIN + "resource.h\0" +END + +2 TEXTINCLUDE +BEGIN + "#define APSTUDIO_HIDDEN_SYMBOLS\r\n" + "#include ""windows.h""\r\n" + "#undef APSTUDIO_HIDDEN_SYMBOLS\r\n" + "\0" +END + +3 TEXTINCLUDE +BEGIN + "\r\n" + "\0" +END + +#endif // APSTUDIO_INVOKED + + +///////////////////////////////////////////////////////////////////////////// +// +// DESIGNINFO +// + +#ifdef APSTUDIO_INVOKED +GUIDELINES DESIGNINFO +BEGIN + IDD_MAIN_DIALOG, DIALOG + BEGIN + LEFTMARGIN, 7 + RIGHTMARGIN, 412 + TOPMARGIN, 7 + BOTTOMMARGIN, 140 + END +END +#endif // APSTUDIO_INVOKED + + +///////////////////////////////////////////////////////////////////////////// +// +// Bitmap +// + +IDB_PICTURE BITMAP "glasses.bmp" +#endif // English (U.S.) resources +///////////////////////////////////////////////////////////////////////////// + + + +#ifndef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 3 resource. +// + + +///////////////////////////////////////////////////////////////////////////// +#endif // not APSTUDIO_INVOKED + diff --git a/tools/channel_changer/channel_changer.sln b/tools/channel_changer/channel_changer.sln new file mode 100644 index 0000000..48eb35b --- /dev/null +++ b/tools/channel_changer/channel_changer.sln @@ -0,0 +1,20 @@ + +Microsoft Visual Studio Solution File, Format Version 9.00 +# Visual Studio 2005 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "channel_changer", "channel_changer.vcproj", "{93D9CC27-A861-4430-A08B-AEB2FC223D78}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {93D9CC27-A861-4430-A08B-AEB2FC223D78}.Debug|Win32.ActiveCfg = Debug|Win32 + {93D9CC27-A861-4430-A08B-AEB2FC223D78}.Debug|Win32.Build.0 = Debug|Win32 + {93D9CC27-A861-4430-A08B-AEB2FC223D78}.Release|Win32.ActiveCfg = Release|Win32 + {93D9CC27-A861-4430-A08B-AEB2FC223D78}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/tools/channel_changer/channel_changer.vcproj b/tools/channel_changer/channel_changer.vcproj new file mode 100644 index 0000000..743dc5b --- /dev/null +++ b/tools/channel_changer/channel_changer.vcproj @@ -0,0 +1,237 @@ +<?xml version="1.0" encoding="Windows-1252"?> +<VisualStudioProject + ProjectType="Visual C++" + Version="8.00" + Name="channel_changer" + ProjectGUID="{93D9CC27-A861-4430-A08B-AEB2FC223D78}" + RootNamespace="channel_changer" + Keyword="Win32Proj" + > + <Platforms> + <Platform + Name="Win32" + /> + </Platforms> + <ToolFiles> + </ToolFiles> + <Configurations> + <Configuration + Name="Debug|Win32" + OutputDirectory="$(SolutionDir)$(ConfigurationName)" + IntermediateDirectory="$(ConfigurationName)" + ConfigurationType="1" + UseOfATL="1" + CharacterSet="1" + > + <Tool + Name="VCPreBuildEventTool" + /> + <Tool + Name="VCCustomBuildTool" + /> + <Tool + Name="VCXMLDataGeneratorTool" + /> + <Tool + Name="VCWebServiceProxyGeneratorTool" + /> + <Tool + Name="VCMIDLTool" + /> + <Tool + Name="VCCLCompilerTool" + Optimization="0" + AdditionalIncludeDirectories="../.." + PreprocessorDefinitions="WIN32;_DEBUG;_WINDOWS" + MinimalRebuild="true" + BasicRuntimeChecks="3" + RuntimeLibrary="3" + UsePrecompiledHeader="0" + WarningLevel="3" + Detect64BitPortabilityProblems="true" + DebugInformationFormat="4" + /> + <Tool + Name="VCManagedResourceCompilerTool" + /> + <Tool + Name="VCResourceCompilerTool" + /> + <Tool + Name="VCPreLinkEventTool" + /> + <Tool + Name="VCLinkerTool" + LinkIncremental="2" + GenerateDebugInformation="true" + SubSystem="2" + TargetMachine="1" + /> + <Tool + Name="VCALinkTool" + /> + <Tool + Name="VCManifestTool" + /> + <Tool + Name="VCXDCMakeTool" + /> + <Tool + Name="VCBscMakeTool" + /> + <Tool + Name="VCFxCopTool" + /> + <Tool + Name="VCAppVerifierTool" + /> + <Tool + Name="VCWebDeploymentTool" + /> + <Tool + Name="VCPostBuildEventTool" + /> + </Configuration> + <Configuration + Name="Release|Win32" + OutputDirectory="$(SolutionDir)$(ConfigurationName)" + IntermediateDirectory="$(ConfigurationName)" + ConfigurationType="1" + UseOfATL="1" + CharacterSet="1" + WholeProgramOptimization="1" + > + <Tool + Name="VCPreBuildEventTool" + /> + <Tool + Name="VCCustomBuildTool" + /> + <Tool + Name="VCXMLDataGeneratorTool" + /> + <Tool + Name="VCWebServiceProxyGeneratorTool" + /> + <Tool + Name="VCMIDLTool" + /> + <Tool + Name="VCCLCompilerTool" + AdditionalIncludeDirectories="../.." + PreprocessorDefinitions="WIN32;NDEBUG;_WINDOWS" + RuntimeLibrary="2" + UsePrecompiledHeader="0" + WarningLevel="3" + Detect64BitPortabilityProblems="true" + DebugInformationFormat="3" + /> + <Tool + Name="VCManagedResourceCompilerTool" + /> + <Tool + Name="VCResourceCompilerTool" + /> + <Tool + Name="VCPreLinkEventTool" + /> + <Tool + Name="VCLinkerTool" + LinkIncremental="1" + GenerateDebugInformation="true" + SubSystem="2" + OptimizeReferences="2" + EnableCOMDATFolding="2" + TargetMachine="1" + /> + <Tool + Name="VCALinkTool" + /> + <Tool + Name="VCManifestTool" + /> + <Tool + Name="VCXDCMakeTool" + /> + <Tool + Name="VCBscMakeTool" + /> + <Tool + Name="VCFxCopTool" + /> + <Tool + Name="VCAppVerifierTool" + /> + <Tool + Name="VCWebDeploymentTool" + /> + <Tool + Name="VCPostBuildEventTool" + /> + </Configuration> + </Configurations> + <References> + </References> + <Files> + <Filter + Name="Source Files" + Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx" + UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" + > + <File + RelativePath=".\channel_changer.cc" + > + </File> + <File + RelativePath="..\..\base\registry.cc" + > + <FileConfiguration + Name="Debug|Win32" + > + <Tool + Name="VCCLCompilerTool" + UsePrecompiledHeader="0" + /> + </FileConfiguration> + </File> + </Filter> + <Filter + Name="Header Files" + Filter="h;hpp;hxx;hm;inl;inc;xsd" + UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}" + > + <File + RelativePath="..\..\base\registry.h" + > + </File> + <File + RelativePath=".\Resource.h" + > + </File> + </Filter> + <Filter + Name="Resource Files" + Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav" + UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}" + > + <File + RelativePath=".\channel_changer.ico" + > + </File> + <File + RelativePath=".\channel_changer.rc" + > + </File> + <File + RelativePath=".\glasses.bmp" + > + </File> + <File + RelativePath=".\switcher_icon.bmp" + > + </File> + </Filter> + </Files> + <Globals> + </Globals> +</VisualStudioProject> diff --git a/tools/channel_changer/glasses.bmp b/tools/channel_changer/glasses.bmp Binary files differnew file mode 100644 index 0000000..ee80550 --- /dev/null +++ b/tools/channel_changer/glasses.bmp diff --git a/tools/channel_changer/resource.h b/tools/channel_changer/resource.h new file mode 100644 index 0000000..8da97a8 --- /dev/null +++ b/tools/channel_changer/resource.h @@ -0,0 +1,24 @@ +//{{NO_DEPENDENCIES}} +// Microsoft Visual C++ generated include file. +// Used by channel_changer.rc +// +#define IDI_BRANCH_SWITCHER 101 +#define IDD_MAIN_DIALOG 128 +#define IDB_PICTURE 129 +#define IDC_IMAGEVIEW 130 +#define IDC_LABEL_MAIN 1000 +#define IDC_STABLE 1001 +#define IDC_CUTTING_EDGE 1002 +#define IDC_SECONDARY_LABEL 1003 + +// Next default values for new objects +// +#ifdef APSTUDIO_INVOKED +#ifndef APSTUDIO_READONLY_SYMBOLS +#define _APS_NO_MFC 1 +#define _APS_NEXT_RESOURCE_VALUE 131 +#define _APS_NEXT_COMMAND_VALUE 32771 +#define _APS_NEXT_CONTROL_VALUE 1004 +#define _APS_NEXT_SYMED_VALUE 102 +#endif +#endif diff --git a/tools/grit/README b/tools/grit/README new file mode 100644 index 0000000..8fcdafe --- /dev/null +++ b/tools/grit/README @@ -0,0 +1,2 @@ +GRIT (Google Resource and Internationalization Tool) is a tool for Windows +projects to manage resources and simplify the localization workflow. diff --git a/tools/grit/build/grit_resource_file.bat b/tools/grit/build/grit_resource_file.bat new file mode 100644 index 0000000..3be0231 --- /dev/null +++ b/tools/grit/build/grit_resource_file.bat @@ -0,0 +1,15 @@ +:: Batch file run as build command for .grd files +:: The custom build rule is set to expect (inputfile).h and (inputfile).rc +:: our grd files must generate files with the same basename. +@echo off + +setlocal + +set InFile=%~1 +set SolutionDir=%~2 +set InputDir=%~3 + +:: Use GNU tools +call %SolutionDir%\..\third_party\gnu\setup_env.bat + +%SolutionDir%\..\third_party\python_24\python.exe %SolutionDir%\..\tools\grit\grit.py -i %InFile% build -o %InputDir% diff --git a/tools/grit/build/grit_resource_file.rules b/tools/grit/build/grit_resource_file.rules new file mode 100644 index 0000000..5ab70f5 --- /dev/null +++ b/tools/grit/build/grit_resource_file.rules @@ -0,0 +1,103 @@ +<?xml version="1.0" encoding="utf-8"?> +<VisualStudioToolFile + Name="GRIT resource data file" + Version="8.00" + > + <Rules> + <CustomBuildRule + Name="GRIT Generated Resources" + DisplayName="GRIT Generated Resources" + CommandLine="$(SolutionDir)..\tools\grit\build\grit_resource_file.bat [inputs] "$(SolutionDir)" "$(IntDir)"" + Outputs="$(IntDir)\$(InputName).h; + $(IntDir)\$(InputName)_ar.rc; + $(IntDir)\$(InputName)_bg.rc; + $(IntDir)\$(InputName)_ca.rc; + $(IntDir)\$(InputName)_cs.rc; + $(IntDir)\$(InputName)_da.rc; + $(IntDir)\$(InputName)_de.rc; + $(IntDir)\$(InputName)_el.rc; + $(IntDir)\$(InputName)_en-GB.rc; + $(IntDir)\$(InputName)_en-US.rc; + $(IntDir)\$(InputName)_es.rc; + $(IntDir)\$(InputName)_es-419.rc; + $(IntDir)\$(InputName)_et.rc; + $(IntDir)\$(InputName)_fi.rc; + $(IntDir)\$(InputName)_fil.rc; + $(IntDir)\$(InputName)_fr.rc; + $(IntDir)\$(InputName)_he.rc; + $(IntDir)\$(InputName)_hi.rc; + $(IntDir)\$(InputName)_hr.rc; + $(IntDir)\$(InputName)_hu.rc; + $(IntDir)\$(InputName)_id.rc; + $(IntDir)\$(InputName)_it.rc; + $(IntDir)\$(InputName)_ja.rc; + $(IntDir)\$(InputName)_ko.rc; + $(IntDir)\$(InputName)_lt.rc; + $(IntDir)\$(InputName)_lv.rc; + $(IntDir)\$(InputName)_nl.rc; + $(IntDir)\$(InputName)_nb.rc; + $(IntDir)\$(InputName)_pl.rc; + $(IntDir)\$(InputName)_pt-BR.rc; + $(IntDir)\$(InputName)_pt-PT.rc; + $(IntDir)\$(InputName)_ro.rc; + $(IntDir)\$(InputName)_ru.rc; + $(IntDir)\$(InputName)_sk.rc; + $(IntDir)\$(InputName)_sl.rc; + $(IntDir)\$(InputName)_sr.rc; + $(IntDir)\$(InputName)_sv.rc; + $(IntDir)\$(InputName)_th.rc; + $(IntDir)\$(InputName)_tr.rc; + $(IntDir)\$(InputName)_uk.rc; + $(IntDir)\$(InputName)_vi.rc; + $(IntDir)\$(InputName)_zh-CN.rc; + $(IntDir)\$(InputName)_zh-TW.rc;" + AdditionalDependencies="$(SolutionDir)..\tools\grit\build\grit_resource_file.bat;$(SolutionDir)..\tools\grit\grit.py; + resources\$(InputName)_ar.xtb; + resources\$(InputName)_bg.xtb; + resources\$(InputName)_ca.xtb; + resources\$(InputName)_cs.xtb; + resources\$(InputName)_da.xtb; + resources\$(InputName)_de.xtb; + resources\$(InputName)_el.xtb; + resources\$(InputName)_en-GB.xtb; + resources\$(InputName)_es.xtb; + resources\$(InputName)_es-419.xtb; + resources\$(InputName)_et.xtb; + resources\$(InputName)_fi.xtb; + resources\$(InputName)_fil.xtb; + resources\$(InputName)_fr.xtb; + resources\$(InputName)_he.xtb; + resources\$(InputName)_hi.xtb; + resources\$(InputName)_hr.xtb; + resources\$(InputName)_hu.xtb; + resources\$(InputName)_id.xtb; + resources\$(InputName)_it.xtb; + resources\$(InputName)_ja.xtb; + resources\$(InputName)_ko.xtb; + resources\$(InputName)_lt.xtb; + resources\$(InputName)_lv.xtb; + resources\$(InputName)_nl.xtb; + resources\$(InputName)_no.xtb; + resources\$(InputName)_pl.xtb; + resources\$(InputName)_pt-BR.xtb; + resources\$(InputName)_pt-PT.xtb; + resources\$(InputName)_ro.xtb; + resources\$(InputName)_ru.xtb; + resources\$(InputName)_sk.xtb; + resources\$(InputName)_sl.xtb; + resources\$(InputName)_sr.xtb; + resources\$(InputName)_sv.xtb; + resources\$(InputName)_th.xtb; + resources\$(InputName)_tr.xtb; + resources\$(InputName)_uk.xtb; + resources\$(InputName)_vi.xtb; + resources\$(InputName)_zh-CN.xtb; + resources\$(InputName)_zh-TW.xtb;" + FileExtensions="*.grd" + ExecutionDescription="Generating resources..." + > + <Properties> + </Properties> + </CustomBuildRule> + </Rules> +</VisualStudioToolFile> diff --git a/tools/grit/grit.py b/tools/grit/grit.py new file mode 100644 index 0000000..ed74c23 --- /dev/null +++ b/tools/grit/grit.py @@ -0,0 +1,40 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Bootstrapping for GRIT. +''' + +import sys + +import grit.grit_runner + + +if __name__ == '__main__': + grit.grit_runner.Main(sys.argv[1:]) diff --git a/tools/grit/grit/__init__.py b/tools/grit/grit/__init__.py new file mode 100644 index 0000000..3fb788f --- /dev/null +++ b/tools/grit/grit/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Package 'grit' +''' + +pass diff --git a/tools/grit/grit/clique.py b/tools/grit/grit/clique.py new file mode 100644 index 0000000..0b9d84b --- /dev/null +++ b/tools/grit/grit/clique.py @@ -0,0 +1,467 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Collections of messages and their translations, called cliques. Also +collections of cliques (uber-cliques). +''' + +import types + +from grit import constants +from grit import exception +from grit import pseudo +from grit import tclib + + +class UberClique(object): + '''A factory (NOT a singleton factory) for making cliques. It has several + methods for working with the cliques created using the factory. + ''' + + def __init__(self): + # A map from message ID to list of cliques whose source messages have + # that ID. This will contain all cliques created using this factory. + # Different messages can have the same ID because they have the + # same translateable portion and placeholder names, but occur in different + # places in the resource tree. + self.cliques_ = {} + + # A map of clique IDs to list of languages to indicate translations where we + # fell back to English. + self.fallback_translations_ = {} + + # A map of clique IDs to list of languages to indicate missing translations. + self.missing_translations_ = {} + + def _AddMissingTranslation(self, lang, clique, is_error): + tl = self.fallback_translations_ + if is_error: + tl = self.missing_translations_ + id = clique.GetId() + if id not in tl: + tl[id] = {} + if lang not in tl[id]: + tl[id][lang] = 1 + + def HasMissingTranslations(self): + return len(self.missing_translations_) > 0 + + def MissingTranslationsReport(self): + '''Returns a string suitable for printing to report missing + and fallback translations to the user. + ''' + def ReportTranslation(clique, langs): + text = clique.GetMessage().GetPresentableContent() + extract = text[0:40] + ellipsis = '' + if len(text) > 40: + ellipsis = '...' + langs_extract = langs[0:6] + describe_langs = ','.join(langs_extract) + if len(langs) > 6: + describe_langs += " and %d more" % (len(langs) - 6) + return " %s \"%s%s\" %s" % (clique.GetId(), extract, ellipsis, + describe_langs) + lines = [] + if len(self.fallback_translations_): + lines.append( + "WARNING: Fell back to English for the following translations:") + for (id, langs) in self.fallback_translations_.items(): + lines.append(ReportTranslation(self.cliques_[id][0], langs.keys())) + if len(self.missing_translations_): + lines.append("ERROR: The following translations are MISSING:") + for (id, langs) in self.missing_translations_.items(): + lines.append(ReportTranslation(self.cliques_[id][0], langs.keys())) + return '\n'.join(lines) + + def MakeClique(self, message, translateable=True): + '''Create a new clique initialized with a message. + + Args: + message: tclib.Message() + translateable: True | False + ''' + clique = MessageClique(self, message, translateable) + + # Enable others to find this clique by its message ID + if message.GetId() in self.cliques_: + presentable_text = clique.GetMessage().GetPresentableContent() + for c in self.cliques_[message.GetId()]: + assert c.GetMessage().GetPresentableContent() == presentable_text + self.cliques_[message.GetId()].append(clique) + else: + self.cliques_[message.GetId()] = [clique] + + return clique + + def FindCliqueAndAddTranslation(self, translation, language): + '''Adds the specified translation to the clique with the source message + it is a translation of. + + Args: + translation: tclib.Translation() + language: 'en' | 'fr' ... + + Return: + True if the source message was found, otherwise false. + ''' + if translation.GetId() in self.cliques_: + for clique in self.cliques_[translation.GetId()]: + clique.AddTranslation(translation, language) + return True + else: + return False + + def BestClique(self, id): + '''Returns the "best" clique from a list of cliques. All the cliques + must have the same ID. The "best" clique is chosen in the following + order of preference: + - The first clique that has a non-ID-based description + - If no such clique found, one of the cliques with an ID-based description + - Otherwise an arbitrary clique + ''' + clique_list = self.cliques_[id] + clique_to_ret = None + for clique in clique_list: + if not clique_to_ret: + clique_to_ret = clique + + description = clique.GetMessage().GetDescription() + if description and len(description) > 0: + clique_to_ret = clique + if not description.startswith('ID:'): + break # this is the preferred case so we exit right away + return clique_to_ret + + def BestCliquePerId(self): + '''Iterates over the list of all cliques and returns the best clique for + each ID. This will be the first clique with a source message that has a + non-empty description, or an arbitrary clique if none of them has a + description. + ''' + for id in self.cliques_: + yield self.BestClique(id) + + def BestCliqueByOriginalText(self, text, meaning): + '''Finds the "best" (as in BestClique()) clique that has original text + 'text' and meaning 'meaning'. Returns None if there is no such clique. + ''' + # If needed, this can be optimized by maintaining a map of + # fingerprints of original text+meaning to cliques. + for c in self.BestCliquePerId(): + msg = c.GetMessage() + if msg.GetRealContent() == text and msg.GetMeaning() == meaning: + return msg + return None + + def AllMessageIds(self): + '''Returns a list of all defined message IDs. + ''' + return self.cliques_.keys() + + def AllCliques(self): + '''Iterates over all cliques. Note that this can return multiple cliques + with the same ID. + ''' + for cliques in self.cliques_.values(): + for c in cliques: + yield c + + def GenerateXtbParserCallback(self, lang, debug=False): + '''Creates a callback function as required by grit.xtb_reader.Parse(). + This callback will create Translation objects for each message from + the XTB that exists in this uberclique, and add them as translations for + the relevant cliques. The callback will add translations to the language + specified by 'lang' + + Args: + lang: 'fr' + debug: True | False + ''' + def Callback(id, structure): + if id not in self.cliques_: + if debug: print "Ignoring translation #%s" % id + return + + if debug: print "Adding translation #%s" % id + + # We fetch placeholder information from the original message (the XTB file + # only contains placeholder names). + original_msg = self.BestClique(id).GetMessage() + + translation = tclib.Translation(id=id) + for is_ph,text in structure: + if not is_ph: + translation.AppendText(text) + else: + found_placeholder = False + for ph in original_msg.GetPlaceholders(): + if ph.GetPresentation() == text: + translation.AppendPlaceholder(tclib.Placeholder( + ph.GetPresentation(), ph.GetOriginal(), ph.GetExample())) + found_placeholder = True + break + if not found_placeholder: + raise exception.MismatchingPlaceholders( + 'Translation for message ID %s had <ph name="%s%/>, no match\n' + 'in original message' % (id, text)) + self.FindCliqueAndAddTranslation(translation, lang) + return Callback + + +class CustomType(object): + '''A base class you should implement if you wish to specify a custom type + for a message clique (i.e. custom validation and optional modification of + translations).''' + + def Validate(self, message): + '''Returns true if the message (a tclib.Message object) is valid, + otherwise false. + ''' + raise NotImplementedError() + + def ValidateAndModify(self, lang, translation): + '''Returns true if the translation (a tclib.Translation object) is valid, + otherwise false. The language is also passed in. This method may modify + the translation that is passed in, if it so wishes. + ''' + raise NotImplementedError() + + def ModifyTextPart(self, lang, text): + '''If you call ModifyEachTextPart, it will turn around and call this method + for each text part of the translation. You should return the modified + version of the text, or just the original text to not change anything. + ''' + raise NotImplementedError() + + def ModifyEachTextPart(self, lang, translation): + '''Call this to easily modify one or more of the textual parts of a + translation. It will call ModifyTextPart for each part of the + translation. + ''' + contents = translation.GetContent() + for ix in range(len(contents)): + if (isinstance(contents[ix], types.StringTypes)): + contents[ix] = self.ModifyTextPart(lang, contents[ix]) + + +class OneOffCustomType(CustomType): + '''A very simple custom type that performs the validation expressed by + the input expression on all languages including the source language. + The expression can access the variables 'lang', 'msg' and 'text()' where 'lang' + is the language of 'msg', 'msg' is the message or translation being + validated and 'text()' returns the real contents of 'msg' (for shorthand). + ''' + def __init__(self, expression): + self.expr = expression + def Validate(self, message): + return self.ValidateAndModify(MessageClique.source_language, message) + def ValidateAndModify(self, lang, msg): + def text(): + return msg.GetRealContent() + return eval(self.expr, {}, + {'lang' : lang, + 'text' : text, + 'msg' : msg, + }) + + +class MessageClique(object): + '''A message along with all of its translations. Also code to bring + translations together with their original message.''' + + # change this to the language code of Messages you add to cliques_. + # TODO(joi) Actually change this based on the <grit> node's source language + source_language = 'en' + + # A constant translation we use when asked for a translation into the + # special language constants.CONSTANT_LANGUAGE. + CONSTANT_TRANSLATION = tclib.Translation(text='TTTTTT') + + def __init__(self, uber_clique, message, translateable=True, custom_type=None): + '''Create a new clique initialized with just a message. + + Args: + uber_clique: Our uber-clique (collection of cliques) + message: tclib.Message() + translateable: True | False + custom_type: instance of clique.CustomType interface + ''' + # Our parent + self.uber_clique = uber_clique + # If not translateable, we only store the original message. + self.translateable = translateable + # A mapping of language identifiers to tclib.BaseMessage and its + # subclasses (i.e. tclib.Message and tclib.Translation). + self.clique = { MessageClique.source_language : message } + # A list of the "shortcut groups" this clique is + # part of. Within any given shortcut group, no shortcut key (e.g. &J) + # must appear more than once in each language for all cliques that + # belong to the group. + self.shortcut_groups = [] + # An instance of the CustomType interface, or None. If this is set, it will + # be used to validate the original message and translations thereof, and + # will also get a chance to modify translations of the message. + self.SetCustomType(custom_type) + + def GetMessage(self): + '''Retrieves the tclib.Message that is the source for this clique.''' + return self.clique[MessageClique.source_language] + + def GetId(self): + '''Retrieves the message ID of the messages in this clique.''' + return self.GetMessage().GetId() + + def IsTranslateable(self): + return self.translateable + + def AddToShortcutGroup(self, group): + self.shortcut_groups.append(group) + + def SetCustomType(self, custom_type): + '''Makes this clique use custom_type for validating messages and + translations, and optionally modifying translations. + ''' + self.custom_type = custom_type + if custom_type and not custom_type.Validate(self.GetMessage()): + raise exception.InvalidMessage(self.GetMessage().GetRealContent()) + + def MessageForLanguage(self, lang, pseudo_if_no_match=True, fallback_to_english=False): + '''Returns the message/translation for the specified language, providing + a pseudotranslation if there is no available translation and a pseudo- + translation is requested. + + The translation of any message whatsoever in the special language + 'x_constant' is the message "TTTTTT". + + Args: + lang: 'en' + pseudo_if_no_match: True + fallback_to_english: False + + Return: + tclib.BaseMessage + ''' + if not self.translateable: + return self.GetMessage() + + if lang == constants.CONSTANT_LANGUAGE: + return self.CONSTANT_TRANSLATION + + for msglang in self.clique.keys(): + if lang == msglang: + return self.clique[msglang] + + if fallback_to_english: + self.uber_clique._AddMissingTranslation(lang, self, is_error=False) + return self.GetMessage() + + # If we're not supposed to generate pseudotranslations, we add an error + # report to a list of errors, then fail at a higher level, so that we + # get a list of all messages that are missing translations. + if not pseudo_if_no_match: + self.uber_clique._AddMissingTranslation(lang, self, is_error=True) + + return pseudo.PseudoMessage(self.GetMessage()) + + def AllMessagesThatMatch(self, lang_re, include_pseudo = True): + '''Returns a map of all messages that match 'lang', including the pseudo + translation if requested. + + Args: + lang_re: re.compile('fr|en') + include_pseudo: True + + Return: + { 'en' : tclib.Message, + 'fr' : tclib.Translation, + pseudo.PSEUDO_LANG : tclib.Translation } + ''' + if not self.translateable: + return [self.GetMessage()] + + matches = {} + for msglang in self.clique: + if lang_re.match(msglang): + matches[msglang] = self.clique[msglang] + + if include_pseudo: + matches[pseudo.PSEUDO_LANG] = pseudo.PseudoMessage(self.GetMessage()) + + return matches + + def AddTranslation(self, translation, language): + '''Add a translation to this clique. The translation must have the same + ID as the message that is the source for this clique. + + If this clique is not translateable, the function just returns. + + Args: + translation: tclib.Translation() + language: 'en' + + Throws: + grit.exception.InvalidTranslation if the translation you're trying to add + doesn't have the same message ID as the source message of this clique. + ''' + if not self.translateable: + return + if translation.GetId() != self.GetId(): + raise exception.InvalidTranslation( + 'Msg ID %s, transl ID %s' % (self.GetId(), translation.GetId())) + + assert not language in self.clique + + # Because two messages can differ in the original content of their + # placeholders yet share the same ID (because they are otherwise the + # same), the translation we are getting may have different original + # content for placeholders than our message, yet it is still the right + # translation for our message (because it is for the same ID). We must + # therefore fetch the original content of placeholders from our original + # English message. + # + # See grit.clique_unittest.MessageCliqueUnittest.testSemiIdenticalCliques + # for a concrete explanation of why this is necessary. + + original = self.MessageForLanguage(self.source_language, False) + if len(original.GetPlaceholders()) != len(translation.GetPlaceholders()): + print ("ERROR: '%s' translation of message id %s does not match" % + (language, translation.GetId())) + assert False + + transl_msg = tclib.Translation(id=self.GetId(), + text=translation.GetPresentableContent(), + placeholders=original.GetPlaceholders()) + + if self.custom_type and not self.custom_type.ValidateAndModify(language, transl_msg): + print "WARNING: %s translation failed validation: %s" % ( + language, transl_msg.GetId()) + + self.clique[language] = transl_msg diff --git a/tools/grit/grit/clique_unittest.py b/tools/grit/grit/clique_unittest.py new file mode 100644 index 0000000..e36969c --- /dev/null +++ b/tools/grit/grit/clique_unittest.py @@ -0,0 +1,228 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.clique''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import re +import StringIO +import unittest + +from grit import clique +from grit import exception +from grit import pseudo +from grit import tclib +from grit import grd_reader +from grit import util + +class MessageCliqueUnittest(unittest.TestCase): + def testClique(self): + factory = clique.UberClique() + msg = tclib.Message(text='Hello USERNAME, how are you?', + placeholders=[ + tclib.Placeholder('USERNAME', '%s', 'Joi')]) + c = factory.MakeClique(msg) + + self.failUnless(c.GetMessage() == msg) + self.failUnless(c.GetId() == msg.GetId()) + + msg_fr = tclib.Translation(text='Bonjour USERNAME, comment ca va?', + id=msg.GetId(), placeholders=[ + tclib.Placeholder('USERNAME', '%s', 'Joi')]) + msg_de = tclib.Translation(text='Guten tag USERNAME, wie geht es dir?', + id=msg.GetId(), placeholders=[ + tclib.Placeholder('USERNAME', '%s', 'Joi')]) + + c.AddTranslation(msg_fr, 'fr') + factory.FindCliqueAndAddTranslation(msg_de, 'de') + + # sort() sorts lists in-place and does not return them + for lang in ('en', 'fr', 'de'): + self.failUnless(lang in c.clique) + + self.failUnless(c.MessageForLanguage('fr').GetRealContent() == + msg_fr.GetRealContent()) + + try: + c.MessageForLanguage('zh-CN', False) + self.fail('Should have gotten exception') + except: + pass + + self.failUnless(c.MessageForLanguage('zh-CN', True) != None) + + rex = re.compile('fr|de|bingo') + self.failUnless(len(c.AllMessagesThatMatch(rex, False)) == 2) + self.failUnless(c.AllMessagesThatMatch(rex, True)[pseudo.PSEUDO_LANG] != None) + + def testBestClique(self): + factory = clique.UberClique() + factory.MakeClique(tclib.Message(text='Alfur', description='alfaholl')) + factory.MakeClique(tclib.Message(text='Alfur', description='')) + factory.MakeClique(tclib.Message(text='Vaettur', description='')) + factory.MakeClique(tclib.Message(text='Vaettur', description='')) + factory.MakeClique(tclib.Message(text='Troll', description='')) + factory.MakeClique(tclib.Message(text='Gryla', description='ID: IDS_GRYLA')) + factory.MakeClique(tclib.Message(text='Gryla', description='vondakerling')) + factory.MakeClique(tclib.Message(text='Leppaludi', description='ID: IDS_LL')) + factory.MakeClique(tclib.Message(text='Leppaludi', description='')) + + count_best_cliques = 0 + for c in factory.BestCliquePerId(): + count_best_cliques += 1 + msg = c.GetMessage() + text = msg.GetRealContent() + description = msg.GetDescription() + if text == 'Alfur': + self.failUnless(description == 'alfaholl') + elif text == 'Gryla': + self.failUnless(description == 'vondakerling') + elif text == 'Leppaludi': + self.failUnless(description == 'ID: IDS_LL') + self.failUnless(count_best_cliques == 5) + + def testAllInUberClique(self): + resources = grd_reader.Parse(util.WrapInputStream( + StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> +<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <messages> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + </messages> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" encoding="utf-16" file="grit/test/data/klonk.rc" /> + <structure type="tr_html" name="ID_HTML" file="grit/test/data/simple.html" /> + </structures> + </release> +</grit>''')), util.PathFromRoot('.')) + resources.RunGatherers(True) + content_list = [] + for clique_list in resources.UberClique().cliques_.values(): + for clique in clique_list: + content_list.append(clique.GetMessage().GetRealContent()) + self.failUnless('Hello %s, how are you doing today?' in content_list) + self.failUnless('Jack "Black" Daniels' in content_list) + self.failUnless('Hello!' in content_list) + + def testCorrectExceptionIfWrongEncodingOnResourceFile(self): + '''This doesn't really belong in this unittest file, but what the heck.''' + resources = grd_reader.Parse(util.WrapInputStream( + StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> +<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" file="grit/test/data/klonk.rc" /> + </structures> + </release> +</grit>''')), util.PathFromRoot('.')) + self.assertRaises(exception.SectionNotFound, resources.RunGatherers, True) + + def testSemiIdenticalCliques(self): + messages = [ + tclib.Message(text='Hello USERNAME', + placeholders=[tclib.Placeholder('USERNAME', '$1', 'Joi')]), + tclib.Message(text='Hello USERNAME', + placeholders=[tclib.Placeholder('USERNAME', '%s', 'Joi')]), + ] + self.failUnless(messages[0].GetId() == messages[1].GetId()) + + # Both of the above would share a translation. + translation = tclib.Translation(id=messages[0].GetId(), + text='Bonjour USERNAME', + placeholders=[tclib.Placeholder( + 'USERNAME', '$1', 'Joi')]) + + factory = clique.UberClique() + cliques = [factory.MakeClique(msg) for msg in messages] + + for clq in cliques: + clq.AddTranslation(translation, 'fr') + + self.failUnless(cliques[0].MessageForLanguage('fr').GetRealContent() == + 'Bonjour $1') + self.failUnless(cliques[1].MessageForLanguage('fr').GetRealContent() == + 'Bonjour %s') + + def testMissingTranslations(self): + messages = [ tclib.Message(text='Hello'), tclib.Message(text='Goodbye') ] + factory = clique.UberClique() + cliques = [factory.MakeClique(msg) for msg in messages] + + cliques[1].MessageForLanguage('fr', False, True) + + self.failUnless(not factory.HasMissingTranslations()) + + cliques[0].MessageForLanguage('de', False, False) + + self.failUnless(factory.HasMissingTranslations()) + + report = factory.MissingTranslationsReport() + self.failUnless(report.count('WARNING') == 1) + self.failUnless(report.count('8053599568341804890 "Goodbye" fr') == 1) + self.failUnless(report.count('ERROR') == 1) + self.failUnless(report.count('800120468867715734 "Hello" de') == 1) + + def testCustomTypes(self): + factory = clique.UberClique() + message = tclib.Message(text='Bingo bongo') + c = factory.MakeClique(message) + try: + c.SetCustomType(DummyCustomType()) + self.fail() + except: + pass # expected case - 'Bingo bongo' does not start with 'jjj' + + message = tclib.Message(text='jjjBingo bongo') + c = factory.MakeClique(message) + c.SetCustomType(util.NewClassInstance( + 'grit.clique_unittest.DummyCustomType', clique.CustomType)) + translation = tclib.Translation(id=message.GetId(), text='Bilingo bolongo') + c.AddTranslation(translation, 'fr') + self.failUnless(c.MessageForLanguage('fr').GetRealContent().startswith('jjj')) + + +class DummyCustomType(clique.CustomType): + def Validate(self, message): + return message.GetRealContent().startswith('jjj') + def ValidateAndModify(self, lang, translation): + is_ok = self.Validate(translation) + self.ModifyEachTextPart(lang, translation) + def ModifyTextPart(self, lang, text): + return 'jjj%s' % text + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/constants.py b/tools/grit/grit/constants.py new file mode 100644 index 0000000..5f59883 --- /dev/null +++ b/tools/grit/grit/constants.py @@ -0,0 +1,45 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Constant definitions for GRIT. +''' + + +# This is the Icelandic noun meaning "grit" and is used to check that our +# input files are in the correct encoding. The middle character gets encoded +# as two bytes in UTF-8, so this is sufficient to detect incorrect encoding. +ENCODING_CHECK = u'm\u00f6l' + +# A special language, translations into which are always "TTTTTT". +CONSTANT_LANGUAGE = 'x_constant' + +# The Unicode byte-order-marker character (this is the Unicode code point, +# not the encoding of that character into any particular Unicode encoding). +BOM = u"\ufeff" diff --git a/tools/grit/grit/exception.py b/tools/grit/grit/exception.py new file mode 100644 index 0000000..d5a9c16 --- /dev/null +++ b/tools/grit/grit/exception.py @@ -0,0 +1,177 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Exception types for GRIT. +''' + +class Base(Exception): + '''A base exception that uses the class's docstring in addition to any + user-provided message as the body of the Base. + ''' + def __init__(self, msg=''): + if len(msg): + if self.__doc__: + msg = self.__doc__ + ': ' + msg + else: + msg = self.__doc__ + Exception.__init__(self, msg) + + +class Parsing(Base): + '''An error occurred parsing a GRD or XTB file.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class UnknownElement(Parsing): + '''An unknown node type was encountered.''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class MissingElement(Parsing): + '''An expected element was missing.''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class UnexpectedChild(Parsing): + '''An unexpected child element was encountered (on a leaf node).''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class UnexpectedAttribute(Parsing): + '''The attribute was not expected''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class UnexpectedContent(Parsing): + '''This element should not have content''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class MissingMandatoryAttribute(Parsing): + '''This element is missing a mandatory attribute''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class MutuallyExclusiveMandatoryAttribute(Parsing): + '''This element has 2 mutually exclusive mandatory attributes''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class DuplicateKey(Parsing): + '''A duplicate key attribute was found.''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class TooManyExamples(Parsing): + '''Only one <ex> element is allowed for each <ph> element.''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class GotPathExpectedFilenameOnly(Parsing): + '''The 'filename' attribute of an <output> node must not be a path, only + a filename. + ''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class InvalidMessage(Base): + '''The specified message failed validation.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class InvalidTranslation(Base): + '''Attempt to add an invalid translation to a clique.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class NoSuchTranslation(Base): + '''Requested translation not available''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class NotReady(Base): + '''Attempt to use an object before it is ready, or attempt to translate + an empty document.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class TooManyPlaceholders(Base): + '''Too many placeholders for elements of the same type.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class MismatchingPlaceholders(Base): + '''Placeholders do not match.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class InvalidPlaceholderName(Base): + '''Placeholder name can only contain A-Z, a-z, 0-9 and underscore.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class BlockTagInTranslateableChunk(Base): + '''A block tag was encountered where it wasn't expected.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class SectionNotFound(Base): + '''The section you requested was not found in the RC file. Make +sure the section ID is correct (matches the section's ID in the RC file). +Also note that you may need to specify the RC file's encoding (using the +encoding="" attribute) if it is not in the default Windows-1252 encoding. + ''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class IdRangeOverlap(Base): + '''ID range overlap.''' + def __init__(self, msg=''): + Base.__init__(self, msg) diff --git a/tools/grit/grit/extern/FP.py b/tools/grit/grit/extern/FP.py new file mode 100644 index 0000000..d6704b6 --- /dev/null +++ b/tools/grit/grit/extern/FP.py @@ -0,0 +1,54 @@ +#!/usr/bin/python2.2 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import md5 + +"""64-bit fingerprint support for strings. + +Usage: + from extern import FP + print 'Fingerprint is %ld' % FP.FingerPrint('Hello world!') +""" + + +def UnsignedFingerPrint(str, encoding='utf-8'): + """Generate a 64-bit fingerprint by taking the first half of the md5 + of the string.""" + hex128 = md5.new(str).hexdigest() + int64 = long(hex128[:16], 16) + return int64 + +def FingerPrint(str, encoding='utf-8'): + fp = UnsignedFingerPrint(str, encoding=encoding) + # interpret fingerprint as signed longs + if fp & 0x8000000000000000L: + fp = - ((~fp & 0xFFFFFFFFFFFFFFFFL) + 1) + return fp +
\ No newline at end of file diff --git a/tools/grit/grit/extern/__init__.py b/tools/grit/grit/extern/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/grit/grit/extern/__init__.py diff --git a/tools/grit/grit/extern/tclib.py b/tools/grit/grit/extern/tclib.py new file mode 100644 index 0000000..901c792 --- /dev/null +++ b/tools/grit/grit/extern/tclib.py @@ -0,0 +1,527 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The tclib module contains tools for aggregating, verifying, and storing +# messages destined for the Translation Console, as well as for reading +# translations back and outputting them in some desired format. +# +# This has been stripped down to include only the functionality needed by grit +# for creating Windows .rc and .h files. These are the only parts needed by +# the Chrome build process. + +import exceptions + +from grit.extern import FP + +# This module assumes that within a bundle no two messages can have the +# same id unless they're identical. + +# The basic classes defined here for external use are Message and Translation, +# where the former is used for English messages and the latter for +# translations. These classes have a lot of common functionality, as expressed +# by the common parent class BaseMessage. Perhaps the most important +# distinction is that translated text is stored in UTF-8, whereas original text +# is stored in whatever encoding the client uses (presumably Latin-1). + +# -------------------- +# The public interface +# -------------------- + +# Generate message id from message text and meaning string (optional), +# both in utf-8 encoding +# +def GenerateMessageId(message, meaning=''): + fp = FP.FingerPrint(message) + if meaning: + # combine the fingerprints of message and meaning + fp2 = FP.FingerPrint(meaning) + if fp < 0: + fp = fp2 + (fp << 1) + 1 + else: + fp = fp2 + (fp << 1) + # To avoid negative ids we strip the high-order bit + return str(fp & 0x7fffffffffffffffL) + +# ------------------------------------------------------------------------- +# The MessageTranslationError class is used to signal tclib-specific errors. + +class MessageTranslationError(exceptions.Exception): + def __init__(self, args = ''): + self.args = args + + +# ----------------------------------------------------------- +# The Placeholder class represents a placeholder in a message. + +class Placeholder(object): + # String representation + def __str__(self): + return '%s, "%s", "%s"' % \ + (self.__presentation, self.__original, self.__example) + + # Getters + def GetOriginal(self): + return self.__original + + def GetPresentation(self): + return self.__presentation + + def GetExample(self): + return self.__example + + def __eq__(self, other): + return self.EqualTo(other, strict=1, ignore_trailing_spaces=0) + + # Equality test + # + # ignore_trailing_spaces: TC is using varchar to store the + # phrwr fields, as a result of that, the trailing spaces + # are removed by MySQL when the strings are stored into TC:-( + # ignore_trailing_spaces parameter is used to ignore + # trailing spaces during equivalence comparison. + # + def EqualTo(self, other, strict = 1, ignore_trailing_spaces = 1): + if type(other) is not Placeholder: + return 0 + if StringEquals(self.__presentation, other.__presentation, + ignore_trailing_spaces): + if not strict or (StringEquals(self.__original, other.__original, + ignore_trailing_spaces) and + StringEquals(self.__example, other.__example, + ignore_trailing_spaces)): + return 1 + return 0 + + +# ----------------------------------------------------------------- +# BaseMessage is the common parent class of Message and Translation. +# It is not meant for direct use. + +class BaseMessage(object): + # Three types of message construction is supported. If the message text is a + # simple string with no dynamic content, you can pass it to the constructor + # as the "text" parameter. Otherwise, you can omit "text" and assemble the + # message step by step using AppendText() and AppendPlaceholder(). Or, as an + # alternative, you can give the constructor the "presentable" version of the + # message and a list of placeholders; it will then parse the presentation and + # build the message accordingly. For example: + # Message(text = "There are NUM_BUGS bugs in your code", + # placeholders = [Placeholder("NUM_BUGS", "%d", "33")], + # description = "Bla bla bla") + def __eq__(self, other): + # "source encoding" is nonsense, so ignore it + return _ObjectEquals(self, other, ['_BaseMessage__source_encoding']) + + def GetName(self): + return self.__name + + def GetSourceEncoding(self): + return self.__source_encoding + + # Append a placeholder to the message + def AppendPlaceholder(self, placeholder): + if not isinstance(placeholder, Placeholder): + raise MessageTranslationError, ("Invalid message placeholder %s in " + "message %s" % (placeholder, self.GetId())) + # Are there other placeholders with the same presentation? + # If so, they need to be the same. + for other in self.GetPlaceholders(): + if placeholder.GetPresentation() == other.GetPresentation(): + if not placeholder.EqualTo(other): + raise MessageTranslationError, \ + "Conflicting declarations of %s within message" % \ + placeholder.GetPresentation() + # update placeholder list + dup = 0 + for item in self.__content: + if isinstance(item, Placeholder) and placeholder.EqualTo(item): + dup = 1 + break + if not dup: + self.__placeholders.append(placeholder) + + # update content + self.__content.append(placeholder) + + # Strips leading and trailing whitespace, and returns a tuple + # containing the leading and trailing space that was removed. + def Strip(self): + leading = trailing = '' + if len(self.__content) > 0: + s0 = self.__content[0] + if not isinstance(s0, Placeholder): + s = s0.lstrip() + leading = s0[:-len(s)] + self.__content[0] = s + + s0 = self.__content[-1] + if not isinstance(s0, Placeholder): + s = s0.rstrip() + trailing = s0[len(s):] + self.__content[-1] = s + return leading, trailing + + # Return the id of this message + def GetId(self): + if self.__id is None: + return self.GenerateId() + return self.__id + + # Set the id of this message + def SetId(self, id): + if id is None: + self.__id = None + else: + self.__id = str(id) # Treat numerical ids as strings + + # Return content of this message as a list (internal use only) + def GetContent(self): + return self.__content + + # Return a human-readable version of this message + def GetPresentableContent(self): + presentable_content = "" + for item in self.__content: + if isinstance(item, Placeholder): + presentable_content += item.GetPresentation() + else: + presentable_content += item + + return presentable_content + + # Return a fragment of a message in escaped format + def EscapeFragment(self, fragment): + return fragment.replace('%', '%%') + + # Return the "original" version of this message, doing %-escaping + # properly. If source_msg is specified, the placeholder original + # information inside source_msg will be used instead. + def GetOriginalContent(self, source_msg = None): + original_content = "" + for item in self.__content: + if isinstance(item, Placeholder): + if source_msg: + ph = source_msg.GetPlaceholder(item.GetPresentation()) + if not ph: + raise MessageTranslationError, \ + "Placeholder %s doesn't exist in message: %s" % \ + (item.GetPresentation(), source_msg); + original_content += ph.GetOriginal() + else: + original_content += item.GetOriginal() + else: + original_content += self.EscapeFragment(item) + return original_content + + # Return the example of this message + def GetExampleContent(self): + example_content = "" + for item in self.__content: + if isinstance(item, Placeholder): + example_content += item.GetExample() + else: + example_content += item + return example_content + + # Return a list of all unique placeholders in this message + def GetPlaceholders(self): + return self.__placeholders + + # Return a placeholder in this message + def GetPlaceholder(self, presentation): + for item in self.__content: + if (isinstance(item, Placeholder) and + item.GetPresentation() == presentation): + return item + return None + + # Return this message's description + def GetDescription(self): + return self.__description + + # Add a message source + def AddSource(self, source): + self.__sources.append(source) + + # Return this message's sources as a list + def GetSources(self): + return self.__sources + + # Return this message's sources as a string + def GetSourcesAsText(self, delimiter = "; "): + return delimiter.join(self.__sources) + + # Set the obsolete flag for a message (internal use only) + def SetObsolete(self): + self.__obsolete = 1 + + # Get the obsolete flag for a message (internal use only) + def IsObsolete(self): + return self.__obsolete + + # Get the sequence number (0 by default) + def GetSequenceNumber(self): + return self.__sequence_number + + # Set the sequence number + def SetSequenceNumber(self, number): + self.__sequence_number = number + + # Increment instance counter + def AddInstance(self): + self.__num_instances += 1 + + # Return instance count + def GetNumInstances(self): + return self.__num_instances + + def GetErrors(self, from_tc=0): + """ + Returns a description of the problem if the message is not + syntactically valid, or None if everything is fine. + + Args: + from_tc: indicates whether this message came from the TC. We let + the TC get away with some things we normally wouldn't allow for + historical reasons. + """ + # check that placeholders are unambiguous + pos = 0 + phs = {} + for item in self.__content: + if isinstance(item, Placeholder): + phs[pos] = item + pos += len(item.GetPresentation()) + else: + pos += len(item) + presentation = self.GetPresentableContent() + for ph in self.GetPlaceholders(): + for pos in FindOverlapping(presentation, ph.GetPresentation()): + # message contains the same text as a placeholder presentation + other_ph = phs.get(pos) + if ((not other_ph + and not IsSubstringInPlaceholder(pos, len(ph.GetPresentation()), phs)) + or + (other_ph and len(other_ph.GetPresentation()) < len(ph.GetPresentation()))): + return "message contains placeholder name '%s':\n%s" % ( + ph.GetPresentation(), presentation) + return None + + + def __CopyTo(self, other): + """ + Returns a copy of this BaseMessage. + """ + assert isinstance(other, self.__class__) or isinstance(self, other.__class__) + other.__source_encoding = self.__source_encoding + other.__content = self.__content[:] + other.__description = self.__description + other.__id = self.__id + other.__num_instances = self.__num_instances + other.__obsolete = self.__obsolete + other.__name = self.__name + other.__placeholders = self.__placeholders[:] + other.__sequence_number = self.__sequence_number + other.__sources = self.__sources[:] + + return other + + def HasText(self): + """Returns true iff this message has anything other than placeholders.""" + for item in self.__content: + if not isinstance(item, Placeholder): + return True + return False + +# -------------------------------------------------------- +# The Message class represents original (English) messages + +class Message(BaseMessage): + # See BaseMessage constructor + def __init__(self, source_encoding, text=None, id=None, + description=None, meaning="", placeholders=None, + source=None, sequence_number=0, clone_from=None, + time_created=0, name=None, is_hidden = 0): + + if clone_from is not None: + BaseMessage.__init__(self, None, clone_from=clone_from) + self.__meaning = clone_from.__meaning + self.__time_created = clone_from.__time_created + self.__is_hidden = clone_from.__is_hidden + return + + BaseMessage.__init__(self, source_encoding, text, id, description, + placeholders, source, sequence_number, + name=name) + self.__meaning = meaning + self.__time_created = time_created + self.SetIsHidden(is_hidden) + + # String representation + def __str__(self): + s = 'source: %s, id: %s, content: "%s", meaning: "%s", ' \ + 'description: "%s"' % \ + (self.GetSourcesAsText(), self.GetId(), self.GetPresentableContent(), + self.__meaning, self.GetDescription()) + if self.GetName() is not None: + s += ', name: "%s"' % self.GetName() + placeholders = self.GetPlaceholders() + for i in range(len(placeholders)): + s += ", placeholder[%d]: %s" % (i, placeholders[i]) + return s + + # Strips leading and trailing whitespace, and returns a tuple + # containing the leading and trailing space that was removed. + def Strip(self): + leading = trailing = '' + content = self.GetContent() + if len(content) > 0: + s0 = content[0] + if not isinstance(s0, Placeholder): + s = s0.lstrip() + leading = s0[:-len(s)] + content[0] = s + + s0 = content[-1] + if not isinstance(s0, Placeholder): + s = s0.rstrip() + trailing = s0[len(s):] + content[-1] = s + return leading, trailing + + # Generate an id by hashing message content + def GenerateId(self): + self.SetId(GenerateMessageId(self.GetPresentableContent(), + self.__meaning)) + return self.GetId() + + def GetMeaning(self): + return self.__meaning + + def GetTimeCreated(self): + return self.__time_created + + # Equality operator + def EqualTo(self, other, strict = 1): + # Check id, meaning, content + if self.GetId() != other.GetId(): + return 0 + if self.__meaning != other.__meaning: + return 0 + if self.GetPresentableContent() != other.GetPresentableContent(): + return 0 + # Check descriptions if comparison is strict + if (strict and + self.GetDescription() is not None and + other.GetDescription() is not None and + self.GetDescription() != other.GetDescription()): + return 0 + # Check placeholders + ph1 = self.GetPlaceholders() + ph2 = other.GetPlaceholders() + if len(ph1) != len(ph2): + return 0 + for i in range(len(ph1)): + if not ph1[i].EqualTo(ph2[i], strict): + return 0 + + return 1 + + def Copy(self): + """ + Returns a copy of this Message. + """ + assert isinstance(self, Message) + return Message(None, clone_from=self) + + def SetIsHidden(self, is_hidden): + """Sets whether this message should be hidden. + + Args: + is_hidden : 0 or 1 - if the message should be hidden, 0 otherwise + """ + if is_hidden not in [0, 1]: + raise MessageTranslationError, "is_hidden must be 0 or 1, got %s" + self.__is_hidden = is_hidden + + def IsHidden(self): + """Returns 1 if this message is hidden, and 0 otherwise.""" + return self.__is_hidden + +# ---------------------------------------------------- +# The Translation class represents translated messages + +class Translation(BaseMessage): + # See BaseMessage constructor + def __init__(self, source_encoding, text=None, id=None, + description=None, placeholders=None, source=None, + sequence_number=0, clone_from=None, ignore_ph_errors=0, + name=None): + if clone_from is not None: + BaseMessage.__init__(self, None, clone_from=clone_from) + return + + BaseMessage.__init__(self, source_encoding, text, id, description, + placeholders, source, sequence_number, + ignore_ph_errors=ignore_ph_errors, name=name) + + # String representation + def __str__(self): + s = 'source: %s, id: %s, content: "%s", description: "%s"' % \ + (self.GetSourcesAsText(), self.GetId(), self.GetPresentableContent(), + self.GetDescription()); + placeholders = self.GetPlaceholders() + for i in range(len(placeholders)): + s += ", placeholder[%d]: %s" % (i, placeholders[i]) + return s + + # Equality operator + def EqualTo(self, other, strict=1): + # Check id and content + if self.GetId() != other.GetId(): + return 0 + if self.GetPresentableContent() != other.GetPresentableContent(): + return 0 + # Check placeholders + ph1 = self.GetPlaceholders() + ph2 = other.GetPlaceholders() + if len(ph1) != len(ph2): + return 0 + for i in range(len(ph1)): + if not ph1[i].EqualTo(ph2[i], strict): + return 0 + + return 1 + + def Copy(self): + """ + Returns a copy of this Translation. + """ + return Translation(None, clone_from=self) diff --git a/tools/grit/grit/format/__init__.py b/tools/grit/grit/format/__init__.py new file mode 100644 index 0000000..e811820 --- /dev/null +++ b/tools/grit/grit/format/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Module grit.format +''' + +pass diff --git a/tools/grit/grit/format/interface.py b/tools/grit/grit/format/interface.py new file mode 100644 index 0000000..33104dc --- /dev/null +++ b/tools/grit/grit/format/interface.py @@ -0,0 +1,56 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Base classes for item formatters and file formatters. +''' + + +import re + + +class ItemFormatter(object): + '''Base class for a formatter that knows how to format a single item.''' + + def Format(self, item, lang='', begin_item=True, output_dir='.'): + '''Returns a Unicode string representing 'item' in the format known by this + item formatter, for the language 'lang'. May be called once at the + start of the item (begin_item == True) and again at the end + (begin_item == False), or only at the start of the item (begin_item == True) + + Args: + item: anything + lang: 'en' + begin_item: True | False + output_dir: '.' + + Return: + u'hello' + ''' + raise NotImplementedError() diff --git a/tools/grit/grit/format/rc.py b/tools/grit/grit/format/rc.py new file mode 100644 index 0000000..219a9a0 --- /dev/null +++ b/tools/grit/grit/format/rc.py @@ -0,0 +1,456 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Support for formatting an RC file for compilation. +''' + +import os +import types +import re + +from grit import util +from grit.format import interface + + +# Matches all different types of linebreaks. +_LINEBREAKS = re.compile('\r\n|\n|\r') + +''' +This dictionary defines the langauge charset pair lookup table, which is used +for replacing the GRIT expand variables for language info in Product Version +resource. The key is the language ISO country code, and the value +is the language and character-set pair, which is a hexadecimal string +consisting of the concatenation of the language and character-set identifiers. +The first 4 digit of the value is the hex value of LCID, the remaining +4 digits is the hex value of character-set id(code page)of the language. + +We have defined three GRIT expand_variables to be used in the version resource +file to set the language info. Here is an example how they should be used in +the VS_VERSION_INFO section of the resource file to allow GRIT to localize +the language info correctly according to product locale. + +VS_VERSION_INFO VERSIONINFO +... +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "[GRITVERLANGCHARSETHEX]" + BEGIN + ... + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", [GRITVERLANGID], [GRITVERCHARSETID] + END +END + +''' + +_LANGUAGE_CHARSET_PAIR = { + 'ar' : '040104e8', + 'fi' : '040b04e4', + 'ko' : '041203b5', + 'es' : '040a04e4', + 'bg' : '040204e3', + 'fr' : '040c04e4', + 'lv' : '042604e9', + 'sv' : '041d04e4', + 'ca' : '040304e4', + 'de' : '040704e4', + 'lt' : '042704e9', + 'tl' : '0c0004b0', # no lcid for tl(Tagalog), use default custom locale + 'zh-CN' : '080403a8', + 'el' : '040804e5', + 'no' : '041404e4', + 'th' : '041e036a', + 'zh-TW' : '040403b6', + 'iw' : '040d04e7', + 'pl' : '041504e2', + 'tr' : '041f04e6', + 'hr' : '041a04e4', + 'hi' : '043904b0', # no codepage for hindi, use unicode(1200) + 'pt-BR' : '041604e4', + 'uk' : '042204e3', + 'cs' : '040504e2', + 'hu' : '040e04e2', + 'ro' : '041804e2', + 'ur' : '042004b0', # no codepage for urdu, use unicode(1200) + 'da' : '040604e4', + 'is' : '040f04e4', + 'ru' : '041904e3', + 'vi' : '042a04ea', + 'nl' : '041304e4', + 'id' : '042104e4', + 'sr' : '081a04e2', + 'en-GB' : '0809040e', + 'it' : '041004e4', + 'sk' : '041b04e2', + 'et' : '042504e9', + 'ja' : '041103a4', + 'sl' : '042404e2', + 'en' : '040904b0', +} + +_LANGUAGE_DIRECTIVE_PAIR = { + 'ar' : 'LANG_ARABIC, SUBLANG_DEFAULT', + 'fi' : 'LANG_FINNISH, SUBLANG_DEFAULT', + 'ko' : 'LANG_KOREAN, SUBLANG_KOREAN', + 'es' : 'LANG_SPANISH, SUBLANG_SPANISH_MODERN', + 'bg' : 'LANG_BULGARIAN, SUBLANG_DEFAULT', + 'fr' : 'LANG_FRENCH, SUBLANG_FRENCH', + 'lv' : 'LANG_LATVIAN, SUBLANG_DEFAULT', + 'sv' : 'LANG_SWEDISH, SUBLANG_SWEDISH', + 'ca' : 'LANG_CATALAN, SUBLANG_DEFAULT', + 'de' : 'LANG_GERMAN, SUBLANG_GERMAN', + 'lt' : 'LANG_LITHUANIAN, SUBLANG_LITHUANIAN', + 'tl' : 'LANG_NEUTRAL, SUBLANG_DEFAULT', + 'zh-CN' : 'LANG_CHINESE, SUBLANG_CHINESE_SIMPLIFIED', + 'el' : 'LANG_GREEK, SUBLANG_DEFAULT', + 'no' : 'LANG_NORWEGIAN, SUBLANG_DEFAULT', + 'th' : 'LANG_THAI, SUBLANG_DEFAULT', + 'zh-TW' : 'LANG_CHINESE, SUBLANG_CHINESE_TRADITIONAL', + 'iw' : 'LANG_HEBREW, SUBLANG_DEFAULT', + 'pl' : 'LANG_POLISH, SUBLANG_DEFAULT', + 'tr' : 'LANG_TURKISH, SUBLANG_DEFAULT', + 'hr' : 'LANG_CROATIAN, SUBLANG_DEFAULT', + 'hi' : 'LANG_HINDI, SUBLANG_DEFAULT', + 'pt-BR' : 'LANG_PORTUGUESE, SUBLANG_DEFAULT', + 'uk' : 'LANG_UKRAINIAN, SUBLANG_DEFAULT', + 'cs' : 'LANG_CZECH, SUBLANG_DEFAULT', + 'hu' : 'LANG_HUNGARIAN, SUBLANG_DEFAULT', + 'ro' : 'LANG_ROMANIAN, SUBLANG_DEFAULT', + 'ur' : 'LANG_URDU, SUBLANG_DEFAULT', + 'da' : 'LANG_DANISH, SUBLANG_DEFAULT', + 'is' : 'LANG_ICELANDIC, SUBLANG_DEFAULT', + 'ru' : 'LANG_RUSSIAN, SUBLANG_DEFAULT', + 'vi' : 'LANG_VIETNAMESE, SUBLANG_DEFAULT', + 'nl' : 'LANG_DUTCH, SUBLANG_DEFAULT', + 'id' : 'LANG_INDONESIAN, SUBLANG_DEFAULT', + 'sr' : 'LANG_SERBIAN, SUBLANG_SERBIAN_CYRILLIC', + 'en-GB' : 'LANG_ENGLISH, SUBLANG_ENGLISH_UK', + 'it' : 'LANG_ITALIAN, SUBLANG_DEFAULT', + 'sk' : 'LANG_SLOVAK, SUBLANG_DEFAULT', + 'et' : 'LANG_ESTONIAN, SUBLANG_DEFAULT', + 'ja' : 'LANG_JAPANESE, SUBLANG_DEFAULT', + 'sl' : 'LANG_SLOVENIAN, SUBLANG_DEFAULT', + 'en' : 'LANG_ENGLISH, SUBLANG_ENGLISH_US', +} + +def GetLangCharsetPair(language) : + if _LANGUAGE_CHARSET_PAIR.has_key(language) : + return _LANGUAGE_CHARSET_PAIR[language] + else : + print 'Warning:GetLangCharsetPair() found undefined language %s' %(language) + return '' + +def GetLangDirectivePair(language) : + if _LANGUAGE_DIRECTIVE_PAIR.has_key(language) : + return _LANGUAGE_DIRECTIVE_PAIR[language] + else : + print 'Warning:GetLangDirectivePair() found undefined language %s' %(language) + return 'unknown language: see tools/grit/format/rc.py' + +def GetLangIdHex(language) : + if _LANGUAGE_CHARSET_PAIR.has_key(language) : + langcharset = _LANGUAGE_CHARSET_PAIR[language] + lang_id = '0x' + langcharset[0:4] + return lang_id + else : + print 'Warning:GetLangIdHex() found undefined language %s' %(language) + return '' + + +def GetCharsetIdDecimal(language) : + if _LANGUAGE_CHARSET_PAIR.has_key(language) : + langcharset = _LANGUAGE_CHARSET_PAIR[language] + charset_decimal = int(langcharset[4:], 16) + return str(charset_decimal) + else : + print 'Warning:GetCharsetIdDecimal() found undefined language %s' %(language) + return '' + + +def GetUnifiedLangCode(language) : + r = re.compile('([a-z]{1,2})_([a-z]{1,2})') + if r.match(language) : + underscore = language.find('_') + return language[0:underscore] + '-' + language[underscore + 1:].upper() + else : + return language + + +def _MakeRelativePath(base_path, path_to_make_relative): + '''Returns a relative path such from the base_path to + the path_to_make_relative. + + In other words, os.join(base_path, + MakeRelativePath(base_path, path_to_make_relative)) + is the same location as path_to_make_relative. + + Args: + base_path: the root path + path_to_make_relative: an absolute path that is on the same drive + as base_path + ''' + + def _GetPathAfterPrefix(prefix_path, path_with_prefix): + '''Gets the subpath within in prefix_path for the path_with_prefix + with no beginning or trailing path separators. + + Args: + prefix_path: the base path + path_with_prefix: a path that starts with prefix_path + ''' + assert path_with_prefix.startswith(prefix_path) + path_without_prefix = path_with_prefix[len(prefix_path):] + normalized_path = os.path.normpath(path_without_prefix.strip(os.path.sep)) + if normalized_path == '.': + normalized_path = '' + return normalized_path + + def _GetCommonBaseDirectory(*args): + '''Returns the common prefix directory for the given paths + + Args: + The list of paths (at least one of which should be a directory) + ''' + prefix = os.path.commonprefix(args) + # prefix is a character-by-character prefix (i.e. it does not end + # on a directory bound, so this code fixes that) + + # if the prefix ends with the separator, then it is prefect. + if len(prefix) > 0 and prefix[-1] == os.path.sep: + return prefix + + # We need to loop through all paths or else we can get + # tripped up by "c:\a" and "c:\abc". The common prefix + # is "c:\a" which is a directory and looks good with + # respect to the first directory but it is clear that + # isn't a common directory when the second path is + # examined. + for path in args: + assert len(path) >= len(prefix) + # If the prefix the same length as the path, + # then the prefix must be a directory (since one + # of the arguements should be a directory). + if path == prefix: + continue + # if the character after the prefix in the path + # is the separator, then the prefix appears to be a + # valid a directory as well for the given path + if path[len(prefix)] == os.path.sep: + continue + # Otherwise, the prefix is not a directory, so it needs + # to be shortened to be one + index_sep = prefix.rfind(os.path.sep) + # The use "index_sep + 1" because it includes the final sep + # and it handles the case when the index_sep is -1 as well + prefix = prefix[:index_sep + 1] + # At this point we backed up to a directory bound which is + # common to all paths, so we can quit going through all of + # the paths. + break + return prefix + + prefix = _GetCommonBaseDirectory(base_path, path_to_make_relative) + # If the paths had no commonality at all, then return the absolute path + # because it is the best that can be done. If the path had to be relative + # then eventually this absolute path will be discovered (when a build breaks) + # and an appropriate fix can be made, but having this allows for the best + # backward compatibility with the absolute path behavior in the past. + if len(prefix) <= 0: + return path_to_make_relative + # Build a path from the base dir to the common prefix + remaining_base_path = _GetPathAfterPrefix(prefix, base_path) + + # The follow handles two case: "" and "foo\\bar" + path_pieces = remaining_base_path.split(os.path.sep) + base_depth_from_prefix = len([d for d in path_pieces if len(d)]) + base_to_prefix = (".." + os.path.sep) * base_depth_from_prefix + + # Put add in the path from the prefix to the path_to_make_relative + remaining_other_path = _GetPathAfterPrefix(prefix, path_to_make_relative) + return base_to_prefix + remaining_other_path + + +class TopLevel(interface.ItemFormatter): + '''Writes out the required preamble for RC files.''' + def Format(self, item, lang='en', begin_item=True, output_dir='.'): + assert isinstance(lang, types.StringTypes) + if not begin_item: + return '' + else: + # Find the location of the resource header file, so that we can include + # it. + resource_header = 'resource.h' # fall back to this + language_directive = '' + for child in item.GetRoot().children: + if child.name == 'outputs': + for output in child.children: + if output.attrs['type'] == 'rc_header': + resource_header = os.path.abspath(output.GetOutputFilename()) + resource_header = _MakeRelativePath(output_dir, resource_header) + if output.attrs['lang'] != lang: + continue + if output.attrs['language_section'] == '': + # If no language_section is requested, no directive is added + # (Used when the generated rc will be included from another rc + # file that will have the appropriate language directive) + language_directive = '' + elif output.attrs['language_section'] == 'neutral': + # If a neutral language section is requested (default), add a + # neutral language directive + language_directive = 'LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL' + elif output.attrs['language_section'] == 'lang': + language_directive = 'LANGUAGE %s' % GetLangDirectivePair(lang) + resource_header = resource_header.replace('\\', '\\\\') + return '''// Copyright (c) Google Inc. %d +// All rights reserved. +// This file is automatically generated by GRIT. Do not edit. + +#include "%s" +#include <winres.h> +#include <winresrc.h> + +%s + + +''' % (util.GetCurrentYear(), resource_header, language_directive) +# end Format() function + + + +class StringTable(interface.ItemFormatter): + '''Surrounds a collection of string messages with the required begin and + end blocks to declare a string table.''' + + def Format(self, item, lang='en', begin_item=True, output_dir='.'): + assert isinstance(lang, types.StringTypes) + if begin_item: + return 'STRINGTABLE\nBEGIN\n' + else: + return 'END\n\n' + + +class Message(interface.ItemFormatter): + '''Writes out a single message to a string table.''' + + def Format(self, item, lang='en', begin_item=True, output_dir='.'): + from grit.node import message + if not begin_item: + return '' + + assert isinstance(lang, types.StringTypes) + assert isinstance(item, message.MessageNode) + + message = item.ws_at_start + item.Translate(lang) + item.ws_at_end + # Escape quotation marks (RC format uses doubling-up + message = message.replace('"', '""') + # Replace linebreaks with a \n escape + message = _LINEBREAKS.sub(r'\\n', message) + + name_attr = item.GetTextualIds()[0] + + return ' %-15s "%s"\n' % (name_attr, message) + + +class RcSection(interface.ItemFormatter): + '''Writes out an .rc file section.''' + + def Format(self, item, lang='en', begin_item=True, output_dir='.'): + if not begin_item: + return '' + + assert isinstance(lang, types.StringTypes) + from grit.node import structure + assert isinstance(item, structure.StructureNode) + + if item.IsExcludedFromRc(): + return '' + else: + text = item.gatherer.Translate( + lang, skeleton_gatherer=item.GetSkeletonGatherer(), + pseudo_if_not_available=item.PseudoIsAllowed(), + fallback_to_english=item.ShouldFallbackToEnglish()) + '\n\n' + + # Replace the language expand_variables in version rc info. + unified_lang_code = GetUnifiedLangCode(lang) + text = text.replace('[GRITVERLANGCHARSETHEX]', + GetLangCharsetPair(unified_lang_code)) + text = text.replace('[GRITVERLANGID]', GetLangIdHex(unified_lang_code)) + text = text.replace('[GRITVERCHARSETID]', + GetCharsetIdDecimal(unified_lang_code)) + + return text + + +class RcInclude(interface.ItemFormatter): + '''Writes out an item that is included in an .rc file (e.g. an ICON)''' + + def __init__(self, type, filenameWithoutPath = 0, relative_path = 0): + '''Indicates to the instance what the type of the resource include is, + e.g. 'ICON' or 'HTML'. Case must be correct, i.e. if the type is all-caps + the parameter should be all-caps. + + Args: + type: 'ICON' + ''' + self.type_ = type + self.filenameWithoutPath = filenameWithoutPath + self.relative_path_ = relative_path + + def Format(self, item, lang='en', begin_item=True, output_dir='.'): + if not begin_item: + return '' + + assert isinstance(lang, types.StringTypes) + from grit.node import structure + from grit.node import include + assert isinstance(item, (structure.StructureNode, include.IncludeNode)) + assert (isinstance(item, include.IncludeNode) or + item.attrs['type'] in ['tr_html', 'admin_template', 'txt', 'muppet']) + + # By default, we use relative pathnames to included resources so that + # sharing the resulting .rc files is possible. + # + # The FileForLanguage() Function has the side effect of generating the file + # if needed (e.g. if it is an HTML file include). + filename = os.path.abspath(item.FileForLanguage(lang, output_dir)) + if self.filenameWithoutPath: + filename = os.path.basename(filename) + elif self.relative_path_: + filename = _MakeRelativePath(output_dir, filename) + filename = filename.replace('\\', '\\\\') # escape for the RC format + + if isinstance(item, structure.StructureNode) and item.IsExcludedFromRc(): + return '' + else: + return '%-18s %-18s "%s"\n' % (item.attrs['name'], self.type_, filename) diff --git a/tools/grit/grit/format/rc_header.py b/tools/grit/grit/format/rc_header.py new file mode 100644 index 0000000..63010b2 --- /dev/null +++ b/tools/grit/grit/format/rc_header.py @@ -0,0 +1,207 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Item formatters for RC headers. +''' + +import re +import time + +from grit.format import interface +from grit import exception +from grit import util + +from grit.extern import FP + + +class TopLevel(interface.ItemFormatter): + '''Writes the necessary preamble for a resource.h file.''' + + def Format(self, item, lang='', begin_item=True, output_dir='.'): + print 'TopLevel formatter called for item %s, begin %s' % (item, begin_item) + if not begin_item: + return '' + else: + header_string = '''// Copyright (c) Google Inc. %d +// All rights reserved. +// This file is automatically generated by GRIT. Do not edit. +// Built on %s + +#pragma once +''' % (util.GetCurrentYear(), time.asctime()) + # Check for emit nodes under the rc_header. If any emit node + # is present, we assume it means the GRD file wants to override + # the default header, with no includes. + for output_node in item.GetOutputFiles(): + if output_node.GetType() == 'rc_header': + for child in output_node.children: + if child.name == 'emit': + if child.attrs['emit_type'] == 'prepend': + return header_string + # else print out the default header with include + return header_string + ''' +#include <atlres.h> + +''' + + +class EmitAppender(interface.ItemFormatter): + '''Adds the content of the <emit> nodes to the RC header file.''' + + def Format(self, item, lang='', begin_item=True, output_dir='.'): + if not begin_item: + return '' + else: + return '%s\n' % (item.GetCdata()) + +class Item(interface.ItemFormatter): + '''Writes the #define line(s) for a single item in a resource.h file. If + your node has multiple IDs that need to be defined (as is the case e.g. for + dialog resources) it should define a function GetTextIds(self) that returns + a list of textual IDs (strings). Otherwise the formatter will use the + 'name' attribute of the node.''' + + # All IDs allocated so far, mapped to the textual ID they represent. + # Used to detect and resolve collisions. + ids_ = {} + + # All textual IDs allocated so far, mapped to the numerical ID they + # represent. Used when literal IDs are being defined in the 'identifiers' + # section of the GRD file to define other message IDs. + tids_ = {} + + def _VerifyId(self, id, tid, msg_if_error): + if id in self.ids_ and self.ids_[id] != tid: + raise exception.IdRangeOverlap(msg_if_error + + '\nUse the first_id attribute on grouping nodes (<structures>,\n' + '<includes>, <messages> and <ids>) to fix this problem.') + if id < 101: + print ('WARNING: Numeric resource IDs should be greater than 100 to avoid\n' + 'conflicts with system-defined resource IDs.') + + def Format(self, item, lang='', begin_item=True, output_dir='.'): + if not begin_item: + return '' + + # Resources that use the RES protocol don't need + # any numerical ids generated, so we skip them altogether. + # This is accomplished by setting the flag 'generateid' to false + # in the GRD file. + if 'generateid' in item.attrs: + if item.attrs['generateid'] == 'false': + return '' + + text_ids = item.GetTextualIds() + + # We consider the "parent" of the item to be the GroupingNode containing + # the item, as its immediate parent may be an <if> node. + item_parent = item.parent + import grit.node.empty + while item_parent and not isinstance(item_parent, + grit.node.empty.GroupingNode): + item_parent = item_parent.parent + + lines = [] + for tid in text_ids: + if util.SYSTEM_IDENTIFIERS.match(tid): + # Don't emit a new ID for predefined IDs + continue + + # Some identifier nodes can provide their own id, + # and we use that id in the generated header in that case. + if hasattr(item, 'GetId') and item.GetId(): + id = long(item.GetId()) + + elif ('offset' in item.attrs and item_parent and + 'first_id' in item_parent.attrs and item_parent.attrs['first_id'] != ''): + offset_text = item.attrs['offset'] + parent_text = item_parent.attrs['first_id'] + + try: + offset_id = long(offset_text) + except ValueError: + offset_id = self.tids_[offset_text] + + try: + parent_id = long(parent_text) + except ValueError: + parent_id = self.tids_[parent_text] + + id = parent_id + offset_id + + # We try to allocate IDs sequentially for blocks of items that might + # be related, for instance strings in a stringtable (as their IDs might be + # used e.g. as IDs for some radio buttons, in which case the IDs must + # be sequential). + # + # We do this by having the first item in a section store its computed ID + # (computed from a fingerprint) in its parent object. Subsequent children + # of the same parent will then try to get IDs that sequentially follow + # the currently stored ID (on the parent) and increment it. + elif not item_parent or not hasattr(item_parent, '_last_id_'): + # First check if the starting ID is explicitly specified by the parent. + if (item_parent and 'first_id' in item_parent.attrs and + item_parent.attrs['first_id'] != ''): + id = long(item_parent.attrs['first_id']) + self._VerifyId(id, tid, + 'Explicitly specified numeric first_id %d conflicts with one of the\n' + 'ID ranges already used.' % id) + else: + # Automatically generate the ID based on the first clique from the + # first child of the first child node of our parent (i.e. when we + # first get to this location in the code). + + # According to + # http://msdn.microsoft.com/en-us/library/t2zechd4(VS.71).aspx + # the safe usable range for resource IDs in Windows is from decimal + # 101 to 0x7FFF. + + id = FP.UnsignedFingerPrint(tid) + id = id % (0x7FFF - 101) + id += 101 + + self._VerifyId(id, tid, + 'Automatic (fingerprint-based) numeric ID for %s (%d) overlapped\n' + 'with a previously allocated range.' % (tid, id)) + + if item_parent: + item_parent._last_id_ = id + else: + assert hasattr(item_parent, '_last_id_') + id = item_parent._last_id_ = item_parent._last_id_ + 1 + self._VerifyId(id, tid, + 'Wanted to make numeric value for ID %s (%d) follow the numeric value of\n' + 'the previous ID in the .grd file, but it was already used.' % (tid, id)) + + if tid not in self.ids_.values(): + self.ids_[id] = tid + self.tids_[tid] = id + lines.append('#define %s %d\n' % (tid, id)) + return ''.join(lines) diff --git a/tools/grit/grit/format/rc_header_unittest.py b/tools/grit/grit/format/rc_header_unittest.py new file mode 100644 index 0000000..44d7c1b --- /dev/null +++ b/tools/grit/grit/format/rc_header_unittest.py @@ -0,0 +1,129 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for the rc_header formatter''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import StringIO +import unittest + +from grit.format import rc_header +from grit.node import message +from grit.node import structure +from grit.node import include +from grit.node import misc +from grit import grd_reader +from grit import exception + + +class RcHeaderFormatterUnittest(unittest.TestCase): + def setUp(self): + self.formatter = rc_header.Item() + self.formatter.ids_ = {} # need to reset this between tests + + def FormatAll(self, grd): + output = [] + for node in grd: + if isinstance(node, (message.MessageNode, structure.StructureNode, + include.IncludeNode, misc.IdentifierNode)): + output.append(self.formatter.Format(node)) + output = ''.join(output) + return output.replace(' ', '') + + def testFormatter(self): + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en" current_release="3" base_dir="."> + <release seq="3"> + <includes first_id="300" comment="bingo"> + <include type="gif" name="ID_LOGO" file="images/logo.gif" /> + </includes> + <messages first_id="10000"> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + <message name="IDS_BONGO"> + Bongo! + </message> + </messages> + <structures> + <structure type="dialog" name="IDD_NARROW_DIALOG" file="rc_files/dialogs.rc" /> + <structure type="version" name="VS_VERSION_INFO" file="rc_files/version.rc" /> + </structures> + </release> + </grit>'''), '.') + output = self.FormatAll(grd) + self.failUnless(output.count('IDS_GREETING10000')) + self.failUnless(output.count('ID_LOGO300')) + + def testExplicitFirstIdOverlaps(self): + # second first_id will overlap preexisting range + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en" current_release="3" base_dir="."> + <release seq="3"> + <includes first_id="300" comment="bingo"> + <include type="gif" name="ID_LOGO" file="images/logo.gif" /> + <include type="gif" name="ID_LOGO2" file="images/logo2.gif" /> + </includes> + <messages first_id="301"> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + <message name="IDS_SMURFGEBURF">Frubegfrums</message> + </messages> + </release> + </grit>'''), '.') + self.assertRaises(exception.IdRangeOverlap, self.FormatAll, grd) + + def testImplicitOverlapsPreexisting(self): + # second message in <messages> will overlap preexisting range + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en" current_release="3" base_dir="."> + <release seq="3"> + <includes first_id="301" comment="bingo"> + <include type="gif" name="ID_LOGO" file="images/logo.gif" /> + <include type="gif" name="ID_LOGO2" file="images/logo2.gif" /> + </includes> + <messages first_id="300"> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + <message name="IDS_SMURFGEBURF">Frubegfrums</message> + </messages> + </release> + </grit>'''), '.') + self.assertRaises(exception.IdRangeOverlap, self.FormatAll, grd) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/format/rc_unittest.py b/tools/grit/grit/format/rc_unittest.py new file mode 100644 index 0000000..769545d --- /dev/null +++ b/tools/grit/grit/format/rc_unittest.py @@ -0,0 +1,287 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.format.rc''' + +import os +import re +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import tempfile +import unittest +import StringIO + +from grit.format import rc +from grit import grd_reader +from grit import util +from grit.tool import build + +class DummyOutput(object): + def __init__(self, type, language): + self.type = type + self.language = language + def GetType(self): + return self.type + def GetLanguage(self): + return self.language + def GetOutputFilename(self): + return 'hello.gif' + +class FormatRcUnittest(unittest.TestCase): + def testMessages(self): + root = grd_reader.Parse(StringIO.StringIO(''' + <messages> + <message name="IDS_BTN_GO" desc="Button text" meaning="verb">Go!</message> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + <message name="BONGO" desc="Flippo nippo"> + Howdie "Mr. Elephant", how are you doing? \'\'\' + </message> + <message name="IDS_WITH_LINEBREAKS"> +Good day sir, +I am a bee +Sting sting + </message> + </messages> + '''), flexible_root = True) + util.FixRootForUnittest(root) + + buf = StringIO.StringIO() + build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf) + output = buf.getvalue() + self.failUnless(output.strip() == u''' +STRINGTABLE +BEGIN + IDS_BTN_GO "Go!" + IDS_GREETING "Hello %s, how are you doing today?" + BONGO "Howdie ""Mr. Elephant"", how are you doing? " + IDS_WITH_LINEBREAKS "Good day sir,\\nI am a bee\\nSting sting" +END'''.strip()) + + + def testRcSection(self): + root = grd_reader.Parse(StringIO.StringIO(''' + <structures> + <structure type="menu" name="IDC_KLONKMENU" file="grit\\test\data\klonk.rc" encoding="utf-16" /> + <structure type="dialog" name="IDD_ABOUTBOX" file="grit\\test\data\klonk.rc" encoding="utf-16" /> + <structure type="version" name="VS_VERSION_INFO" file="grit\\test\data\klonk.rc" encoding="utf-16" /> + </structures>'''), flexible_root = True) + util.FixRootForUnittest(root) + root.RunGatherers(recursive = True) + + buf = StringIO.StringIO() + build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf) + output = buf.getvalue() + self.failUnless(output.strip() == u''' +IDC_KLONKMENU MENU +BEGIN + POPUP "&File" + BEGIN + MENUITEM "E&xit", IDM_EXIT + MENUITEM "This be ""Klonk"" me like", ID_FILE_THISBE + POPUP "gonk" + BEGIN + MENUITEM "Klonk && is ""good""", ID_GONK_KLONKIS + END + END + POPUP "&Help" + BEGIN + MENUITEM "&About ...", IDM_ABOUT + END +END + +IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75 +STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU +CAPTION "About" +FONT 8, "System", 0, 0, 0x0 +BEGIN + ICON IDI_KLONK,IDC_MYICON,14,9,20,20 + LTEXT "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8, + SS_NOPREFIX + LTEXT "Copyright (C) 2005",IDC_STATIC,49,20,119,8 + DEFPUSHBUTTON "OK",IDOK,195,6,30,11,WS_GROUP + CONTROL "Jack ""Black"" Daniels",IDC_RADIO1,"Button", + BS_AUTORADIOBUTTON,46,51,84,10 +END + +VS_VERSION_INFO VERSIONINFO + FILEVERSION 1,0,0,1 + PRODUCTVERSION 1,0,0,1 + FILEFLAGSMASK 0x17L +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS 0x4L + FILETYPE 0x1L + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "FileDescription", "klonk Application" + VALUE "FileVersion", "1, 0, 0, 1" + VALUE "InternalName", "klonk" + VALUE "LegalCopyright", "Copyright (C) 2005" + VALUE "OriginalFilename", "klonk.exe" + VALUE "ProductName", " klonk Application" + VALUE "ProductVersion", "1, 0, 0, 1" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END'''.strip()) + + + def testRcIncludeStructure(self): + root = grd_reader.Parse(StringIO.StringIO(''' + <structures> + <structure type="tr_html" name="IDR_HTML" file="bingo.html"/> + <structure type="tr_html" name="IDR_HTML2" file="bingo2.html"/> + </structures>'''), flexible_root = True) + util.FixRootForUnittest(root, '/temp') + # We do not run gatherers as it is not needed and wouldn't find the file + + buf = StringIO.StringIO() + build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf) + output = buf.getvalue() + expected = (u'IDR_HTML HTML "%s"\n' + u'IDR_HTML2 HTML "%s"' + % (util.normpath('/temp/bingo.html').replace('\\', '\\\\'), + util.normpath('/temp/bingo2.html').replace('\\', '\\\\'))) + # hackety hack to work on win32&lin + output = re.sub('"[c-zC-Z]:', '"', output) + self.failUnless(output.strip() == expected) + + def testRcIncludeFile(self): + root = grd_reader.Parse(StringIO.StringIO(''' + <includes> + <include type="TXT" name="TEXT_ONE" file="bingo.txt"/> + <include type="TXT" name="TEXT_TWO" file="bingo2.txt" filenameonly="true" /> + </includes>'''), flexible_root = True) + util.FixRootForUnittest(root, '/temp') + + buf = StringIO.StringIO() + build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf) + output = buf.getvalue() + expected = (u'TEXT_ONE TXT "%s"\n' + u'TEXT_TWO TXT "%s"' + % (util.normpath('/temp/bingo.txt').replace('\\', '\\\\'), + 'bingo2.txt')) + # hackety hack to work on win32&lin + output = re.sub('"[c-zC-Z]:', '"', output) + self.failUnless(output.strip() == expected) + + + def testStructureNodeOutputfile(self): + input_file = util.PathFromRoot('grit/test/data/simple.html') + root = grd_reader.Parse(StringIO.StringIO( + '<structure type="tr_html" name="IDR_HTML" file="%s" />' %input_file), + flexible_root = True) + util.FixRootForUnittest(root, '.') + # We must run the gatherers since we'll be wanting the translation of the + # file. The file exists in the location pointed to. + root.RunGatherers(recursive=True) + + output_dir = tempfile.gettempdir() + en_file = root.FileForLanguage('en', output_dir) + self.failUnless(en_file == input_file) + fr_file = root.FileForLanguage('fr', output_dir) + self.failUnless(fr_file == os.path.join(output_dir, 'fr_simple.html')) + + fo = file(fr_file) + contents = fo.read() + fo.close() + + self.failUnless(contents.find('<p>') != -1) # should contain the markup + self.failUnless(contents.find('Hello!') == -1) # should be translated + + + def testFallbackToEnglish(self): + root = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="1" allow_pseudo="False"> + <structures fallback_to_english="True"> + <structure type="dialog" name="IDD_ABOUTBOX" file="grit\\test\data\klonk.rc" encoding="utf-16" /> + </structures> + </release> + </grit>'''), util.PathFromRoot('.')) + util.FixRootForUnittest(root) + root.RunGatherers(recursive = True) + + node = root.GetNodeById("IDD_ABOUTBOX") + formatter = node.ItemFormatter('rc_all') + output = formatter.Format(node, 'bingobongo') + self.failUnless(output.strip() == '''IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75 +STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU +CAPTION "About" +FONT 8, "System", 0, 0, 0x0 +BEGIN + ICON IDI_KLONK,IDC_MYICON,14,9,20,20 + LTEXT "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8, + SS_NOPREFIX + LTEXT "Copyright (C) 2005",IDC_STATIC,49,20,119,8 + DEFPUSHBUTTON "OK",IDOK,195,6,30,11,WS_GROUP + CONTROL "Jack ""Black"" Daniels",IDC_RADIO1,"Button", + BS_AUTORADIOBUTTON,46,51,84,10 +END''') + + + def testRelativePath(self): + ''' Verify that _MakeRelativePath works in some tricky cases.''' + def TestRelativePathCombinations(base_path, other_path, expected_result): + ''' Verify that the relative path function works for + the given paths regardless of whether or not they end with + a trailing slash.''' + for path1 in [base_path, base_path + os.path.sep]: + for path2 in [other_path, other_path + os.path.sep]: + result = rc._MakeRelativePath(path1, path2) + self.failUnless(result == expected_result) + + # set-up variables + root_dir = 'c:%sa' % os.path.sep + result1 = '..%sabc' % os.path.sep + path1 = root_dir + 'bc' + result2 = 'bc' + path2 = '%s%s%s' % (root_dir, os.path.sep, result2) + # run the tests + TestRelativePathCombinations(root_dir, path1, result1) + TestRelativePathCombinations(root_dir, path2, result2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/gather/__init__.py b/tools/grit/grit/gather/__init__.py new file mode 100644 index 0000000..d60af7c --- /dev/null +++ b/tools/grit/grit/gather/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Module grit.gather +''' + +pass
\ No newline at end of file diff --git a/tools/grit/grit/gather/admin_template.py b/tools/grit/grit/gather/admin_template.py new file mode 100644 index 0000000..b719de1 --- /dev/null +++ b/tools/grit/grit/gather/admin_template.py @@ -0,0 +1,103 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Gatherer for administrative template files. +''' + +import re +import types + +from grit.gather import regexp +from grit import exception +from grit import tclib +from grit import util + + +class MalformedAdminTemplateException(exception.Base): + '''This file doesn't look like a .adm file to me.''' + def __init__(self, msg=''): + exception.Base.__init__(self, msg) + + +class AdmGatherer(regexp.RegexpGatherer): + '''Gatherer for the translateable portions of an admin template. + + This gatherer currently makes the following assumptions: + - there is only one [strings] section and it is always the last section + of the file + - translateable strings do not need to be escaped. + ''' + + # Finds the strings section as the group named 'strings' + _STRINGS_SECTION = re.compile('(?P<first_part>.+^\[strings\])(?P<strings>.+)\Z', + re.MULTILINE | re.DOTALL) + + # Finds the translateable sections from within the [strings] section. + _TRANSLATEABLES = re.compile('^\s*[A-Za-z0-9_]+\s*=\s*"(?P<text>.+)"\s*$', + re.MULTILINE) + + def __init__(self, text): + regexp.RegexpGatherer.__init__(self, text) + + def Escape(self, text): + return text.replace('\n', '\\n') + + def UnEscape(self, text): + return text.replace('\\n', '\n') + + def Parse(self): + if self.have_parsed_: + return + m = self._STRINGS_SECTION.match(self.text_) + if not m: + raise MalformedAdminTemplateException() + # Add the first part, which is all nontranslateable, to the skeleton + self._AddNontranslateableChunk(m.group('first_part')) + # Then parse the rest using the _TRANSLATEABLES regexp. + self._RegExpParse(self._TRANSLATEABLES, m.group('strings')) + + # static method + def FromFile(adm_file, ext_key=None, encoding='cp1252'): + '''Loads the contents of 'adm_file' in encoding 'encoding' and creates + an AdmGatherer instance that gathers from those contents. + + The 'ext_key' parameter is ignored. + + Args: + adm_file: file('bingo.rc') | 'filename.rc' + encoding: 'utf-8' + + Return: + AdmGatherer(contents_of_file) + ''' + if isinstance(adm_file, types.StringTypes): + adm_file = util.WrapInputStream(file(adm_file, 'r'), encoding) + return AdmGatherer(adm_file.read()) + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/admin_template_unittest.py b/tools/grit/grit/gather/admin_template_unittest.py new file mode 100644 index 0000000..230b3b5e --- /dev/null +++ b/tools/grit/grit/gather/admin_template_unittest.py @@ -0,0 +1,141 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for the admin template gatherer.''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import StringIO +import tempfile +import unittest + +from grit.gather import admin_template +from grit import util +from grit import grd_reader +from grit import grit_runner +from grit.tool import build + + +class AdmGathererUnittest(unittest.TestCase): + def testParsingAndTranslating(self): + pseudofile = StringIO.StringIO( + 'bingo bongo\n' + 'ding dong\n' + '[strings] \n' + 'whatcha="bingo bongo"\n' + 'gotcha = "bingolabongola "the wise" fingulafongula" \n') + gatherer = admin_template.AdmGatherer.FromFile(pseudofile) + gatherer.Parse() + self.failUnless(len(gatherer.GetCliques()) == 2) + self.failUnless(gatherer.GetCliques()[1].GetMessage().GetRealContent() == + 'bingolabongola "the wise" fingulafongula') + + translation = gatherer.Translate('en') + self.failUnless(translation == gatherer.GetText().strip()) + + def testErrorHandling(self): + pseudofile = StringIO.StringIO( + 'bingo bongo\n' + 'ding dong\n' + 'whatcha="bingo bongo"\n' + 'gotcha = "bingolabongola "the wise" fingulafongula" \n') + gatherer = admin_template.AdmGatherer.FromFile(pseudofile) + self.assertRaises(admin_template.MalformedAdminTemplateException, + gatherer.Parse) + + _TRANSLATABLES_FROM_FILE = ( + 'Google', 'Google Desktop Search', 'Preferences', + 'Controls Google Deskop Search preferences', + 'Indexing and Capture Control', + 'Controls what files, web pages, and other content will be indexed by Google Desktop Search.', + 'Prevent indexing of e-mail', + # there are lots more but we don't check any further + ) + + def VerifyCliquesFromAdmFile(self, cliques): + self.failUnless(len(cliques) > 20) + for ix in range(len(self._TRANSLATABLES_FROM_FILE)): + text = cliques[ix].GetMessage().GetRealContent() + self.failUnless(text == self._TRANSLATABLES_FROM_FILE[ix]) + + def testFromFile(self): + fname = util.PathFromRoot('grit/test/data/GoogleDesktopSearch.adm') + gatherer = admin_template.AdmGatherer.FromFile(fname) + gatherer.Parse() + cliques = gatherer.GetCliques() + self.VerifyCliquesFromAdmFile(cliques) + + def MakeGrd(self): + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3"> + <release seq="3"> + <structures> + <structure type="admin_template" name="IDAT_GOOGLE_DESKTOP_SEARCH" + file="GoogleDesktopSearch.adm" exclude_from_rc="true" /> + <structure type="txt" name="BINGOBONGO" + file="README.txt" exclude_from_rc="true" /> + </structures> + </release> + <outputs> + <output filename="de_res.rc" type="rc_all" lang="de" /> + </outputs> + </grit>'''), util.PathFromRoot('grit/test/data')) + grd.RunGatherers(recursive=True) + return grd + + def testInGrd(self): + grd = self.MakeGrd() + cliques = grd.children[0].children[0].children[0].GetCliques() + self.VerifyCliquesFromAdmFile(cliques) + + def testFileIsOutput(self): + grd = self.MakeGrd() + dirname = tempfile.mkdtemp() + try: + tool = build.RcBuilder() + tool.o = grit_runner.Options() + tool.output_directory = dirname + tool.res = grd + tool.Process() + + self.failUnless(os.path.isfile( + os.path.join(dirname, 'de_GoogleDesktopSearch.adm'))) + self.failUnless(os.path.isfile( + os.path.join(dirname, 'de_README.txt'))) + finally: + for f in os.listdir(dirname): + os.unlink(os.path.join(dirname, f)) + os.rmdir(dirname) + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/gather/interface.py b/tools/grit/grit/gather/interface.py new file mode 100644 index 0000000..821b567 --- /dev/null +++ b/tools/grit/grit/gather/interface.py @@ -0,0 +1,132 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Interface for all gatherers. +''' + + +from grit import clique + + +class GathererBase(object): + '''Interface for all gatherer implementations. Subclasses must implement + all methods that raise NotImplemented.''' + + def __init__(self): + # A default uberclique that is local to this object. Users can override + # this with the uberclique they are using. + self.uberclique = clique.UberClique() + # Indicates whether this gatherer is a skeleton gatherer, in which case + # we should not do some types of processing on the translateable bits. + self.is_skeleton = False + + def SetUberClique(self, uberclique): + '''Overrides the default uberclique so that cliques created by this object + become part of the uberclique supplied by the user. + ''' + self.uberclique = uberclique + + def SetSkeleton(self, is_skeleton): + self.is_skeleton = is_skeleton + + def IsSkeleton(self): + return self.is_skeleton + + def Parse(self): + '''Parses the contents of what is being gathered.''' + raise NotImplementedError() + + def GetText(self): + '''Returns the text of what is being gathered.''' + raise NotImplementedError() + + def GetTextualIds(self): + '''Returns the mnemonic IDs that need to be defined for the resource + being gathered to compile correctly.''' + return [] + + def GetCliques(self): + '''Returns the MessageClique objects for all translateable portions.''' + return [] + + def Translate(self, lang, pseudo_if_not_available=True, + skeleton_gatherer=None, fallback_to_english=False): + '''Returns the resource being gathered, with translateable portions filled + with the translation for language 'lang'. + + If pseudo_if_not_available is true, a pseudotranslation will be used for any + message that doesn't have a real translation available. + + If no translation is available and pseudo_if_not_available is false, + fallback_to_english controls the behavior. If it is false, throw an error. + If it is true, use the English version of the message as its own + "translation". + + If skeleton_gatherer is specified, the translation will use the nontranslateable + parts from the gatherer 'skeleton_gatherer', which must be of the same type + as 'self'. + + If fallback_to_english + + Args: + lang: 'en' + pseudo_if_not_available: True | False + skeleton_gatherer: other_gatherer + fallback_to_english: True | False + + Return: + e.g. 'ID_THIS_SECTION TYPE\n...BEGIN\n "Translated message"\n......\nEND' + + Raises: + grit.exception.NotReady() if used before Parse() has been successfully + called. + grit.exception.NoSuchTranslation() if 'pseudo_if_not_available' and + fallback_to_english are both false and there is no translation for the + requested language. + ''' + raise NotImplementedError() + + def FromFile(rc_file, extkey=None, encoding = 'cp1252'): + '''Loads the resource from the file 'rc_file'. Optionally an external key + (which gets passed to the gatherer's constructor) can be specified. + + If 'rc_file' is a filename, it will be opened for reading using 'encoding'. + Otherwise the 'encoding' parameter is ignored. + + Args: + rc_file: file('') | 'filename.rc' + extkey: e.g. 'ID_MY_DIALOG' + encoding: 'utf-8' + + Return: + grit.gather.interface.GathererBase subclass + ''' + raise NotImplementedError() + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/muppet_strings.py b/tools/grit/grit/gather/muppet_strings.py new file mode 100644 index 0000000..7594762 --- /dev/null +++ b/tools/grit/grit/gather/muppet_strings.py @@ -0,0 +1,165 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Support for "strings.xml" format used by Muppet plug-ins in Google Desktop.''' + +import StringIO +import types +import re +import xml.sax +import xml.sax.handler +import xml.sax.saxutils + +from grit.gather import regexp +from grit import util +from grit import tclib + +# Placeholders can be defined in strings.xml files by putting the name of the +# placeholder between [![ and ]!] e.g. <MSG>Hello [![USER]!] how are you<MSG> +PLACEHOLDER_RE = re.compile('(\[!\[|\]!\])') + + +class MuppetStringsContentHandler(xml.sax.handler.ContentHandler): + '''A very dumb parser for splitting the strings.xml file into translateable + and nontranslateable chunks.''' + + def __init__(self, parent): + self.curr_elem = '' + self.curr_text = '' + self.parent = parent + self.description = '' + self.meaning = '' + self.translateable = True + + def startElement(self, name, attrs): + if (name != 'strings'): + self.curr_elem = name + + attr_names = attrs.getQNames() + if 'desc' in attr_names: + self.description = attrs.getValueByQName('desc') + if 'meaning' in attr_names: + self.meaning = attrs.getValueByQName('meaning') + if 'translateable' in attr_names: + value = attrs.getValueByQName('translateable') + if value.lower() not in ['true', 'yes']: + self.translateable = False + + att_text = [] + for attr_name in attr_names: + att_text.append(' ') + att_text.append(attr_name) + att_text.append('=') + att_text.append( + xml.sax.saxutils.quoteattr(attrs.getValueByQName(attr_name))) + + self.parent._AddNontranslateableChunk("<%s%s>" % + (name, ''.join(att_text))) + + def characters(self, content): + if self.curr_elem != '': + self.curr_text += content + + def endElement(self, name): + if name != 'strings': + self.parent.AddMessage(self.curr_text, self.description, + self.meaning, self.translateable) + self.parent._AddNontranslateableChunk("</%s>\n" % name) + self.curr_elem = '' + self.curr_text = '' + self.description = '' + self.meaning = '' + self.translateable = True + + def ignorableWhitespace(self, whitespace): + pass + +class MuppetStrings(regexp.RegexpGatherer): + '''Supports the strings.xml format used by Muppet gadgets.''' + + def __init__(self, text): + if util.IsExtraVerbose(): + print text + regexp.RegexpGatherer.__init__(self, text) + + def AddMessage(self, msgtext, description, meaning, translateable): + if msgtext == '': + return + + msg = tclib.Message(description=description, meaning=meaning) + + unescaped_text = self.UnEscape(msgtext) + parts = PLACEHOLDER_RE.split(unescaped_text) + in_placeholder = False + for part in parts: + if part == '': + continue + elif part == '[![': + in_placeholder = True + elif part == ']!]': + in_placeholder = False + else: + if in_placeholder: + msg.AppendPlaceholder(tclib.Placeholder(part, '[![%s]!]' % part, + '(placeholder)')) + else: + msg.AppendText(part) + + self.skeleton_.append( + self.uberclique.MakeClique(msg, translateable=translateable)) + + # if statement needed because this is supposed to be idempotent (so never + # set back to false) + if translateable: + self.translatable_chunk_ = True + + # Although we use the RegexpGatherer base class, we do not use the + # _RegExpParse method of that class to implement Parse(). Instead, we + # parse using a SAX parser. + def Parse(self): + if (self.have_parsed_): + return + self._AddNontranslateableChunk(u'<strings>\n') + stream = StringIO.StringIO(self.text_) + handler = MuppetStringsContentHandler(self) + xml.sax.parse(stream, handler) + self._AddNontranslateableChunk(u'</strings>\n') + + def Escape(self, text): + return util.EncodeCdata(text) + + def FromFile(filename_or_stream, extkey=None, encoding='cp1252'): + if isinstance(filename_or_stream, types.StringTypes): + if util.IsVerbose(): + print "MuppetStrings reading file %s, encoding %s" % ( + filename_or_stream, encoding) + filename_or_stream = util.WrapInputStream(file(filename_or_stream, 'r'), encoding) + return MuppetStrings(filename_or_stream.read()) + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/muppet_strings_unittest.py b/tools/grit/grit/gather/muppet_strings_unittest.py new file mode 100644 index 0000000..b43c1d1 --- /dev/null +++ b/tools/grit/grit/gather/muppet_strings_unittest.py @@ -0,0 +1,90 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.gather.muppet_strings''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest + +from grit.gather import muppet_strings + +class MuppetStringsUnittest(unittest.TestCase): + def testParsing(self): + original = '''<strings><BLA desc="Says hello">hello!</BLA><BINGO>YEEEESSS!!!</BINGO></strings>''' + gatherer = muppet_strings.MuppetStrings(original) + gatherer.Parse() + self.failUnless(len(gatherer.GetCliques()) == 2) + self.failUnless(gatherer.Translate('en').replace('\n', '') == original) + + def testEscapingAndLinebreaks(self): + original = ('''\ +<strings> +<LINEBREAK desc="Howdie">Hello +there +how +are +you?</LINEBREAK> <ESCAPED meaning="bingo">4 < 6</ESCAPED> +</strings>''') + gatherer = muppet_strings.MuppetStrings(original) + gatherer.Parse() + self.failUnless(gatherer.GetCliques()[0].translateable) + self.failUnless(len(gatherer.GetCliques()) == 2) + self.failUnless(gatherer.GetCliques()[0].GetMessage().GetRealContent() == + 'Hello\nthere\nhow\nare\nyou?') + self.failUnless(gatherer.GetCliques()[0].GetMessage().GetDescription() == 'Howdie') + self.failUnless(gatherer.GetCliques()[1].GetMessage().GetRealContent() == + '4 < 6') + self.failUnless(gatherer.GetCliques()[1].GetMessage().GetMeaning() == 'bingo') + + def testPlaceholders(self): + original = "<strings><MESSAGE translateable='True'>Hello [![USER]!] how are you? [![HOUR]!]:[![MINUTE]!]</MESSAGE></strings>" + gatherer = muppet_strings.MuppetStrings(original) + gatherer.Parse() + self.failUnless(gatherer.GetCliques()[0].translateable) + msg = gatherer.GetCliques()[0].GetMessage() + self.failUnless(len(msg.GetPlaceholders()) == 3) + ph = msg.GetPlaceholders()[0] + self.failUnless(ph.GetOriginal() == '[![USER]!]') + self.failUnless(ph.GetPresentation() == 'USER') + + def testTranslateable(self): + original = "<strings><BINGO translateable='false'>Yo yo hi there</BINGO></strings>" + gatherer = muppet_strings.MuppetStrings(original) + gatherer.Parse() + msg = gatherer.GetCliques()[0].GetMessage() + self.failUnless(msg.GetRealContent() == "Yo yo hi there") + self.failUnless(not gatherer.GetCliques()[0].translateable) + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/gather/rc.py b/tools/grit/grit/gather/rc.py new file mode 100644 index 0000000..e056427 --- /dev/null +++ b/tools/grit/grit/gather/rc.py @@ -0,0 +1,427 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Support for gathering resources from RC files. +''' + + +import re +import types + +from grit import clique +from grit import exception +from grit import util +from grit import tclib + +from grit.gather import regexp + + +# Find portions that need unescaping in resource strings. We need to be +# careful that a \\n is matched _first_ as a \\ rather than matching as +# a \ followed by a \n. +# TODO(joi) Handle ampersands if we decide to change them into <ph> +# TODO(joi) May need to handle other control characters than \n +_NEED_UNESCAPE = re.compile(r'""|\\\\|\\n|\\t') + +# Find portions that need escaping to encode string as a resource string. +_NEED_ESCAPE = re.compile(r'"|\n|\t|\\|\ \;') + +# How to escape certain characters +_ESCAPE_CHARS = { + '"' : '""', + '\n' : '\\n', + '\t' : '\\t', + '\\' : '\\\\', + ' ' : ' ' +} + +# How to unescape certain strings +_UNESCAPE_CHARS = dict([[value, key] for key, value in _ESCAPE_CHARS.items()]) + + + +class Section(regexp.RegexpGatherer): + '''A section from a resource file.''' + + def __init__(self, section_text): + '''Creates a new object. + + Args: + section_text: 'ID_SECTION_ID SECTIONTYPE\n.....\nBEGIN\n.....\nEND' + ''' + regexp.RegexpGatherer.__init__(self, section_text) + + # static method + def Escape(text): + '''Returns a version of 'text' with characters escaped that need to be + for inclusion in a resource section.''' + def Replace(match): + return _ESCAPE_CHARS[match.group()] + return _NEED_ESCAPE.sub(Replace, text) + Escape = staticmethod(Escape) + + # static method + def UnEscape(text): + '''Returns a version of 'text' with escaped characters unescaped.''' + def Replace(match): + return _UNESCAPE_CHARS[match.group()] + return _NEED_UNESCAPE.sub(Replace, text) + UnEscape = staticmethod(UnEscape) + + def _RegExpParse(self, rexp, text_to_parse): + '''Overrides _RegExpParse to add shortcut group handling. Otherwise + the same. + ''' + regexp.RegexpGatherer._RegExpParse(self, rexp, text_to_parse) + + if not self.IsSkeleton() and len(self.GetTextualIds()) > 0: + group_name = self.GetTextualIds()[0] + for c in self.GetCliques(): + c.AddToShortcutGroup(group_name) + + # Static method + def FromFileImpl(rc_file, extkey, encoding, type): + '''Implementation of FromFile. Need to keep separate so we can have + a FromFile in this class that has its type set to Section by default. + ''' + if isinstance(rc_file, types.StringTypes): + rc_file = util.WrapInputStream(file(rc_file, 'r'), encoding) + + out = '' + begin_count = 0 + for line in rc_file.readlines(): + if len(out) > 0 or (line.strip().startswith(extkey) and + line.strip().split()[0] == extkey): + out += line + + # we stop once we reach the END for the outermost block. + begin_count_was = begin_count + if len(out) > 0 and line.strip() == 'BEGIN': + begin_count += 1 + elif len(out) > 0 and line.strip() == 'END': + begin_count -= 1 + if begin_count_was == 1 and begin_count == 0: + break + + if len(out) == 0: + raise exception.SectionNotFound('%s in file %s' % (extkey, rc_file)) + + return type(out) + FromFileImpl = staticmethod(FromFileImpl) + + # static method + def FromFile(rc_file, extkey, encoding='cp1252'): + '''Retrieves the section of 'rc_file' that has the key 'extkey'. This is + matched against the start of a line, and that line and the rest of that + section in the RC file is returned. + + If 'rc_file' is a filename, it will be opened for reading using 'encoding'. + Otherwise the 'encoding' parameter is ignored. + + This method instantiates an object of type 'type' with the text from the + file. + + Args: + rc_file: file('') | 'filename.rc' + extkey: 'ID_MY_DIALOG' + encoding: 'utf-8' + type: class to instantiate with text of section + + Return: + type(text_of_section) + ''' + return Section.FromFileImpl(rc_file, extkey, encoding, Section) + FromFile = staticmethod(FromFile) + + +class Dialog(Section): + '''A resource section that contains a dialog resource.''' + + # A typical dialog resource section looks like this: + # + # IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75 + # STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU + # CAPTION "About" + # FONT 8, "System", 0, 0, 0x0 + # BEGIN + # ICON IDI_KLONK,IDC_MYICON,14,9,20,20 + # LTEXT "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8, + # SS_NOPREFIX + # LTEXT "Copyright (C) 2005",IDC_STATIC,49,20,119,8 + # DEFPUSHBUTTON "OK",IDOK,195,6,30,11,WS_GROUP + # CONTROL "Jack ""Black"" Daniels",IDC_RADIO1,"Button", + # BS_AUTORADIOBUTTON,46,51,84,10 + # END + + # We are using a sorted set of keys, and we assume that the + # group name used for descriptions (type) will come after the "text" + # group in alphabetical order. We also assume that there cannot be + # more than one description per regular expression match. + # If that's not the case some descriptions will be clobbered. + dialog_re_ = re.compile(''' + # The dialog's ID in the first line + (?P<id1>[A-Z0-9_]+)\s+DIALOG(EX)? + | + # The caption of the dialog + (?P<type1>CAPTION)\s+"(?P<text1>.*?([^"]|""))"\s + | + # Lines for controls that have text and an ID + \s+(?P<type2>[A-Z]+)\s+"(?P<text2>.*?([^"]|"")?)"\s*,\s*(?P<id2>[A-Z0-9_]+)\s*, + | + # Lines for controls that have text only + \s+(?P<type3>[A-Z]+)\s+"(?P<text3>.*?([^"]|"")?)"\s*, + | + # Lines for controls that reference other resources + \s+[A-Z]+\s+[A-Z0-9_]+\s*,\s*(?P<id3>[A-Z0-9_]*[A-Z][A-Z0-9_]*) + | + # This matches "NOT SOME_STYLE" so that it gets consumed and doesn't get + # matched by the next option (controls that have only an ID and then just + # numbers) + \s+NOT\s+[A-Z][A-Z0-9_]+ + | + # Lines for controls that have only an ID and then just numbers + \s+[A-Z]+\s+(?P<id4>[A-Z0-9_]*[A-Z][A-Z0-9_]*)\s*, + ''', re.MULTILINE | re.VERBOSE) + + def Parse(self): + '''Knows how to parse dialog resource sections.''' + self._RegExpParse(self.dialog_re_, self.text_) + + # static method + def FromFile(rc_file, extkey, encoding = 'cp1252'): + return Section.FromFileImpl(rc_file, extkey, encoding, Dialog) + FromFile = staticmethod(FromFile) + + +class Menu(Section): + '''A resource section that contains a menu resource.''' + + # A typical menu resource section looks something like this: + # + # IDC_KLONK MENU + # BEGIN + # POPUP "&File" + # BEGIN + # MENUITEM "E&xit", IDM_EXIT + # MENUITEM "This be ""Klonk"" me like", ID_FILE_THISBE + # POPUP "gonk" + # BEGIN + # MENUITEM "Klonk && is ""good""", ID_GONK_KLONKIS + # END + # END + # POPUP "&Help" + # BEGIN + # MENUITEM "&About ...", IDM_ABOUT + # END + # END + + # Description used for the messages generated for menus, to explain to + # the translators how to handle them. + MENU_MESSAGE_DESCRIPTION = ( + 'This message represents a menu. Each of the items appears in sequence ' + '(some possibly within sub-menus) in the menu. The XX01XX placeholders ' + 'serve to separate items. Each item contains an & (ampersand) character ' + 'in front of the keystroke that should be used as a shortcut for that item ' + 'in the menu. Please make sure that no two items in the same menu share ' + 'the same shortcut.' + ) + + # A dandy regexp to suck all the IDs and translateables out of a menu + # resource + menu_re_ = re.compile(''' + # Match the MENU ID on the first line + ^(?P<id1>[A-Z0-9_]+)\s+MENU + | + # Match the translateable caption for a popup menu + POPUP\s+"(?P<text1>.*?([^"]|""))"\s + | + # Match the caption & ID of a MENUITEM + MENUITEM\s+"(?P<text2>.*?([^"]|""))"\s*,\s*(?P<id2>[A-Z0-9_]+) + ''', re.MULTILINE | re.VERBOSE) + + def Parse(self): + '''Knows how to parse menu resource sections. Because it is important that + menu shortcuts are unique within the menu, we return each menu as a single + message with placeholders to break up the different menu items, rather than + return a single message per menu item. we also add an automatic description + with instructions for the translators.''' + self.single_message_ = tclib.Message(description=self.MENU_MESSAGE_DESCRIPTION) + self._RegExpParse(self.menu_re_, self.text_) + + # static method + def FromFile(rc_file, extkey, encoding = 'cp1252'): + return Section.FromFileImpl(rc_file, extkey, encoding, Menu) + FromFile = staticmethod(FromFile) + + +class Version(Section): + '''A resource section that contains a VERSIONINFO resource.''' + + # A typical version info resource can look like this: + # + # VS_VERSION_INFO VERSIONINFO + # FILEVERSION 1,0,0,1 + # PRODUCTVERSION 1,0,0,1 + # FILEFLAGSMASK 0x3fL + # #ifdef _DEBUG + # FILEFLAGS 0x1L + # #else + # FILEFLAGS 0x0L + # #endif + # FILEOS 0x4L + # FILETYPE 0x2L + # FILESUBTYPE 0x0L + # BEGIN + # BLOCK "StringFileInfo" + # BEGIN + # BLOCK "040904e4" + # BEGIN + # VALUE "CompanyName", "TODO: <Company name>" + # VALUE "FileDescription", "TODO: <File description>" + # VALUE "FileVersion", "1.0.0.1" + # VALUE "LegalCopyright", "TODO: (c) <Company name>. All rights reserved." + # VALUE "InternalName", "res_format_test.dll" + # VALUE "OriginalFilename", "res_format_test.dll" + # VALUE "ProductName", "TODO: <Product name>" + # VALUE "ProductVersion", "1.0.0.1" + # END + # END + # BLOCK "VarFileInfo" + # BEGIN + # VALUE "Translation", 0x409, 1252 + # END + # END + # + # + # In addition to the above fields, VALUE fields named "Comments" and + # "LegalTrademarks" may also be translateable. + + version_re_ = re.compile(''' + # Match the ID on the first line + ^(?P<id1>[A-Z0-9_]+)\s+VERSIONINFO + | + # Match all potentially translateable VALUE sections + \s+VALUE\s+" + ( + CompanyName|FileDescription|LegalCopyright| + ProductName|Comments|LegalTrademarks + )",\s+"(?P<text1>.*?([^"]|""))"\s + ''', re.MULTILINE | re.VERBOSE) + + def Parse(self): + '''Knows how to parse VERSIONINFO resource sections.''' + self._RegExpParse(self.version_re_, self.text_) + + # TODO(joi) May need to override the Translate() method to change the + # "Translation" VALUE block to indicate the correct language code. + + # static method + def FromFile(rc_file, extkey, encoding = 'cp1252'): + return Section.FromFileImpl(rc_file, extkey, encoding, Version) + FromFile = staticmethod(FromFile) + +class RCData(Section): + '''A resource section that contains some data .''' + + # A typical rcdataresource section looks like this: + # + # IDR_BLAH RCDATA { 1, 2, 3, 4 } + + dialog_re_ = re.compile(''' + ^(?P<id1>[A-Z0-9_]+)\s+RCDATA\s+(DISCARDABLE)?\s+\{.*?\} + ''', re.MULTILINE | re.VERBOSE | re.DOTALL) + + def Parse(self): + '''Knows how to parse RCDATA resource sections.''' + self._RegExpParse(self.dialog_re_, self.text_) + + # static method + def FromFile(rc_file, extkey, encoding = 'cp1252'): + '''Implementation of FromFile for resource types w/braces (not BEGIN/END) + ''' + if isinstance(rc_file, types.StringTypes): + rc_file = util.WrapInputStream(file(rc_file, 'r'), encoding) + + out = '' + begin_count = 0 + openbrace_count = 0 + for line in rc_file.readlines(): + if len(out) > 0 or line.strip().startswith(extkey): + out += line + + # we stop once balance the braces (could happen on one line) + begin_count_was = begin_count + if len(out) > 0: + openbrace_count += line.count('{') + begin_count += line.count('{') + begin_count -= line.count('}') + if ((begin_count_was == 1 and begin_count == 0) or + (openbrace_count > 0 and begin_count == 0)): + break + + if len(out) == 0: + raise exception.SectionNotFound('%s in file %s' % (extkey, rc_file)) + + return RCData(out) + FromFile = staticmethod(FromFile) + + +class Accelerators(Section): + '''An ACCELERATORS table. + ''' + + # A typical ACCELERATORS section looks like this: + # + # IDR_ACCELERATOR1 ACCELERATORS + # BEGIN + # "^C", ID_ACCELERATOR32770, ASCII, NOINVERT + # "^V", ID_ACCELERATOR32771, ASCII, NOINVERT + # VK_INSERT, ID_ACCELERATOR32772, VIRTKEY, CONTROL, NOINVERT + # END + + accelerators_re_ = re.compile(''' + # Match the ID on the first line + ^(?P<id1>[A-Z0-9_]+)\s+ACCELERATORS\s+ + | + # Match accelerators specified as VK_XXX + \s+VK_[A-Z0-9_]+,\s*(?P<id2>[A-Z0-9_]+)\s*, + | + # Match accelerators specified as e.g. "^C" + \s+"[^"]*",\s+(?P<id3>[A-Z0-9_]+)\s*, + ''', re.MULTILINE | re.VERBOSE) + + def Parse(self): + '''Knows how to parse ACCELERATORS resource sections.''' + self._RegExpParse(self.accelerators_re_, self.text_) + + # static method + def FromFile(rc_file, extkey, encoding = 'cp1252'): + return Section.FromFileImpl(rc_file, extkey, encoding, Accelerators) + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/rc_unittest.py b/tools/grit/grit/gather/rc_unittest.py new file mode 100644 index 0000000..150703a --- /dev/null +++ b/tools/grit/grit/gather/rc_unittest.py @@ -0,0 +1,390 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.gather.rc''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest +import StringIO + +from grit.gather import rc +from grit import util + + +class RcUnittest(unittest.TestCase): + + part_we_want = '''IDC_KLONKACC ACCELERATORS +BEGIN + "?", IDM_ABOUT, ASCII, ALT + "/", IDM_ABOUT, ASCII, ALT +END''' + + def testSectionFromFile(self): + buf = '''IDC_SOMETHINGELSE BINGO +BEGIN + BLA BLA + BLA BLA +END +%s + +IDC_KLONK BINGOBONGO +BEGIN + HONGO KONGO +END +''' % self.part_we_want + + f = StringIO.StringIO(buf) + + out = rc.Section.FromFile(f, 'IDC_KLONKACC') + self.failUnless(out.GetText() == self.part_we_want) + + out = rc.Section.FromFile(util.PathFromRoot(r'grit/test/data/klonk.rc'), + 'IDC_KLONKACC', + encoding='utf-16') + self.failUnless(out.GetText() == self.part_we_want) + + + def testDialog(self): + dlg = rc.Dialog('''IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75 +STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU +CAPTION "About" +FONT 8, "System", 0, 0, 0x0 +BEGIN + ICON IDI_KLONK,IDC_MYICON,14,9,20,20 + LTEXT "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8, + SS_NOPREFIX + LTEXT "Copyright (C) 2005",IDC_STATIC,49,20,119,8 + DEFPUSHBUTTON "OK",IDOK,195,6,30,11,WS_GROUP + CONTROL "Jack ""Black"" Daniels",IDC_RADIO1,"Button", + BS_AUTORADIOBUTTON,46,51,84,10 + // try a line where the ID is on the continuation line + LTEXT "blablablabla blablabla blablablablablablablabla blablabla", + ID_SMURF, whatever... +END +''') + dlg.Parse() + self.failUnless(len(dlg.GetTextualIds()) == 7) + self.failUnless(len(dlg.GetCliques()) == 6) + self.failUnless(dlg.GetCliques()[1].GetMessage().GetRealContent() == + 'klonk Version "yibbee" 1.0') + + transl = dlg.Translate('en') + self.failUnless(transl.strip() == dlg.GetText().strip()) + + def testAlternateSkeleton(self): + dlg = rc.Dialog('''IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75 +STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU +CAPTION "About" +FONT 8, "System", 0, 0, 0x0 +BEGIN + LTEXT "Yipee skippy",IDC_STATIC,49,10,119,8, + SS_NOPREFIX +END +''') + dlg.Parse() + + alt_dlg = rc.Dialog('''IDD_ABOUTBOX DIALOGEX 040704, 17, 230, 75 +STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU +CAPTION "XXXXXXXXX" +FONT 8, "System", 0, 0, 0x0 +BEGIN + LTEXT "XXXXXXXXXXXXXXXXX",IDC_STATIC,110978,10,119,8, + SS_NOPREFIX +END +''') + alt_dlg.Parse() + + transl = dlg.Translate('en', skeleton_gatherer=alt_dlg) + self.failUnless(transl.count('040704') and + transl.count('110978')) + self.failUnless(transl.count('Yipee skippy')) + + def testMenu(self): + menu = rc.Menu('''IDC_KLONK MENU +BEGIN + POPUP "&File """ + BEGIN + MENUITEM "E&xit", IDM_EXIT + MENUITEM "This be ""Klonk"" me like", ID_FILE_THISBE + POPUP "gonk" + BEGIN + MENUITEM "Klonk && is ""good""", ID_GONK_KLONKIS + END + MENUITEM "This is a very long menu caption to try to see if we can make the ID go to a continuation line, blablabla blablabla bla blabla blablabla blablabla blablabla blablabla...", + ID_FILE_THISISAVERYLONGMENUCAPTIONTOTRYTOSEEIFWECANMAKETHEIDGOTOACONTINUATIONLINE + END + POPUP "&Help" + BEGIN + MENUITEM "&About ...", IDM_ABOUT + END +END''') + + menu.Parse() + self.failUnless(len(menu.GetTextualIds()) == 6) + self.failUnless(len(menu.GetCliques()) == 1) + self.failUnless(len(menu.GetCliques()[0].GetMessage().GetPlaceholders()) == + 9) + + transl = menu.Translate('en') + self.failUnless(transl.strip() == menu.GetText().strip()) + + def testVersion(self): + version = rc.Version(''' +VS_VERSION_INFO VERSIONINFO + FILEVERSION 1,0,0,1 + PRODUCTVERSION 1,0,0,1 + FILEFLAGSMASK 0x3fL +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS 0x4L + FILETYPE 0x2L + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904e4" + BEGIN + VALUE "CompanyName", "TODO: <Company name>" + VALUE "FileDescription", "TODO: <File description>" + VALUE "FileVersion", "1.0.0.1" + VALUE "LegalCopyright", "TODO: (c) <Company name>. All rights reserved." + VALUE "InternalName", "res_format_test.dll" + VALUE "OriginalFilename", "res_format_test.dll" + VALUE "ProductName", "TODO: <Product name>" + VALUE "ProductVersion", "1.0.0.1" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1252 + END +END +'''.strip()) + version.Parse() + self.failUnless(len(version.GetTextualIds()) == 1) + self.failUnless(len(version.GetCliques()) == 4) + + transl = version.Translate('en') + self.failUnless(transl.strip() == version.GetText().strip()) + + + def testRegressionDialogBox(self): + dialog = rc.Dialog(''' +IDD_SIDEBAR_WEATHER_PANEL_PROPPAGE DIALOGEX 0, 0, 205, 157 +STYLE DS_SETFONT | DS_FIXEDSYS | WS_CHILD +FONT 8, "MS Shell Dlg", 400, 0, 0x1 +BEGIN + EDITTEXT IDC_SIDEBAR_WEATHER_NEW_CITY,3,27,112,14,ES_AUTOHSCROLL + DEFPUSHBUTTON "Add Location",IDC_SIDEBAR_WEATHER_ADD,119,27,50,14 + LISTBOX IDC_SIDEBAR_WEATHER_CURR_CITIES,3,48,127,89, + LBS_NOINTEGRALHEIGHT | WS_VSCROLL | WS_TABSTOP + PUSHBUTTON "Move Up",IDC_SIDEBAR_WEATHER_MOVE_UP,134,104,50,14 + PUSHBUTTON "Move Down",IDC_SIDEBAR_WEATHER_MOVE_DOWN,134,121,50,14 + PUSHBUTTON "Remove",IDC_SIDEBAR_WEATHER_DELETE,134,48,50,14 + LTEXT "To see current weather conditions and forecasts in the USA, enter the zip code (example: 94043) or city and state (example: Mountain View, CA).", + IDC_STATIC,3,0,199,25 + CONTROL "Fahrenheit",IDC_SIDEBAR_WEATHER_FAHRENHEIT,"Button", + BS_AUTORADIOBUTTON | WS_GROUP | WS_TABSTOP,3,144,51,10 + CONTROL "Celsius",IDC_SIDEBAR_WEATHER_CELSIUS,"Button", + BS_AUTORADIOBUTTON,57,144,38,10 +END'''.strip()) + dialog.Parse() + self.failUnless(len(dialog.GetTextualIds()) == 10) + + + def testRegressionDialogBox2(self): + dialog = rc.Dialog(''' +IDD_SIDEBAR_EMAIL_PANEL_PROPPAGE DIALOG DISCARDABLE 0, 0, 264, 220 +STYLE WS_CHILD +FONT 8, "MS Shell Dlg" +BEGIN + GROUPBOX "Email Filters",IDC_STATIC,7,3,250,190 + LTEXT "Click Add Filter to create the email filter.",IDC_STATIC,16,41,130,9 + PUSHBUTTON "Add Filter...",IDC_SIDEBAR_EMAIL_ADD_FILTER,196,38,50,14 + PUSHBUTTON "Remove",IDC_SIDEBAR_EMAIL_REMOVE,196,174,50,14 + PUSHBUTTON "", IDC_SIDEBAR_EMAIL_HIDDEN, 200, 178, 5, 5, NOT WS_VISIBLE + LISTBOX IDC_SIDEBAR_EMAIL_LIST,16,60,230,108, + LBS_NOINTEGRALHEIGHT | WS_VSCROLL | WS_TABSTOP + LTEXT "You can prevent certain emails from showing up in the sidebar with a filter.", + IDC_STATIC,16,18,234,18 +END'''.strip()) + dialog.Parse() + self.failUnless('IDC_SIDEBAR_EMAIL_HIDDEN' in dialog.GetTextualIds()) + + + def testRegressionMenuId(self): + menu = rc.Menu(''' +IDR_HYPERMENU_FOLDER MENU +BEGIN + POPUP "HyperFolder" + BEGIN + MENUITEM "Open Containing Folder", IDM_OPENFOLDER + END +END'''.strip()) + menu.Parse() + self.failUnless(len(menu.GetTextualIds()) == 2) + + def testRegressionNewlines(self): + menu = rc.Menu(''' +IDR_HYPERMENU_FOLDER MENU +BEGIN + POPUP "Hyper\\nFolder" + BEGIN + MENUITEM "Open Containing Folder", IDM_OPENFOLDER + END +END'''.strip()) + menu.Parse() + transl = menu.Translate('en') + # Shouldn't find \\n (the \n shouldn't be changed to \\n) + self.failUnless(transl.find('\\\\n') == -1) + + def testRegressionTabs(self): + menu = rc.Menu(''' +IDR_HYPERMENU_FOLDER MENU +BEGIN + POPUP "Hyper\\tFolder" + BEGIN + MENUITEM "Open Containing Folder", IDM_OPENFOLDER + END +END'''.strip()) + menu.Parse() + transl = menu.Translate('en') + # Shouldn't find \\t (the \t shouldn't be changed to \\t) + self.failUnless(transl.find('\\\\t') == -1) + + def testEscapeUnescape(self): + original = 'Hello "bingo"\n How\\are\\you\\n?' + escaped = rc.Section.Escape(original) + self.failUnless(escaped == 'Hello ""bingo""\\n How\\\\are\\\\you\\\\n?') + unescaped = rc.Section.UnEscape(escaped) + self.failUnless(unescaped == original) + + def testRegressionPathsWithSlashN(self): + original = '..\\\\..\\\\trs\\\\res\\\\nav_first.gif' + unescaped = rc.Section.UnEscape(original) + self.failUnless(unescaped == '..\\..\\trs\\res\\nav_first.gif') + + def testRegressionDialogItemsTextOnly(self): + dialog = rc.Dialog('''IDD_OPTIONS_SEARCH DIALOGEX 0, 0, 280, 292 +STYLE DS_SETFONT | DS_MODALFRAME | DS_FIXEDSYS | DS_CENTER | WS_POPUP | + WS_DISABLED | WS_CAPTION | WS_SYSMENU +CAPTION "Search" +FONT 8, "MS Shell Dlg", 400, 0, 0x1 +BEGIN + GROUPBOX "Select search buttons and options",-1,7,5,266,262 + CONTROL "",IDC_OPTIONS,"SysTreeView32",TVS_DISABLEDRAGDROP | + WS_BORDER | WS_TABSTOP | 0x800,16,19,248,218 + LTEXT "Use Google site:",-1,26,248,52,8 + COMBOBOX IDC_GOOGLE_HOME,87,245,177,256,CBS_DROPDOWNLIST | + WS_VSCROLL | WS_TABSTOP + PUSHBUTTON "Restore Defaults...",IDC_RESET,187,272,86,14 +END''') + dialog.Parse() + translateables = [c.GetMessage().GetRealContent() + for c in dialog.GetCliques()] + self.failUnless('Select search buttons and options' in translateables) + self.failUnless('Use Google site:' in translateables) + + def testAccelerators(self): + acc = rc.Accelerators('''\ +IDR_ACCELERATOR1 ACCELERATORS +BEGIN + "^C", ID_ACCELERATOR32770, ASCII, NOINVERT + "^V", ID_ACCELERATOR32771, ASCII, NOINVERT + VK_INSERT, ID_ACCELERATOR32772, VIRTKEY, CONTROL, NOINVERT +END +''') + acc.Parse() + self.failUnless(len(acc.GetTextualIds()) == 4) + self.failUnless(len(acc.GetCliques()) == 0) + + transl = acc.Translate('en') + self.failUnless(transl.strip() == acc.GetText().strip()) + + + def testRegressionEmptyString(self): + dlg = rc.Dialog('''\ +IDD_CONFIRM_QUIT_GD_DLG DIALOGEX 0, 0, 267, 108 +STYLE DS_SETFONT | DS_MODALFRAME | DS_FIXEDSYS | DS_CENTER | WS_POPUP | + WS_CAPTION +EXSTYLE WS_EX_TOPMOST +CAPTION "Google Desktop" +FONT 8, "MS Shell Dlg", 400, 0, 0x1 +BEGIN + DEFPUSHBUTTON "&Yes",IDYES,82,87,50,14 + PUSHBUTTON "&No",IDNO,136,87,50,14 + ICON 32514,IDC_STATIC,7,9,21,20 + EDITTEXT IDC_TEXTBOX,34,7,231,60,ES_MULTILINE | ES_READONLY | NOT WS_BORDER + CONTROL "", + IDC_ENABLE_GD_AUTOSTART,"Button",BS_AUTOCHECKBOX | + WS_TABSTOP,33,70,231,10 +END''') + dlg.Parse() + + def Check(): + self.failUnless(transl.count('IDC_ENABLE_GD_AUTOSTART')) + self.failUnless(transl.count('END')) + + transl = dlg.Translate('de', pseudo_if_not_available=True, + fallback_to_english=True) + Check() + transl = dlg.Translate('de', pseudo_if_not_available=True, + fallback_to_english=False) + Check() + transl = dlg.Translate('de', pseudo_if_not_available=False, + fallback_to_english=True) + Check() + transl = dlg.Translate('de', pseudo_if_not_available=False, + fallback_to_english=False) + Check() + transl = dlg.Translate('en', pseudo_if_not_available=True, + fallback_to_english=True) + Check() + transl = dlg.Translate('en', pseudo_if_not_available=True, + fallback_to_english=False) + Check() + transl = dlg.Translate('en', pseudo_if_not_available=False, + fallback_to_english=True) + Check() + transl = dlg.Translate('en', pseudo_if_not_available=False, + fallback_to_english=False) + Check() + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/gather/regexp.py b/tools/grit/grit/gather/regexp.py new file mode 100644 index 0000000..30c5abdb --- /dev/null +++ b/tools/grit/grit/gather/regexp.py @@ -0,0 +1,224 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''A baseclass for simple gatherers based on regular expressions. +''' + +import re +import types + +from grit.gather import interface +from grit import clique +from grit import tclib + + +class RegexpGatherer(interface.GathererBase): + '''Common functionality of gatherers based on parsing using a single + regular expression. + ''' + + DescriptionMapping_ = { + 'CAPTION' : 'This is a caption for a dialog', + 'CHECKBOX' : 'This is a label for a checkbox', + 'CONTROL': 'This is the text on a control', + 'CTEXT': 'This is a label for a control', + 'DEFPUSHBUTTON': 'This is a button definition', + 'GROUPBOX': 'This is a label for a grouping', + 'ICON': 'This is a label for an icon', + 'LTEXT': 'This is the text for a label', + 'PUSHBUTTON': 'This is the text for a button', + } + + def __init__(self, text): + interface.GathererBase.__init__(self) + # Original text of what we're parsing + self.text_ = text.strip() + # List of parts of the document. Translateable parts are clique.MessageClique + # objects, nontranslateable parts are plain strings. Translated messages are + # inserted back into the skeleton using the quoting rules defined by + # self.Escape() + self.skeleton_ = [] + # A list of the names of IDs that need to be defined for this resource + # section to compile correctly. + self.ids_ = [] + # True if Parse() has already been called. + self.have_parsed_ = False + # True if a translatable chunk has been added + self.translatable_chunk_ = False + # If not None, all parts of the document will be put into this single + # message; otherwise the normal skeleton approach is used. + self.single_message_ = None + # Number to use for the next placeholder name. Used only if single_message + # is not None + self.ph_counter_ = 1 + + def GetText(self): + '''Returns the original text of the section''' + return self.text_ + + def Escape(self, text): + '''Subclasses can override. Base impl is identity. + ''' + return text + + def UnEscape(self, text): + '''Subclasses can override. Base impl is identity. + ''' + return text + + def GetTextualIds(self): + '''Returns the list of textual IDs that need to be defined for this + resource section to compile correctly.''' + return self.ids_ + + def GetCliques(self): + '''Returns the message cliques for each translateable message in the + resource section.''' + return filter(lambda x: isinstance(x, clique.MessageClique), self.skeleton_) + + def Translate(self, lang, pseudo_if_not_available=True, + skeleton_gatherer=None, fallback_to_english=False): + if len(self.skeleton_) == 0: + raise exception.NotReady() + if skeleton_gatherer: + assert len(skeleton_gatherer.skeleton_) == len(self.skeleton_) + + out = [] + for ix in range(len(self.skeleton_)): + if isinstance(self.skeleton_[ix], types.StringTypes): + if skeleton_gatherer: + # Make sure the skeleton is like the original + assert(isinstance(skeleton_gatherer.skeleton_[ix], types.StringTypes)) + out.append(skeleton_gatherer.skeleton_[ix]) + else: + out.append(self.skeleton_[ix]) + else: + if skeleton_gatherer: # Make sure the skeleton is like the original + assert(not isinstance(skeleton_gatherer.skeleton_[ix], + types.StringTypes)) + msg = self.skeleton_[ix].MessageForLanguage(lang, + pseudo_if_not_available, + fallback_to_english) + + def MyEscape(text): + return self.Escape(text) + text = msg.GetRealContent(escaping_function=MyEscape) + out.append(text) + return ''.join(out) + + # Contextualization elements. Used for adding additional information + # to the message bundle description string from RC files. + def AddDescriptionElement(self, string): + if self.DescriptionMapping_.has_key(string): + description = self.DescriptionMapping_[string] + else: + description = string + if self.single_message_: + self.single_message_.SetDescription(description) + else: + if (self.translatable_chunk_): + message = self.skeleton_[len(self.skeleton_) - 1].GetMessage() + message.SetDescription(description) + + def Parse(self): + '''Parses the section. Implemented by subclasses. Idempotent.''' + raise NotImplementedError() + + def _AddNontranslateableChunk(self, chunk): + '''Adds a nontranslateable chunk.''' + if self.single_message_: + ph = tclib.Placeholder('XX%02dXX' % self.ph_counter_, chunk, chunk) + self.ph_counter_ += 1 + self.single_message_.AppendPlaceholder(ph) + else: + self.skeleton_.append(chunk) + + def _AddTranslateableChunk(self, chunk): + '''Adds a translateable chunk. It will be unescaped before being added.''' + # We don't want empty messages since they are redundant and the TC + # doesn't allow them. + if chunk == '': + return + + unescaped_text = self.UnEscape(chunk) + if self.single_message_: + self.single_message_.AppendText(unescaped_text) + else: + self.skeleton_.append(self.uberclique.MakeClique( + tclib.Message(text=unescaped_text))) + self.translatable_chunk_ = True + + def _AddTextualId(self, id): + self.ids_.append(id) + + def _RegExpParse(self, regexp, text_to_parse): + '''An implementation of Parse() that can be used for resource sections that + can be parsed using a single multi-line regular expression. + + All translateables must be in named groups that have names starting with + 'text'. All textual IDs must be in named groups that have names starting + with 'id'. All type definitions that can be included in the description + field for contextualization purposes should have a name that starts with + 'type'. + + Args: + regexp: re.compile('...', re.MULTILINE) + text_to_parse: + ''' + if self.have_parsed_: + return + self.have_parsed_ = True + + chunk_start = 0 + for match in regexp.finditer(text_to_parse): + groups = match.groupdict() + keys = groups.keys() + keys.sort() + self.translatable_chunk_ = False + for group in keys: + if group.startswith('id') and groups[group]: + self._AddTextualId(groups[group]) + elif group.startswith('text') and groups[group]: + self._AddNontranslateableChunk( + text_to_parse[chunk_start : match.start(group)]) + chunk_start = match.end(group) # Next chunk will start after the match + self._AddTranslateableChunk(groups[group]) + elif group.startswith('type') and groups[group]: + # Add the description to the skeleton_ list. This works because + # we are using a sort set of keys, and because we assume that the + # group name used for descriptions (type) will come after the "text" + # group in alphabetical order. We also assume that there cannot be + # more than one description per regular expression match. + self.AddDescriptionElement(groups[group]) + + self._AddNontranslateableChunk(text_to_parse[chunk_start:]) + + if self.single_message_: + self.skeleton_.append(self.uberclique.MakeClique(self.single_message_)) diff --git a/tools/grit/grit/gather/tr_html.py b/tools/grit/grit/gather/tr_html.py new file mode 100644 index 0000000..6591f23 --- /dev/null +++ b/tools/grit/grit/gather/tr_html.py @@ -0,0 +1,703 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''A gatherer for the TotalRecall brand of HTML templates with replaceable +portions. We wanted to reuse extern.tclib.api.handlers.html.TCHTMLParser +but this proved impossible due to the fact that the TotalRecall HTML templates +are in general quite far from parseable HTML and the TCHTMLParser derives +from HTMLParser.HTMLParser which requires relatively well-formed HTML. Some +examples of "HTML" from the TotalRecall HTML templates that wouldn't be +parseable include things like: + + <a [PARAMS]>blabla</a> (not parseable because attributes are invalid) + + <table><tr><td>[LOTSOFSTUFF]</tr></table> (not parseable because closing + </td> is in the HTML [LOTSOFSTUFF] + is replaced by) + +The other problem with using general parsers (such as TCHTMLParser) is that +we want to make sure we output the TotalRecall template with as little changes +as possible in terms of whitespace characters, layout etc. With any parser +that generates a parse tree, and generates output by dumping the parse tree, +we would always have little inconsistencies which could cause bugs (the +TotalRecall template stuff is quite brittle and can break if e.g. a tab +character is replaced with spaces). + +The solution, which may be applicable to some other HTML-like template +languages floating around Google, is to create a parser with a simple state +machine that keeps track of what kind of tag it's inside, and whether it's in +a translateable section or not. Translateable sections are: + +a) text (including [BINGO] replaceables) inside of tags that + can contain translateable text (which is all tags except + for a few) + +b) text inside of an 'alt' attribute in an <image> element, or + the 'value' attribute of a <submit>, <button> or <text> + element. + +The parser does not build up a parse tree but rather a "skeleton" which +is a list of nontranslateable strings intermingled with grit.clique.MessageClique +objects. This simplifies the parser considerably compared to a regular HTML +parser. To output a translated document, each item in the skeleton is +printed out, with the relevant Translation from each MessageCliques being used +for the requested language. + +This implementation borrows some code, constants and ideas from +extern.tclib.api.handlers.html.TCHTMLParser. +''' + + +import re +import types + +from grit import clique +from grit import exception +from grit import util +from grit import tclib + +from grit.gather import interface + + +# HTML tags which break (separate) chunks. +_BLOCK_TAGS = ['script', 'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'br', + 'body', 'style', 'head', 'title', 'table', 'tr', 'td', 'th', + 'ul', 'ol', 'dl', 'nl', 'li', 'div', 'object', 'center', + 'html', 'link', 'form', 'select', 'textarea', + 'button', 'option', 'map', 'area', 'blockquote', 'pre', + 'meta', 'xmp', 'noscript', 'label', 'tbody', 'thead', + 'script', 'style', 'pre', 'iframe', 'img', 'input', 'nowrap'] + +# HTML tags which may appear within a chunk. +_INLINE_TAGS = ['b', 'i', 'u', 'tt', 'code', 'font', 'a', 'span', 'small', + 'key', 'nobr', 'url', 'em', 's', 'sup', 'strike', + 'strong'] + +# HTML tags within which linebreaks are significant. +_PREFORMATTED_TAGS = ['textarea', 'xmp', 'pre'] + +# An array mapping some of the inline HTML tags to more meaningful +# names for those tags. This will be used when generating placeholders +# representing these tags. +_HTML_PLACEHOLDER_NAMES = { 'a' : 'link', 'br' : 'break', 'b' : 'bold', + 'i' : 'italic', 'li' : 'item', 'ol' : 'ordered_list', 'p' : 'paragraph', + 'ul' : 'unordered_list', 'img' : 'image', 'em' : 'emphasis' } + +# We append each of these characters in sequence to distinguish between +# different placeholders with basically the same name (e.g. BOLD1, BOLD2). +# Keep in mind that a placeholder name must not be a substring of any other +# placeholder name in the same message, so we can't simply count (BOLD_1 +# would be a substring of BOLD_10). +_SUFFIXES = '123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + +# Matches whitespace in an HTML document. Also matches HTML comments, which are +# treated as whitespace. +_WHITESPACE = re.compile(r'(\s| |\\n|\\r|<!--\s*desc\s*=.*?-->)+', + re.DOTALL) + +# Finds a non-whitespace character +_NON_WHITESPACE = re.compile(r'\S') + +# Matches two or more in a row (a single   is not changed into +# placeholders because different languages require different numbers of spaces +# and placeholders must match exactly; more than one is probably a "special" +# whitespace sequence and should be turned into a placeholder). +_NBSP = re.compile(r' ( )+') + +# Matches nontranslateable chunks of the document +_NONTRANSLATEABLES = re.compile(r''' + <\s*script.+?<\s*/\s*script\s*> + | + <\s*style.+?<\s*/\s*style\s*> + | + <!--.+?--> + | + <\?IMPORT\s.+?> # import tag + | + <\s*[a-zA-Z_]+:.+?> # custom tag (open) + | + <\s*/\s*[a-zA-Z_]+:.+?> # custom tag (close) + | + <!\s*[A-Z]+\s*([^>]+|"[^"]+"|'[^']+')*?> + ''', re.MULTILINE | re.DOTALL | re.VERBOSE | re.IGNORECASE) + +# Matches a tag and its attributes +_ELEMENT = re.compile(r''' + # Optional closing /, element name + <\s*(?P<closing>/)?\s*(?P<element>[a-zA-Z0-9]+)\s* + # Attributes and/or replaceables inside the tag, if any + (?P<atts>( + \s*([a-zA-Z_][-:.a-zA-Z_0-9]*) # Attribute name + (\s*=\s*(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~\'"@]*))? + | + \s*\[(\$?\~)?([A-Z0-9-_]+?)(\~\$?)?\] + )*) + \s*(?P<empty>/)?\s*> # Optional empty-tag closing /, and tag close + ''', + re.MULTILINE | re.DOTALL | re.VERBOSE) + +# Matches elements that may have translateable attributes. The value of these +# special attributes is given by group 'value1' or 'value2'. Note that this +# regexp demands that the attribute value be quoted; this is necessary because +# the non-tree-building nature of the parser means we don't know when we're +# writing out attributes, so we wouldn't know to escape spaces. +_SPECIAL_ELEMENT = re.compile(r''' + <\s*( + input[^>]+?value\s*=\s*(\'(?P<value3>[^\']*)\'|"(?P<value4>[^"]*)") + [^>]+type\s*=\s*"?'?(button|reset|text|submit)'?"? + | + ( + table[^>]+?title\s*= + | + img[^>]+?alt\s*= + | + input[^>]+?type\s*=\s*"?'?(button|reset|text|submit)'?"?[^>]+?value\s*= + ) + \s*(\'(?P<value1>[^\']*)\'|"(?P<value2>[^"]*)") + )[^>]*?> + ''', re.MULTILINE | re.DOTALL | re.VERBOSE | re.IGNORECASE) + +# Matches stuff that is translateable if it occurs in the right context +# (between tags). This includes all characters and character entities. +# Note that this also matches which needs to be handled as whitespace +# before this regexp is applied. +_CHARACTERS = re.compile(r''' + ( + \w + | + [\!\@\#\$\%\^\*\(\)\-\=\_\+\[\]\{\}\\\|\;\:\'\"\,\.\/\?\`\~] + | + &(\#[0-9]+|\#x[0-9a-fA-F]+|[A-Za-z0-9]+); + )+ + ''', re.MULTILINE | re.DOTALL | re.VERBOSE) + +# Matches Total Recall's "replaceable" tags, which are just any text +# in capitals enclosed by delimiters like [] or [~~] or [$~~$] (e.g. [HELLO], +# [~HELLO~] and [$~HELLO~$]). +_REPLACEABLE = re.compile(r'\[(\$?\~)?(?P<name>[A-Z0-9-_]+?)(\~\$?)?\]', + re.MULTILINE) + + +# Matches the silly [!]-prefixed "header" that is used in some TotalRecall +# templates. +_SILLY_HEADER = re.compile(r'\[!\]\ntitle\t(?P<title>[^\n]+?)\n.+?\n\n', + re.MULTILINE | re.DOTALL) + + +# Matches a comment that provides a description for the message it occurs in. +_DESCRIPTION_COMMENT = re.compile( + r'<!--\s*desc\s*=\s*(?P<description>.+?)\s*-->', re.DOTALL) + + +_DEBUG = 0 +def _DebugPrint(text): + if _DEBUG: + print text.encode('utf-8') + + +class HtmlChunks(object): + '''A parser that knows how to break an HTML-like document into a list of + chunks, where each chunk is either translateable or non-translateable. + The chunks are unmodified sections of the original document, so concatenating + the text of all chunks would result in the original document.''' + + def InTranslateable(self): + return self.last_translateable != -1 + + def Rest(self): + return self.text_[self.current:] + + def StartTranslateable(self): + assert not self.InTranslateable() + if self.current != 0: + # Append a nontranslateable chunk + chunk_text = self.text_[self.chunk_start : self.last_nontranslateable + 1] + # Needed in the case where document starts with a translateable. + if len(chunk_text) > 0: + self.AddChunk(False, chunk_text) + self.chunk_start = self.last_nontranslateable + 1 + self.last_translateable = self.current + self.last_nontranslateable = -1 + + def EndTranslateable(self): + assert self.InTranslateable() + # Append a translateable chunk + self.AddChunk(True, + self.text_[self.chunk_start : self.last_translateable + 1]) + self.chunk_start = self.last_translateable + 1 + self.last_translateable = -1 + self.last_nontranslateable = self.current + + def AdvancePast(self, match): + self.current += match.end() + + def AddChunk(self, translateable, text): + '''Adds a chunk to self, removing linebreaks and duplicate whitespace + if appropriate. + ''' + if translateable and not self.last_element_ in _PREFORMATTED_TAGS: + text = text.replace('\n', ' ') + text = text.replace('\r', ' ') + text = text.replace(' ', ' ') + text = text.replace(' ', ' ') + + m = _DESCRIPTION_COMMENT.search(text) + if m: + self.last_description = m.group('description') + # remove the description from the output text + text = _DESCRIPTION_COMMENT.sub('', text) + + if translateable: + description = self.last_description + self.last_description = '' + else: + description = '' + + if text != '': + self.chunks_.append((translateable, text, description)) + + def Parse(self, text): + '''Parses self.text_ into an intermediate format stored in self.chunks_ + which is translateable and nontranslateable chunks. Also returns + self.chunks_ + + Return: + [chunk1, chunk2, chunk3, ...] (instances of class Chunk) + ''' + # + # Chunker state + # + + self.text_ = text + + # A list of tuples (is_translateable, text) which represents the document + # after chunking. + self.chunks_ = [] + + # Start index of the last chunk, whether translateable or not + self.chunk_start = 0 + + # Index of the last for-sure translateable character if we are parsing + # a translateable chunk, -1 to indicate we are not in a translateable chunk. + # This is needed so that we don't include trailing whitespace in the + # translateable chunk (whitespace is neutral). + self.last_translateable = -1 + + # Index of the last for-sure nontranslateable character if we are parsing + # a nontranslateable chunk, -1 if we are not in a nontranslateable chunk. + # This is needed to make sure we can group e.g. "<b>Hello</b> there" + # together instead of just "Hello</b> there" which would be much worse + # for translation. + self.last_nontranslateable = -1 + + # Index of the character we're currently looking at. + self.current = 0 + + # The name of the last block element parsed. + self.last_element_ = '' + + # The last explicit description we found. + self.last_description = '' + + while self.current < len(self.text_): + _DebugPrint('REST: %s' % self.text_[self.current:self.current+60]) + + # First try to match whitespace + m = _WHITESPACE.match(self.Rest()) + if m: + # Whitespace is neutral, it just advances 'current' and does not switch + # between translateable/nontranslateable. If we are in a + # nontranslateable section that extends to the current point, we extend + # it to include the whitespace. If we are in a translateable section, + # we do not extend it until we find + # more translateable parts, because we never want a translateable chunk + # to end with whitespace. + if (not self.InTranslateable() and + self.last_nontranslateable == self.current - 1): + self.last_nontranslateable = self.current + m.end() - 1 + self.AdvancePast(m) + continue + + # Then we try to match nontranslateables + m = _NONTRANSLATEABLES.match(self.Rest()) + if m: + if self.InTranslateable(): + self.EndTranslateable() + self.last_nontranslateable = self.current + m.end() - 1 + self.AdvancePast(m) + continue + + # Now match all other HTML element tags (opening, closing, or empty, we + # don't care). + m = _ELEMENT.match(self.Rest()) + if m: + element_name = m.group('element').lower() + if element_name in _BLOCK_TAGS: + self.last_element_ = element_name + if self.InTranslateable(): + self.EndTranslateable() + + # Check for "special" elements, i.e. ones that have a translateable + # attribute, and handle them correctly. Note that all of the + # "special" elements are block tags, so no need to check for this + # if the tag is not a block tag. + sm = _SPECIAL_ELEMENT.match(self.Rest()) + if sm: + # Get the appropriate group name + for group in sm.groupdict().keys(): + if sm.groupdict()[group]: + break + + # First make a nontranslateable chunk up to and including the + # quote before the translateable attribute value + self.AddChunk(False, self.text_[ + self.chunk_start : self.current + sm.start(group)]) + # Then a translateable for the translateable bit + self.AddChunk(True, self.Rest()[sm.start(group) : sm.end(group)]) + # Finally correct the data invariant for the parser + self.chunk_start = self.current + sm.end(group) + + self.last_nontranslateable = self.current + m.end() - 1 + elif self.InTranslateable(): + # We're in a translateable and the tag is an inline tag, so we + # need to include it in the translateable. + self.last_translateable = self.current + m.end() - 1 + self.AdvancePast(m) + continue + + # Anything else we find must be translateable, so we advance one character + # at a time until one of the above matches. + if not self.InTranslateable(): + self.StartTranslateable() + else: + self.last_translateable = self.current + self.current += 1 + + # Close the final chunk + if self.InTranslateable(): + self.AddChunk(True, self.text_[self.chunk_start : ]) + else: + self.AddChunk(False, self.text_[self.chunk_start : ]) + + return self.chunks_ + + +def HtmlToMessage(html, include_block_tags=False, description=''): + '''Takes a bit of HTML, which must contain only "inline" HTML elements, + and changes it into a tclib.Message. This involves escaping any entities and + replacing any HTML code with placeholders. + + If include_block_tags is true, no error will be given if block tags (e.g. + <p> or <br>) are included in the HTML. + + Args: + html: 'Hello <b>[USERNAME]</b>, how <i>are</i> you?' + include_block_tags: False + + Return: + tclib.Message('Hello START_BOLD1USERNAMEEND_BOLD, ' + 'howNBSPSTART_ITALICareEND_ITALIC you?', + [ Placeholder('START_BOLD', '<b>', ''), + Placeholder('USERNAME', '[USERNAME]', ''), + Placeholder('END_BOLD', '</b>', ''), + Placeholder('START_ITALIC', '<i>', ''), + Placeholder('END_ITALIC', '</i>', ''), ]) + ''' + # Approach is: + # - first placeholderize, finding <elements>, [REPLACEABLES] and + # - then escape all character entities in text in-between placeholders + + parts = [] # List of strings (for text chunks) and tuples (ID, original) + # for placeholders + + count_names = {} # Map of base names to number of times used + end_names = {} # Map of base names to stack of end tags (for correct nesting) + + def MakeNameClosure(base, type = ''): + '''Returns a closure that can be called once all names have been allocated + to return the final name of the placeholder. This allows us to minimally + number placeholders for non-overlap. + + Also ensures that END_XXX_Y placeholders have the same Y as the + corresponding BEGIN_XXX_Y placeholder when we have nested tags of the same + type. + + Args: + base: 'phname' + type: '' | 'begin' | 'end' + + Return: + Closure() + ''' + name = base + if type != '': + name = ('%s_%s' % (type, base)).upper() + + if name in count_names.keys(): + count_names[name] += 1 + else: + count_names[name] = 1 + + def MakeFinalName(name_ = name, index = count_names[name] - 1): + if (type.lower() == 'end' and + base in end_names.keys() and len(end_names[base])): + return end_names[base].pop(-1) # For correct nesting + if count_names[name_] != 1: + name_ = '%s_%s' % (name_, _SUFFIXES[index]) + # We need to use a stack to ensure that the end-tag suffixes match + # the begin-tag suffixes. Only needed when more than one tag of the + # same type. + if type == 'begin': + end_name = ('END_%s_%s' % (base, _SUFFIXES[index])).upper() + if base in end_names.keys(): + end_names[base].append(end_name) + else: + end_names[base] = [end_name] + + return name_ + + return MakeFinalName + + current = 0 + + while current < len(html): + m = _NBSP.match(html[current:]) + if m: + parts.append((MakeNameClosure('SPACE'), m.group())) + current += m.end() + continue + + m = _REPLACEABLE.match(html[current:]) + if m: + # Replaceables allow - but placeholders don't, so replace - with _ + ph_name = MakeNameClosure('X_%s_X' % m.group('name').replace('-', '_')) + parts.append((ph_name, m.group())) + current += m.end() + continue + + m = _SPECIAL_ELEMENT.match(html[current:]) + if m: + if not include_block_tags: + raise exception.BlockTagInTranslateableChunk(html) + element_name = 'block' # for simplification + # Get the appropriate group name + for group in m.groupdict().keys(): + if m.groupdict()[group]: + break + parts.append((MakeNameClosure(element_name, 'begin'), + html[current : current + m.start(group)])) + parts.append(m.group(group)) + parts.append((MakeNameClosure(element_name, 'end'), + html[current + m.end(group) : current + m.end()])) + current += m.end() + continue + + m = _ELEMENT.match(html[current:]) + if m: + element_name = m.group('element').lower() + if not include_block_tags and not element_name in _INLINE_TAGS: + raise exception.BlockTagInTranslateableChunk(html[current:]) + if element_name in _HTML_PLACEHOLDER_NAMES: # use meaningful names + element_name = _HTML_PLACEHOLDER_NAMES[element_name] + + # Make a name for the placeholder + type = '' + if not m.group('empty'): + if m.group('closing'): + type = 'end' + else: + type = 'begin' + parts.append((MakeNameClosure(element_name, type), m.group())) + current += m.end() + continue + + if len(parts) and isinstance(parts[-1], types.StringTypes): + parts[-1] += html[current] + else: + parts.append(html[current]) + current += 1 + + msg_text = '' + placeholders = [] + for part in parts: + if isinstance(part, types.TupleType): + final_name = part[0]() + original = part[1] + msg_text += final_name + placeholders.append(tclib.Placeholder(final_name, original, '(HTML code)')) + else: + msg_text += part + + msg = tclib.Message(text=msg_text, placeholders=placeholders, + description=description) + content = msg.GetContent() + for ix in range(len(content)): + if isinstance(content[ix], types.StringTypes): + content[ix] = util.UnescapeHtml(content[ix], replace_nbsp=False) + + return msg + + +class TrHtml(interface.GathererBase): + '''Represents a document or message in the template format used by + Total Recall for HTML documents.''' + + def __init__(self, text): + '''Creates a new object that represents 'text'. + Args: + text: '<html>...</html>' + ''' + super(type(self), self).__init__() + + self.text_ = text + self.have_parsed_ = False + self.skeleton_ = [] # list of strings and MessageClique objects + + def GetText(self): + '''Returns the original text of the HTML document''' + return self.text_ + + def GetCliques(self): + '''Returns the message cliques for each translateable message in the + document.''' + return filter(lambda x: isinstance(x, clique.MessageClique), self.skeleton_) + + def Translate(self, lang, pseudo_if_not_available=True, + skeleton_gatherer=None, fallback_to_english=False): + '''Returns this document with translateable messages filled with + the translation for language 'lang'. + + Args: + lang: 'en' + pseudo_if_not_available: True + + Return: + 'ID_THIS_SECTION TYPE\n...BEGIN\n "Translated message"\n......\nEND + + Raises: + grit.exception.NotReady() if used before Parse() has been successfully + called. + grit.exception.NoSuchTranslation() if 'pseudo_if_not_available' is false + and there is no translation for the requested language. + ''' + if len(self.skeleton_) == 0: + raise exception.NotReady() + + # TODO(joi) Implement support for skeleton gatherers here. + + out = [] + for item in self.skeleton_: + if isinstance(item, types.StringTypes): + out.append(item) + else: + msg = item.MessageForLanguage(lang, + pseudo_if_not_available, + fallback_to_english) + for content in msg.GetContent(): + if isinstance(content, tclib.Placeholder): + out.append(content.GetOriginal()) + else: + # We escape " characters to increase the chance that attributes + # will be properly escaped. + out.append(util.EscapeHtml(content, True)) + + return ''.join(out) + + + # Parsing is done in two phases: First, we break the document into + # translateable and nontranslateable chunks. Second, we run through each + # translateable chunk and insert placeholders for any HTML elements, unescape + # escaped characters, etc. + def Parse(self): + if self.have_parsed_: + return + self.have_parsed_ = True + + text = self.text_ + + # First handle the silly little [!]-prefixed header because it's not + # handled by our HTML parsers. + m = _SILLY_HEADER.match(text) + if m: + self.skeleton_.append(text[:m.start('title')]) + self.skeleton_.append(self.uberclique.MakeClique( + tclib.Message(text=text[m.start('title'):m.end('title')]))) + self.skeleton_.append(text[m.end('title') : m.end()]) + text = text[m.end():] + + chunks = HtmlChunks().Parse(text) + + for chunk in chunks: + if chunk[0]: # Chunk is translateable + self.skeleton_.append(self.uberclique.MakeClique( + HtmlToMessage(chunk[1], description=chunk[2]))) + else: + self.skeleton_.append(chunk[1]) + + # Go through the skeleton and change any messages that consist solely of + # placeholders and whitespace into nontranslateable strings. + for ix in range(len(self.skeleton_)): + got_text = False + if isinstance(self.skeleton_[ix], clique.MessageClique): + msg = self.skeleton_[ix].GetMessage() + for item in msg.GetContent(): + if (isinstance(item, types.StringTypes) and _NON_WHITESPACE.search(item) + and item != ' '): + got_text = True + break + if not got_text: + self.skeleton_[ix] = msg.GetRealContent() + + + # Static method + def FromFile(html, extkey=None, encoding = 'utf-8'): + '''Creates a TrHtml object from the contents of 'html' which are decoded + using 'encoding'. Returns a new TrHtml object, upon which Parse() has not + been called. + + Args: + html: file('') | 'filename.html' + extkey: ignored + encoding: 'utf-8' (note that encoding is ignored if 'html' is not a file + name but instead an open file or file-like object) + + Return: + TrHtml(text_of_file) + ''' + if isinstance(html, types.StringTypes): + html = util.WrapInputStream(file(html, 'r'), encoding) + doc = html.read() + + # Ignore the BOM character if the document starts with one. + if len(doc) and doc[0] == u'\ufeff': + doc = doc[1:] + + return TrHtml(doc) + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/tr_html_unittest.py b/tools/grit/grit/gather/tr_html_unittest.py new file mode 100644 index 0000000..b53a8e1 --- /dev/null +++ b/tools/grit/grit/gather/tr_html_unittest.py @@ -0,0 +1,437 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.gather.tr_html''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import types +import unittest + +from grit.gather import tr_html +from grit import clique +from grit import util + + +class ParserUnittest(unittest.TestCase): + def testChunking(self): + p = tr_html.HtmlChunks() + chunks = p.Parse('<p>Hello <b>dear</b> how <i>are</i>you?<p>Fine!') + self.failUnless(chunks == [ + (False, '<p>', ''), (True, 'Hello <b>dear</b> how <i>are</i>you?', ''), + (False, '<p>', ''), (True, 'Fine!', '')]) + + chunks = p.Parse('<p> Hello <b>dear</b> how <i>are</i>you? <p>Fine!') + self.failUnless(chunks == [ + (False, '<p> ', ''), (True, 'Hello <b>dear</b> how <i>are</i>you?', ''), + (False, ' <p>', ''), (True, 'Fine!', '')]) + + chunks = p.Parse('<p> Hello <b>dear how <i>are you? <p> Fine!') + self.failUnless(chunks == [ + (False, '<p> ', ''), (True, 'Hello <b>dear how <i>are you?', ''), + (False, ' <p> ', ''), (True, 'Fine!', '')]) + + # Ensure translateable sections that start with inline tags contain + # the starting inline tag. + chunks = p.Parse('<b>Hello!</b> how are you?<p><i>I am fine.</i>') + self.failUnless(chunks == [ + (True, '<b>Hello!</b> how are you?', ''), (False, '<p>', ''), + (True, '<i>I am fine.</i>', '')]) + + # Ensure translateable sections that end with inline tags contain + # the ending inline tag. + chunks = p.Parse("Hello! How are <b>you?</b><p><i>I'm fine!</i>") + self.failUnless(chunks == [ + (True, 'Hello! How are <b>you?</b>', ''), (False, '<p>', ''), + (True, "<i>I'm fine!</i>", '')]) + + # Check capitals and explicit descriptions + chunks = p.Parse('<!-- desc=bingo! --><B>Hello!</B> how are you?<P><I>I am fine.</I>') + self.failUnless(chunks == [ + (True, '<B>Hello!</B> how are you?', 'bingo!'), (False, '<P>', ''), + (True, '<I>I am fine.</I>', '')]) + chunks = p.Parse('<B><!-- desc=bingo! -->Hello!</B> how are you?<P><I>I am fine.</I>') + self.failUnless(chunks == [ + (True, '<B>Hello!</B> how are you?', 'bingo!'), (False, '<P>', ''), + (True, '<I>I am fine.</I>', '')]) + # Linebreaks get changed to spaces just like any other HTML content + chunks = p.Parse('<B>Hello!</B> <!-- desc=bi\nngo\n! -->how are you?<P><I>I am fine.</I>') + self.failUnless(chunks == [ + (True, '<B>Hello!</B> how are you?', 'bi ngo !'), (False, '<P>', ''), + (True, '<I>I am fine.</I>', '')]) + + # In this case, because the explicit description appears after the first + # translateable, it will actually apply to the second translateable. + chunks = p.Parse('<B>Hello!</B> how are you?<!-- desc=bingo! --><P><I>I am fine.</I>') + self.failUnless(chunks == [ + (True, '<B>Hello!</B> how are you?', ''), (False, '<P>', ''), + (True, '<I>I am fine.</I>', 'bingo!')]) + + # Check that replaceables within block tags (where attributes would go) are + # handled correctly. + chunks = p.Parse('<b>Hello!</b> how are you?<p [BINGO] [$~BONGO~$]>' + '<i>I am fine.</i>') + self.failUnless(chunks == [ + (True, '<b>Hello!</b> how are you?', ''), + (False, '<p [BINGO] [$~BONGO~$]>', ''), + (True, '<i>I am fine.</i>', '')]) + + # Check that the contents of preformatted tags preserve line breaks. + chunks = p.Parse('<textarea>Hello\nthere\nhow\nare\nyou?</textarea>') + self.failUnless(chunks == [(False, '<textarea>', ''), + (True, 'Hello\nthere\nhow\nare\nyou?', ''), (False, '</textarea>', '')]) + + # ...and that other tags' line breaks are converted to spaces + chunks = p.Parse('<p>Hello\nthere\nhow\nare\nyou?</p>') + self.failUnless(chunks == [(False, '<p>', ''), + (True, 'Hello there how are you?', ''), (False, '</p>', '')]) + + def testTranslateableAttributes(self): + p = tr_html.HtmlChunks() + + # Check that the translateable attributes in <img>, <submit>, <button> and + # <text> elements buttons are handled correctly. + chunks = p.Parse('<img src=bingo.jpg alt="hello there">' + '<input type=submit value="hello">' + '<input type="button" value="hello">' + '<input type=\'text\' value=\'Howdie\'>') + self.failUnless(chunks == [ + (False, '<img src=bingo.jpg alt="', ''), (True, 'hello there', ''), + (False, '"><input type=submit value="', ''), (True, 'hello', ''), + (False, '"><input type="button" value="', ''), (True, 'hello', ''), + (False, '"><input type=\'text\' value=\'', ''), (True, 'Howdie', ''), + (False, '\'>', '')]) + + + def testTranslateableHtmlToMessage(self): + msg = tr_html.HtmlToMessage( + 'Hello <b>[USERNAME]</b>, <how> <i>are</i> you?') + pres = msg.GetPresentableContent() + self.failUnless(pres == + 'Hello BEGIN_BOLDX_USERNAME_XEND_BOLD, ' + '<how> BEGIN_ITALICareEND_ITALIC you?') + + msg = tr_html.HtmlToMessage('<b>Hello</b><I>Hello</I><b>Hello</b>') + pres = msg.GetPresentableContent() + self.failUnless(pres == + 'BEGIN_BOLD_1HelloEND_BOLD_1BEGIN_ITALICHelloEND_ITALIC' + 'BEGIN_BOLD_2HelloEND_BOLD_2') + + # Check that nesting (of the <font> tags) is handled correctly - i.e. that + # the closing placeholder numbers match the opening placeholders. + msg = tr_html.HtmlToMessage( + '''<font size=-1><font color=#FF0000>Update!</font> ''' + '''<a href='http://desktop.google.com/whatsnew.html?hl=[$~LANG~$]'>''' + '''New Features</a>: Now search PDFs, MP3s, Firefox web history, and ''' + '''more</font>''') + pres = msg.GetPresentableContent() + self.failUnless(pres == + 'BEGIN_FONT_1BEGIN_FONT_2Update!END_FONT_2 BEGIN_LINK' + 'New FeaturesEND_LINK: Now search PDFs, MP3s, Firefox ' + 'web history, and moreEND_FONT_1') + + msg = tr_html.HtmlToMessage('''<a href='[$~URL~$]'><b>[NUM][CAT]</b></a>''') + pres = msg.GetPresentableContent() + self.failUnless(pres == 'BEGIN_LINKBEGIN_BOLDX_NUM_XX_CAT_XEND_BOLDEND_LINK') + + msg = tr_html.HtmlToMessage( + '''<font size=-1><a class=q onClick='return window.qs?qs(this):1' ''' + '''href='http://[WEBSERVER][SEARCH_URI]'>Desktop</a></font> ''' + ''' ''') + pres = msg.GetPresentableContent() + self.failUnless(pres == + '''BEGIN_FONTBEGIN_LINKDesktopEND_LINKEND_FONTSPACE''') + + msg = tr_html.HtmlToMessage( + '''<br><br><center><font size=-2>©2005 Google </font></center>''', 1) + pres = msg.GetPresentableContent() + self.failUnless(pres == + u'BEGIN_BREAK_1BEGIN_BREAK_2BEGIN_CENTERBEGIN_FONT\xa92005' + u' Google END_FONTEND_CENTER') + + msg = tr_html.HtmlToMessage( + ''' - <a class=c href=[$~CACHE~$]>Cached</a>''') + pres = msg.GetPresentableContent() + self.failUnless(pres == + ' - BEGIN_LINKCachedEND_LINK') + + # Check that upper-case tags are handled correctly. + msg = tr_html.HtmlToMessage( + '''You can read the <A HREF='http://desktop.google.com/privacypolicy.''' + '''html?hl=[LANG_CODE]'>Privacy Policy</A> and <A HREF='http://desktop''' + '''.google.com/privacyfaq.html?hl=[LANG_CODE]'>Privacy FAQ</A> online.''') + pres = msg.GetPresentableContent() + self.failUnless(pres == + 'You can read the BEGIN_LINK_1Privacy PolicyEND_LINK_1 and ' + 'BEGIN_LINK_2Privacy FAQEND_LINK_2 online.') + + # Check that tags with linebreaks immediately preceding them are handled + # correctly. + msg = tr_html.HtmlToMessage( + '''You can read the +<A HREF='http://desktop.google.com/privacypolicy.html?hl=[LANG_CODE]'>Privacy Policy</A> +and <A HREF='http://desktop.google.com/privacyfaq.html?hl=[LANG_CODE]'>Privacy FAQ</A> online.''') + pres = msg.GetPresentableContent() + self.failUnless(pres == '''You can read the +BEGIN_LINK_1Privacy PolicyEND_LINK_1 +and BEGIN_LINK_2Privacy FAQEND_LINK_2 online.''') + + + +class TrHtmlUnittest(unittest.TestCase): + def testTable(self): + html = tr_html.TrHtml('''<table class="shaded-header"><tr> +<td class="header-element b expand">Preferences</td> +<td class="header-element s"> +<a href="http://desktop.google.com/preferences.html">Preferences Help</a> +</td> +</tr></table>''') + html.Parse() + self.failUnless(html.skeleton_[3].GetMessage().GetPresentableContent() == + 'BEGIN_LINKPreferences HelpEND_LINK') + + def testSubmitAttribute(self): + html = tr_html.TrHtml('''</td> +<td class="header-element"><input type=submit value="Save Preferences" +name=submit2></td> +</tr></table>''') + html.Parse() + self.failUnless(html.skeleton_[1].GetMessage().GetPresentableContent() == + 'Save Preferences') + + def testWhitespaceAfterInlineTag(self): + '''Test that even if there is whitespace after an inline tag at the start + of a translateable section the inline tag will be included. + ''' + html = tr_html.TrHtml('''<label for=DISPLAYNONE><font size=-1> Hello</font>''') + html.Parse() + self.failUnless(html.skeleton_[1].GetMessage().GetRealContent() == + '<font size=-1> Hello</font>') + + def testSillyHeader(self): + html = tr_html.TrHtml('''[!] +title\tHello +bingo +bongo +bla + +<p>Other stuff</p>''') + html.Parse() + content = html.skeleton_[1].GetMessage().GetRealContent() + self.failUnless(content == 'Hello') + self.failUnless(html.skeleton_[-1] == '</p>') + # Right after the translateable the nontranslateable should start with + # a linebreak (this catches a bug we had). + self.failUnless(html.skeleton_[2][0] == '\n') + + + def testExplicitDescriptions(self): + html = tr_html.TrHtml('Hello [USER]<br/><!-- desc=explicit --><input type="button">Go!</input>') + html.Parse() + msg = html.GetCliques()[1].GetMessage() + self.failUnless(msg.GetDescription() == 'explicit') + self.failUnless(msg.GetRealContent() == 'Go!') + + + def testRegressionInToolbarAbout(self): + html = tr_html.TrHtml.FromFile( + util.PathFromRoot(r'grit/test/data/toolbar_about.html')) + html.Parse() + cliques = html.GetCliques() + for cl in cliques: + content = cl.GetMessage().GetRealContent() + if content.count('De parvis grandis acervus erit'): + self.failIf(content.count('$/translate')) + + + def HtmlFromFileWithManualCheck(self, f): + html = tr_html.TrHtml.FromFile(f) + html.Parse() + + # For manual results inspection only... + list = [] + for item in html.skeleton_: + if isinstance(item, types.StringTypes): + list.append(item) + else: + list.append(item.GetMessage().GetPresentableContent()) + + return html + + + def testPrivacyHtml(self): + html = self.HtmlFromFileWithManualCheck( + util.PathFromRoot(r'grit/test/data/privacy.html')) + + self.failUnless(html.skeleton_[1].GetMessage().GetRealContent() == + 'Privacy and Google Desktop Search') + self.failUnless(html.skeleton_[3].startswith('<')) + self.failUnless(len(html.skeleton_) > 10) + + + def testPreferencesHtml(self): + html = self.HtmlFromFileWithManualCheck( + util.PathFromRoot(r'grit/test/data/preferences.html')) + + # Verify that we don't get '[STATUS-MESSAGE]' as the original content of + # one of the MessageClique objects (it would be a placeholder-only message + # and we're supposed to have stripped those). + + for item in filter(lambda x: isinstance(x, clique.MessageClique), + html.skeleton_): + if (item.GetMessage().GetRealContent() == '[STATUS-MESSAGE]' or + item.GetMessage().GetRealContent() == '[ADDIN-DO] [ADDIN-OPTIONS]'): + self.fail() + + self.failUnless(len(html.skeleton_) > 100) + + def AssertNumberOfTranslateables(self, files, num): + '''Fails if any of the files in files don't have exactly + num translateable sections. + + Args: + files: ['file1', 'file2'] + num: 3 + ''' + for f in files: + f = util.PathFromRoot(r'grit/test/data/%s' % f) + html = self.HtmlFromFileWithManualCheck(f) + self.failUnless(len(html.GetCliques()) == num) + + def testFewTranslateables(self): + self.AssertNumberOfTranslateables(['browser.html', 'email_thread.html', + 'header.html', 'mini.html', + 'oneclick.html', 'script.html', + 'time_related.html', 'versions.html'], 0) + self.AssertNumberOfTranslateables(['footer.html', 'hover.html'], 1) + + def testOtherHtmlFilesForManualInspection(self): + files = [ + 'about.html', 'bad_browser.html', 'cache_prefix.html', + 'cache_prefix_file.html', 'chat_result.html', 'del_footer.html', + 'del_header.html', 'deleted.html', 'details.html', 'email_result.html', + 'error.html', 'explicit_web.html', 'footer.html', + 'homepage.html', 'indexing_speed.html', + 'install_prefs.html', 'install_prefs2.html', + 'oem_enable.html', 'oem_non_admin.html', 'onebox.html', + 'password.html', 'quit_apps.html', 'recrawl.html', + 'searchbox.html', 'sidebar_h.html', 'sidebar_v.html', 'status.html', + ] + for f in files: + self.HtmlFromFileWithManualCheck( + util.PathFromRoot(r'grit/test/data/%s' % f)) + + def testTranslate(self): + # Note that the English translation of documents that use character + # literals (e.g. ©) will not be the same as the original document + # because the character literal will be transformed into the Unicode + # character itself. So for this test we choose some relatively complex + # HTML without character entities (but with because that's handled + # specially). + html = tr_html.TrHtml(''' <script> + <!-- + function checkOffice() { var w = document.getElementById("h7"); + var e = document.getElementById("h8"); var o = document.getElementById("h10"); + if (!(w.checked || e.checked)) { o.checked=0;o.disabled=1;} else {o.disabled=0;} } + // --> + </script> + <input type=checkbox [CHECK-DOC] name=DOC id=h7 onclick='checkOffice()'> + <label for=h7> Word</label><br> + <input type=checkbox [CHECK-XLS] name=XLS id=h8 onclick='checkOffice()'> + <label for=h8> Excel</label><br> + <input type=checkbox [CHECK-PPT] name=PPT id=h9> + <label for=h9> PowerPoint</label><br> + </span></td><td nowrap valign=top><span class="s"> + <input type=checkbox [CHECK-PDF] name=PDF id=hpdf> + <label for=hpdf> PDF</label><br> + <input type=checkbox [CHECK-TXT] name=TXT id=h6> + <label for=h6> Text, media, and other files</label><br> + </tr> + <tr><td nowrap valign=top colspan=3><span class="s"><br /> + <input type=checkbox [CHECK-SECUREOFFICE] name=SECUREOFFICE id=h10> + <label for=h10> Password-protected Office documents (Word, Excel)</label><br /> + <input type=checkbox [DISABLED-HTTPS] [CHECK-HTTPS] name=HTTPS id=h12><label + for=h12> Secure pages (HTTPS) in web history</label></span></td></tr> + </table>''') + html.Parse() + trans = html.Translate('en') + if (html.GetText() != trans): + self.fail() + + + def testHtmlToMessageWithBlockTags(self): + msg = tr_html.HtmlToMessage( + 'Hello<p>Howdie<img alt="bingo" src="image.gif">', True) + result = msg.GetPresentableContent() + self.failUnless( + result == 'HelloBEGIN_PARAGRAPHHowdieBEGIN_BLOCKbingoEND_BLOCK') + + msg = tr_html.HtmlToMessage( + 'Hello<p>Howdie<input type="button" value="bingo">', True) + result = msg.GetPresentableContent() + self.failUnless( + result == 'HelloBEGIN_PARAGRAPHHowdieBEGIN_BLOCKbingoEND_BLOCK') + + + def testHtmlToMessageRegressions(self): + msg = tr_html.HtmlToMessage(' - ', True) + result = msg.GetPresentableContent() + self.failUnless(result == ' - ') + + + def testEscapeUnescaped(self): + text = '© & "<hello>"' + unescaped = util.UnescapeHtml(text) + self.failUnless(unescaped == u'\u00a9\u00a0 & "<hello>"') + escaped_unescaped = util.EscapeHtml(unescaped, True) + self.failUnless(escaped_unescaped == + u'\u00a9\u00a0 & "<hello>"') + + def testRegressionCjkHtmlFile(self): + # TODO(joi) Fix this problem where unquoted attributes that + # have a value that is CJK characters causes the regular expression + # match never to return. (culprit is the _ELEMENT regexp( + if False: + html = self.HtmlFromFileWithManualCheck(util.PathFromRoot( + r'grit/test/data/ko_oem_enable_bug.html')) + self.failUnless(True) + + def testRegressionCpuHang(self): + # If this regression occurs, the unit test will never return + html = tr_html.TrHtml( + '''<input type=text size=12 id=advFileTypeEntry [~SHOW-FILETYPE-BOX~] value="[EXT]" name=ext>''') + html.Parse() + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/gather/txt.py b/tools/grit/grit/gather/txt.py new file mode 100644 index 0000000..9bc304d --- /dev/null +++ b/tools/grit/grit/gather/txt.py @@ -0,0 +1,76 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Supports making amessage from a text file. +''' + +import types + +from grit.gather import interface +from grit import tclib +from grit import util + + +class TxtFile(interface.GathererBase): + '''A text file gatherer. Very simple, all text from the file becomes a + single clique. + ''' + + def __init__(self, contents): + super(type(self), self).__init__() + self.text_ = contents + self.clique_ = None + + def Parse(self): + self.clique_ = self.uberclique.MakeClique(tclib.Message(text=self.text_)) + pass + + def GetText(self): + '''Returns the text of what is being gathered.''' + return self.text_ + + def GetTextualIds(self): + return [] + + def GetCliques(self): + '''Returns the MessageClique objects for all translateable portions.''' + return [self.clique_] + + def Translate(self, lang, pseudo_if_not_available=True, + skeleton_gatherer=None, fallback_to_english=False): + return self.clique_.MessageForLanguage(lang, + pseudo_if_not_available, + fallback_to_english).GetRealContent() + + def FromFile(filename_or_stream, extkey=None, encoding = 'cp1252'): + if isinstance(filename_or_stream, types.StringTypes): + filename_or_stream = util.WrapInputStream(file(filename_or_stream, 'rb'), encoding) + return TxtFile(filename_or_stream.read()) + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/txt_unittest.py b/tools/grit/grit/gather/txt_unittest.py new file mode 100644 index 0000000..ea5c0b3 --- /dev/null +++ b/tools/grit/grit/gather/txt_unittest.py @@ -0,0 +1,58 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for TxtFile gatherer''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + + +import StringIO +import unittest + +from grit.gather import txt + + +class TxtUnittest(unittest.TestCase): + def testGather(self): + input = StringIO.StringIO('Hello there\nHow are you?') + gatherer = txt.TxtFile.FromFile(input) + gatherer.Parse() + self.failUnless(gatherer.GetText() == input.getvalue()) + self.failUnless(len(gatherer.GetCliques()) == 1) + self.failUnless(gatherer.GetCliques()[0].GetMessage().GetRealContent() == + input.getvalue()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/grd_reader.py b/tools/grit/grit/grd_reader.py new file mode 100644 index 0000000..a6c7ce9 --- /dev/null +++ b/tools/grit/grit/grd_reader.py @@ -0,0 +1,166 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Class for reading GRD files into memory, without processing them. +''' + +import os.path +import types +import xml.sax +import xml.sax.handler + +from grit import exception +from grit.node import base +from grit.node import mapping +from grit import util + + +class StopParsingException(Exception): + '''An exception used to stop parsing.''' + pass + + +class GrdContentHandler(xml.sax.handler.ContentHandler): + def __init__(self, stop_after=None, debug=False): + # Invariant of data: + # 'root' is the root of the parse tree being created, or None if we haven't + # parsed out any elements. + # 'stack' is the a stack of elements that we push new nodes onto and + # pop from when they finish parsing, or [] if we are not currently parsing. + # 'stack[-1]' is the top of the stack. + self.root = None + self.stack = [] + self.stop_after = stop_after + self.debug = debug + + def startElement(self, name, attrs): + assert not self.root or len(self.stack) > 0 + + if self.debug: + attr_list = [] + for attr in attrs.getNames(): + attr_list.append('%s="%s"' % (attr, attrs.getValue(attr))) + if len(attr_list) == 0: attr_list = ['(none)'] + attr_list = ' '.join(attr_list) + print "Starting parsing of element %s with attributes %r" % (name, attr_list) + + typeattr = None + if 'type' in attrs.getNames(): + typeattr = attrs.getValue('type') + + node = mapping.ElementToClass(name, typeattr)() + + if not self.root: + self.root = node + + if len(self.stack) > 0: + self.stack[-1].AddChild(node) + node.StartParsing(name, self.stack[-1]) + else: + node.StartParsing(name, None) + + # Push + self.stack.append(node) + + for attr in attrs.getNames(): + node.HandleAttribute(attr, attrs.getValue(attr)) + + def endElement(self, name): + if self.debug: + print "End parsing of element %s" % name + # Pop + self.stack[-1].EndParsing() + assert len(self.stack) > 0 + self.stack = self.stack[:-1] + if self.stop_after and name == self.stop_after: + raise StopParsingException() + + def characters(self, content): + if self.stack[-1]: + self.stack[-1].AppendContent(content) + + def ignorableWhitespace(self, whitespace): + # TODO(joi) This is not supported by expat. Should use a different XML parser? + pass + + +def Parse(filename_or_stream, dir = None, flexible_root = False, + stop_after=None, debug=False): + '''Parses a GRD file into a tree of nodes (from grit.node). + + If flexible_root is False, the root node must be a <grit> element. Otherwise + it can be any element. The "own" directory of the file will only be fixed up + if the root node is a <grit> element. + + 'dir' should point to the directory of the input file, or be the full path + to the input file (the filename will be stripped). + + If 'stop_after' is provided, the parsing will stop once the first node + with this name has been fully parsed (including all its contents). + + If 'debug' is true, lots of information about the parsing events will be + printed out during parsing of the file. + + Args: + filename_or_stream: './bla.xml' (must be filename if dir is None) + dir: '.' or None (only if filename_or_stream is a filename) + flexible_root: True | False + stop_after: 'inputs' + debug: False + + Return: + Subclass of grit.node.base.Node + + Throws: + grit.exception.Parsing + ''' + handler = GrdContentHandler(stop_after=stop_after, debug=debug) + try: + xml.sax.parse(filename_or_stream, handler) + except StopParsingException: + assert stop_after + pass + except: + raise + + if not flexible_root or hasattr(handler.root, 'SetOwnDir'): + assert isinstance(filename_or_stream, types.StringType) or dir != None + if not dir: + dir = util.dirname(filename_or_stream) + if len(dir) == 0: + dir = '.' + # Fix up the base_dir so it is relative to the input file. + handler.root.SetOwnDir(dir) + return handler.root + + +if __name__ == '__main__': + util.ChangeStdoutEncoding() + print unicode(Parse(sys.argv[1])) diff --git a/tools/grit/grit/grd_reader_unittest.py b/tools/grit/grit/grd_reader_unittest.py new file mode 100644 index 0000000..7f62585 --- /dev/null +++ b/tools/grit/grit/grd_reader_unittest.py @@ -0,0 +1,127 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grd_reader package''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest +import StringIO + +from grit import grd_reader +from grit import constants +from grit import util + + +class GrdReaderUnittest(unittest.TestCase): + def testParsingAndXmlOutput(self): + input = u'''<?xml version="1.0" encoding="UTF-8"?> +<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <includes> + <include name="ID_LOGO" file="images/logo.gif" type="gif" /> + </includes> + <messages> + <if expr="True"> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + </if> + </messages> + <structures> + <structure name="IDD_NARROW_DIALOG" file="rc_files/dialogs.rc" type="dialog"> + <skeleton variant_of_revision="3" expr="lang == 'fr-FR'" file="bla.rc" /> + </structure> + <structure name="VS_VERSION_INFO" file="rc_files/version.rc" type="version" /> + </structures> + </release> + <translations> + <file lang="nl" path="nl_translations.xtb" /> + </translations> + <outputs> + <output type="rc_header" filename="resource.h" /> + <output lang="en-US" type="rc_all" filename="resource.rc" /> + </outputs> +</grit>''' + pseudo_file = StringIO.StringIO(input) + tree = grd_reader.Parse(pseudo_file, '.') + output = unicode(tree) + # All but first two lines are the same (sans enc_check) + self.failUnless('\n'.join(input.split('\n')[2:]) == + '\n'.join(output.split('\n')[2:])) + self.failUnless(tree.GetNodeById('IDS_GREETING')) + + + def testStopAfter(self): + input = u'''<?xml version="1.0" encoding="UTF-8"?> +<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <outputs> + <output filename="resource.h" type="rc_header" /> + <output filename="resource.rc" lang="en-US" type="rc_all" /> + </outputs> + <release seq="3"> + <includes> + <include type="gif" name="ID_LOGO" file="images/logo.gif"/> + </includes> + </release> +</grit>''' + pseudo_file = util.WrapInputStream(StringIO.StringIO(input)) + tree = grd_reader.Parse(pseudo_file, '.', stop_after='outputs') + # only an <outputs> child + self.failUnless(len(tree.children) == 1) + self.failUnless(tree.children[0].name == 'outputs') + + def testLongLinesWithComments(self): + input = u'''<?xml version="1.0" encoding="UTF-8"?> +<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <messages> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + This is a very long line with no linebreaks yes yes it stretches on <!-- + -->and on <!-- + -->and on! + </message> + </messages> + </release> +</grit>''' + pseudo_file = StringIO.StringIO(input) + tree = grd_reader.Parse(pseudo_file, '.') + + greeting = tree.GetNodeById('IDS_GREETING') + self.failUnless(greeting.GetCliques()[0].GetMessage().GetRealContent() == + 'This is a very long line with no linebreaks yes yes it ' + 'stretches on and on and on!') + +if __name__ == '__main__': + unittest.main() +
\ No newline at end of file diff --git a/tools/grit/grit/grit-todo.xml b/tools/grit/grit/grit-todo.xml new file mode 100644 index 0000000..b8c20fd --- /dev/null +++ b/tools/grit/grit/grit-todo.xml @@ -0,0 +1,62 @@ +<?xml version="1.0" encoding="windows-1252"?> +<TODOLIST FILEFORMAT="6" PROJECTNAME="GRIT" NEXTUNIQUEID="56" FILEVERSION="69" LASTMODIFIED="2005-08-19"> + <TASK STARTDATESTRING="2005-04-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38453.49975694" TITLE="check 'name' attribute is unique" TIMEESTUNITS="H" ID="2" PERCENTDONE="100" STARTDATE="38450.00000000" DONEDATESTRING="2005-04-11" POS="22" DONEDATE="38453.00000000"/> + <TASK STARTDATESTRING="2005-04-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38488.48189815" TITLE="import id-calculating code" TIMEESTUNITS="H" ID="3" PERCENTDONE="100" STARTDATE="38450.00000000" DONEDATESTRING="2005-05-16" POS="13" DONEDATE="38488.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38488.48209491" TITLE="Import tool for existing translations" TIMEESTUNITS="H" ID="6" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="12" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00805556" TITLE="Export XMBs" TIMEESTUNITS="H" ID="8" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-08" POS="20" DONEDATE="38511.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00924769" TITLE="Initial Integration" TIMEESTUNITS="H" ID="10" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-08" POS="10" DONEDATE="38511.00000000"> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38496.54048611" TITLE="parser for %s strings" TIMEESTUNITS="H" ID="4" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-24" POS="2" DONEDATE="38496.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38497.00261574" TITLE="import tool for existing RC files" TIMEESTUNITS="H" ID="5" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-25" POS="4" DONEDATE="38497.00000000"> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38496.92990741" TITLE="handle button value= and img alt= in message HTML text" TIMEESTUNITS="H" ID="22" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-05-24" POS="1" DONEDATE="38496.00000000"/> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38497.00258102" TITLE="&nbsp; bug" TIMEESTUNITS="H" ID="23" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-05-25" POS="2" DONEDATE="38497.00000000"/> + </TASK> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38490.61171296" TITLE="grit build" TIMEESTUNITS="H" ID="7" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-18" POS="6" DONEDATE="38490.00000000"> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38490.61168981" TITLE="use IDs gathered from gatherers for .h file" TIMEESTUNITS="H" ID="20" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-18" POS="1" DONEDATE="38490.00000000"/> + </TASK> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38504.55199074" TITLE="SCons Integration" TIMEESTUNITS="H" ID="9" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-01" POS="1" DONEDATE="38504.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38490.61181713" TITLE="handle includes" TIMEESTUNITS="H" ID="12" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-18" POS="5" DONEDATE="38490.00000000"/> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38507.98567130" TITLE="output translated HTML templates" TIMEESTUNITS="H" ID="25" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-04" POS="3" DONEDATE="38507.00000000"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38507.99394676" TITLE="bug: re-escape too much in RC dialogs etc." TIMEESTUNITS="H" ID="38" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-04" POS="7" DONEDATE="38507.00000000"/> + </TASK> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46444444" TITLE="handle structure variants" TIMEESTUNITS="H" ID="11" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="15" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46456019" TITLE="handle include variants" TIMEESTUNITS="H" ID="13" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="17" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46537037" TITLE="handle translateable text for includes (e.g. image text)" TIMEESTUNITS="H" ID="14" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="14" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46712963" TITLE="ddoc" TIMEESTUNITS="H" ID="15" STARTDATE="38488.00000000" POS="4"> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46718750" TITLE="review comments miket" TIMEESTUNITS="H" ID="16" STARTDATE="38488.00000000" POS="2"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46722222" TITLE="review comments pdoyle" TIMEESTUNITS="H" ID="17" STARTDATE="38488.00000000" POS="1"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46732639" TITLE="remove 'extkey' from structure" TIMEESTUNITS="H" ID="18" STARTDATE="38488.00000000" POS="3"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.53537037" TITLE="add 'encoding' to structure" TIMEESTUNITS="H" ID="19" STARTDATE="38488.00000000" POS="6"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38503.55304398" TITLE="document limitation: emitter doesn't emit the translated HTML templates" TIMEESTUNITS="H" ID="30" STARTDATE="38503.00000000" POS="4"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38504.58541667" TITLE="add 'internal_comment' to <message>" TIMEESTUNITS="H" ID="32" STARTDATE="38503.00000000" POS="5"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.73391204" TITLE="<outputs> can not have paths (because of SCons integration - goes to build dir)" TIMEESTUNITS="H" ID="36" STARTDATE="38503.00000000" POS="9"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38506.64265046" TITLE="<identifers> and <identifier> nodes" TIMEESTUNITS="H" ID="37" STARTDATE="38503.00000000" POS="10"/> + <TASK STARTDATESTRING="2005-06-23" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38526.62344907" TITLE="<structure> can have 'exclude_from_rc' attribute (default false)" TIMEESTUNITS="H" ID="47" STARTDATE="38526.00000000" POS="8"/> + <TASK STARTDATESTRING="2005-06-23" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38531.94135417" TITLE="add 'enc_check' to <grit>" TIMEESTUNITS="H" ID="48" STARTDATE="38526.00000000" POS="7"/> + </TASK> + <TASK STARTDATESTRING="2005-05-18" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38492.51549769" TITLE="handle nontranslateable messages (in MessageClique?)" TIMEESTUNITS="H" ID="21" PERCENTDONE="100" STARTDATE="38490.00000000" DONEDATESTRING="2005-06-16" POS="16" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.70454861" TITLE="ask cprince about SCons builder in new mk system" TIMEESTUNITS="H" ID="24" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-02" POS="25" DONEDATE="38505.00000000"/> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38504.57436343" TITLE="fix AOL resource in trunk ("???????")" TIMEESTUNITS="H" ID="26" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-01" POS="19" DONEDATE="38504.00000000"/> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38498.53893519" TITLE="rc_all vs. rc_translateable vs. rc_nontranslateable" TIMEESTUNITS="H" ID="27" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-16" POS="6" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38509.45532407" TITLE="make separate .grb "outputs" file (and change SCons integ) (??)" TIMEESTUNITS="H" ID="28" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-06" POS="8" DONEDATE="38509.00000000"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00939815" TITLE="fix unit tests so they run from any directory" TIMEESTUNITS="H" ID="33" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-08" POS="18" DONEDATE="38511.00000000"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38508.96640046" TITLE="Change R4 tool to CC correct team(s) on GRIT changes" TIMEESTUNITS="H" ID="39" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-05" POS="23" DONEDATE="38508.00000000"/> + <TASK STARTDATESTRING="2005-06-07" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00881944" TITLE="Document why wrapper.rc" TIMEESTUNITS="H" ID="40" PERCENTDONE="100" STARTDATE="38510.00000000" DONEDATESTRING="2005-06-08" POS="21" DONEDATE="38511.00000000"/> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00804398" TITLE="import XTBs" TIMEESTUNITS="H" ID="41" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-16" POS="11" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00875000" TITLE="Nightly build integration" TIMEESTUNITS="H" ID="42" STARTDATE="38511.00000000" POS="3"/> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00891204" TITLE="BUGS" TIMEESTUNITS="H" ID="43" STARTDATE="38511.00000000" POS="24"> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38513.03375000" TITLE="Should report error if RC-section structure refers to does not exist" TIMEESTUNITS="H" ID="44" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-10" POS="1" DONEDATE="38513.00000000"/> + </TASK> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00981481" TITLE="NEW FEATURES" TIMEESTUNITS="H" ID="45" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-16" POS="7" DONEDATE="38519.00000000"> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.70077546" TITLE="Implement line-continuation feature (\ at end of line?)" TIMEESTUNITS="H" ID="34" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-16" POS="1" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.70262731" TITLE="Implement conditional inclusion & reflect the conditionals from R3 RC file" TIMEESTUNITS="H" ID="35" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-16" POS="2" DONEDATE="38519.00000000"/> + </TASK> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.01046296" TITLE="TC integration (one-way TO the TC)" TIMEESTUNITS="H" ID="46" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-16" POS="5" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-06-30" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38533.59072917" TITLE="bazaar20 ad for GRIT help" TIMEESTUNITS="H" ID="49" STARTDATE="38533.00000000" POS="2"> + <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.72346065" TITLE="bazaar20 ideas" TIMEESTUNITS="H" ID="51" STARTDATE="38583.00000000" POS="1"> + <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.72354167" TITLE="GUI for adding/editing messages" TIMEESTUNITS="H" ID="52" STARTDATE="38583.00000000" POS="2"/> + <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.72365741" TITLE="XLIFF import/export" TIMEESTUNITS="H" ID="54" STARTDATE="38583.00000000" POS="1"/> + </TASK> + </TASK> + <TASK STARTDATESTRING="2005-06-30" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.73721065" TITLE="internal_comment for all resource nodes (not just <message>)" TIMEESTUNITS="H" ID="50" PERCENTDONE="100" STARTDATE="38533.00000000" DONEDATESTRING="2005-08-19" POS="9" DONEDATE="38583.73721065"/> + <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.73743056" TITLE="Preserve XML comments - this gives us line continuation and more" TIMEESTUNITS="H" ID="55" STARTDATE="38583.72326389" POS="1"/> +</TODOLIST> diff --git a/tools/grit/grit/grit_runner.py b/tools/grit/grit/grit_runner.py new file mode 100644 index 0000000..e8af95a --- /dev/null +++ b/tools/grit/grit/grit_runner.py @@ -0,0 +1,228 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Command processor for GRIT. This is the script you invoke to run the various +GRIT tools. +''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import getopt + +from grit import util + +import grit.exception + +import grit.tool.build +import grit.tool.count +import grit.tool.diff_structures +import grit.tool.menu_from_parts +import grit.tool.newgrd +import grit.tool.resize +import grit.tool.rc2grd +import grit.tool.test +import grit.tool.transl2tc +import grit.tool.unit + + +# Copyright notice +_COPYRIGHT = ''' +GRIT - the Google Resource and Internationalization Tool +Copyright (c) Google Inc. %d +''' % util.GetCurrentYear() + +# Keys for the following map +_CLASS = 1 +_REQUIRES_INPUT = 2 +_HIDDEN = 3 # optional key - presence indicates tool is hidden + + +# Maps tool names to the tool's module. Done as a list of (key, value) tuples +# instead of a map to preserve ordering. +_TOOLS = [ + ['build', { _CLASS : grit.tool.build.RcBuilder, _REQUIRES_INPUT : True }], + ['newgrd', { _CLASS : grit.tool.newgrd.NewGrd, _REQUIRES_INPUT : False }], + ['rc2grd', { _CLASS : grit.tool.rc2grd.Rc2Grd, _REQUIRES_INPUT : False }], + ['transl2tc', { _CLASS : grit.tool.transl2tc.TranslationToTc, + _REQUIRES_INPUT : False }], + ['sdiff', { _CLASS : grit.tool.diff_structures.DiffStructures, + _REQUIRES_INPUT : False }], + ['resize', { _CLASS : grit.tool.resize.ResizeDialog, _REQUIRES_INPUT : True }], + ['unit', { _CLASS : grit.tool.unit.UnitTestTool, _REQUIRES_INPUT : False }], + ['count', { _CLASS : grit.tool.count.CountMessage, _REQUIRES_INPUT : True }], + ['test', { _CLASS: grit.tool.test.TestTool, _REQUIRES_INPUT : True, _HIDDEN : True }], + ['menufromparts', { _CLASS: grit.tool.menu_from_parts.MenuTranslationsFromParts, + _REQUIRES_INPUT : True, _HIDDEN : True }], +] + + +def PrintUsage(): + tool_list = '' + for (tool, info) in _TOOLS: + if not _HIDDEN in info.keys(): + tool_list += ' %-12s %s\n' % (tool, info[_CLASS]().ShortDescription()) + + # TODO(joi) Put these back into the usage when appropriate: + # + # -d Work disconnected. This causes GRIT not to attempt connections with + # e.g. Perforce. + # + # -c Use the specified Perforce CLIENT when talking to Perforce. + print '''Usage: grit [GLOBALOPTIONS] TOOL [args to tool] + +Global options: + + -i INPUT Specifies the INPUT file to use (a .grd file). If this is not + specified, GRIT will look for the environment variable GRIT_INPUT. + If it is not present either, GRIT will try to find an input file + named 'resource.grd' in the current working directory. + + -v Print more verbose runtime information. + + -x Print extremely verbose runtime information. Implies -v + + -p FNAME Specifies that GRIT should profile its execution and output the + results to the file FNAME. + +Tools: + + TOOL can be one of the following: +%s + For more information on how to use a particular tool, and the specific + arguments you can send to that tool, execute 'grit help TOOL' +''' % (tool_list) + + +class Options(object): + '''Option storage and parsing.''' + + def __init__(self): + self.disconnected = False + self.client = '' + self.input = None + self.verbose = False + self.extra_verbose = False + self.output_stream = sys.stdout + self.profile_dest = None + + def ReadOptions(self, args): + '''Reads options from the start of args and returns the remainder.''' + (opts, args) = getopt.getopt(args, 'g:dvxc:i:p:') + for (key, val) in opts: + if key == '-d': self.disconnected = True + elif key == '-c': self.client = val + elif key == '-i': self.input = val + elif key == '-v': + self.verbose = True + util.verbose = True + elif key == '-x': + self.verbose = True + util.verbose = True + self.extra_verbose = True + util.extra_verbose = True + elif key == '-p': self.profile_dest = val + + if not self.input: + if 'GRIT_INPUT' in os.environ: + self.input = os.environ['GRIT_INPUT'] + else: + self.input = 'resource.grd' + + return args + + def __repr__(self): + return '(disconnected: %d, verbose: %d, client: %s, input: %s)' % ( + self.disconnected, self.verbose, self.client, self.input) + + +def _GetToolInfo(tool): + '''Returns the info map for the tool named 'tool' or None if there is no + such tool.''' + matches = filter(lambda t: t[0] == tool, _TOOLS) + if not len(matches): + return None + else: + return matches[0][1] + + +def Main(args): + '''Parses arguments and does the appropriate thing.''' + util.ChangeStdoutEncoding() + print _COPYRIGHT + + if not len(args) or len(args) == 1 and args[0] == 'help': + PrintUsage() + return 0 + elif len(args) == 2 and args[0] == 'help': + tool = args[1].lower() + if not _GetToolInfo(tool): + print "No such tool. Try running 'grit help' for a list of tools." + return 2 + + print ("Help for 'grit %s' (for general help, run 'grit help'):\n" + % (tool)) + print _GetToolInfo(tool)[_CLASS].__doc__ + return 0 + else: + options = Options() + args = options.ReadOptions(args) # args may be shorter after this + tool = args[0] + if not _GetToolInfo(tool): + print "No such tool. Try running 'grit help' for a list of tools." + return 2 + + try: + if _GetToolInfo(tool)[_REQUIRES_INPUT]: + os.stat(options.input) + except OSError: + print ('Input file %s not found.\n' + 'To specify a different input file:\n' + ' 1. Use the GRIT_INPUT environment variable.\n' + ' 2. Use the -i command-line option. This overrides ' + 'GRIT_INPUT.\n' + ' 3. Specify neither GRIT_INPUT or -i and GRIT will try to load ' + "'resource.grd'\n" + ' from the current directory.' % options.input) + return 2 + + toolobject = _GetToolInfo(tool)[_CLASS]() + if options.profile_dest: + import hotshot + prof = hotshot.Profile(options.profile_dest) + prof.runcall(toolobject.Run, options, args[1:]) + else: + toolobject.Run(options, args[1:]) + + +if __name__ == '__main__': + sys.exit(Main(sys.argv[1:])) diff --git a/tools/grit/grit/grit_runner_unittest.py b/tools/grit/grit/grit_runner_unittest.py new file mode 100644 index 0000000..0639f0c --- /dev/null +++ b/tools/grit/grit/grit_runner_unittest.py @@ -0,0 +1,65 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.py''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest +import StringIO + +from grit import util +import grit.grit_runner + +class OptionArgsUnittest(unittest.TestCase): + def setUp(self): + self.buf = StringIO.StringIO() + self.old_stdout = sys.stdout + sys.stdout = self.buf + + def tearDown(self): + sys.stdout = self.old_stdout + + def testSimple(self): + grit.grit_runner.Main(['-i', + util.PathFromRoot('grit/test/data/simple-input.xml'), + '-d', 'test', 'bla', 'voff', 'ga']) + output = self.buf.getvalue() + self.failUnless(output.count('disconnected')) + self.failUnless(output.count("'test'") == 0) # tool name doesn't occur + self.failUnless(output.count('bla')) + self.failUnless(output.count('simple-input.xml')) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/node/__init__.py b/tools/grit/grit/node/__init__.py new file mode 100644 index 0000000..4980b3e --- /dev/null +++ b/tools/grit/grit/node/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Package 'grit.node' +''' + +pass
\ No newline at end of file diff --git a/tools/grit/grit/node/base.py b/tools/grit/grit/node/base.py new file mode 100644 index 0000000..38cfd1f --- /dev/null +++ b/tools/grit/grit/node/base.py @@ -0,0 +1,548 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Base types for nodes in a GRIT resource tree. +''' + +import os +import types +from xml.sax import saxutils + +from grit import exception +from grit import util +from grit import clique +import grit.format.interface + + +class Node(grit.format.interface.ItemFormatter): + '''An item in the tree that has children. Also implements the + ItemFormatter interface to allow formatting a node as a GRD document.''' + + # Valid content types that can be returned by _ContentType() + _CONTENT_TYPE_NONE = 0 # No CDATA content but may have children + _CONTENT_TYPE_CDATA = 1 # Only CDATA, no children. + _CONTENT_TYPE_MIXED = 2 # CDATA and children, possibly intermingled + + def __init__(self): + self.children = [] # A list of child elements + self.mixed_content = [] # A list of u'' and/or child elements (this + # duplicates 'children' but + # is needed to preserve markup-type content). + self.name = u'' # The name of this element + self.attrs = {} # The set of attributes (keys to values) + self.parent = None # Our parent unless we are the root element. + self.uberclique = None # Allows overriding uberclique for parts of tree + + def __iter__(self): + '''An in-order iteration through the tree that this node is the + root of.''' + return self.inorder() + + def inorder(self): + '''Generator that generates first this node, then the same generator for + any child nodes.''' + yield self + for child in self.children: + for iterchild in child.inorder(): + yield iterchild + + def GetRoot(self): + '''Returns the root Node in the tree this Node belongs to.''' + curr = self + while curr.parent: + curr = curr.parent + return curr + + # TODO(joi) Use this (currently untested) optimization?: + #if hasattr(self, '_root'): + # return self._root + #curr = self + #while curr.parent and not hasattr(curr, '_root'): + # curr = curr.parent + #if curr.parent: + # self._root = curr._root + #else: + # self._root = curr + #return self._root + + def StartParsing(self, name, parent): + '''Called at the start of parsing. + + Args: + name: u'elementname' + parent: grit.node.base.Node or subclass or None + ''' + assert isinstance(name, types.StringTypes) + assert not parent or isinstance(parent, Node) + self.name = name + self.parent = parent + + def AddChild(self, child): + '''Adds a child to the list of children of this node, if it is a valid + child for the node.''' + assert isinstance(child, Node) + if (not self._IsValidChild(child) or + self._ContentType() == self._CONTENT_TYPE_CDATA): + if child.parent: + explanation = 'child %s of parent %s' % (child.name, child.parent.name) + else: + explanation = 'node %s with no parent' % child.name + raise exception.UnexpectedChild(explanation) + self.children.append(child) + self.mixed_content.append(child) + + def RemoveChild(self, child_id): + '''Removes the first node that has a "name" attribute which + matches "child_id" in the list of immediate children of + this node. + + Args: + child_id: String identifying the child to be removed + ''' + index = 0 + # Safe not to copy since we only remove the first element found + for child in self.children: + name_attr = child.attrs['name'] + if name_attr == child_id: + self.children.pop(index) + self.mixed_content.pop(index) + break + index += 1 + + def AppendContent(self, content): + '''Appends a chunk of text as content of this node. + + Args: + content: u'hello' + + Return: + None + ''' + assert isinstance(content, types.StringTypes) + if self._ContentType() != self._CONTENT_TYPE_NONE: + self.mixed_content.append(content) + elif content.strip() != '': + raise exception.UnexpectedContent() + + def HandleAttribute(self, attrib, value): + '''Informs the node of an attribute that was parsed out of the GRD file + for it. + + Args: + attrib: 'name' + value: 'fooblat' + + Return: + None + ''' + assert isinstance(attrib, types.StringTypes) + assert isinstance(value, types.StringTypes) + if self._IsValidAttribute(attrib, value): + self.attrs[attrib] = value + else: + raise exception.UnexpectedAttribute(attrib) + + def EndParsing(self): + '''Called at the end of parsing.''' + + # TODO(joi) Rewrite this, it's extremely ugly! + if len(self.mixed_content): + if isinstance(self.mixed_content[0], types.StringTypes): + # Remove leading and trailing chunks of pure whitespace. + while (len(self.mixed_content) and + isinstance(self.mixed_content[0], types.StringTypes) and + self.mixed_content[0].strip() == ''): + self.mixed_content = self.mixed_content[1:] + # Strip leading and trailing whitespace from mixed content chunks + # at front and back. + if (len(self.mixed_content) and + isinstance(self.mixed_content[0], types.StringTypes)): + self.mixed_content[0] = self.mixed_content[0].lstrip() + # Remove leading and trailing ''' (used to demarcate whitespace) + if (len(self.mixed_content) and + isinstance(self.mixed_content[0], types.StringTypes)): + if self.mixed_content[0].startswith("'''"): + self.mixed_content[0] = self.mixed_content[0][3:] + if len(self.mixed_content): + if isinstance(self.mixed_content[-1], types.StringTypes): + # Same stuff all over again for the tail end. + while (len(self.mixed_content) and + isinstance(self.mixed_content[-1], types.StringTypes) and + self.mixed_content[-1].strip() == ''): + self.mixed_content = self.mixed_content[:-1] + if (len(self.mixed_content) and + isinstance(self.mixed_content[-1], types.StringTypes)): + self.mixed_content[-1] = self.mixed_content[-1].rstrip() + if (len(self.mixed_content) and + isinstance(self.mixed_content[-1], types.StringTypes)): + if self.mixed_content[-1].endswith("'''"): + self.mixed_content[-1] = self.mixed_content[-1][:-3] + + # Check that all mandatory attributes are there. + for node_mandatt in self.MandatoryAttributes(): + mandatt_list = [] + if node_mandatt.find('|') >= 0: + mandatt_list = node_mandatt.split('|') + else: + mandatt_list.append(node_mandatt) + + mandatt_option_found = False + for mandatt in mandatt_list: + assert mandatt not in self.DefaultAttributes().keys() + if mandatt in self.attrs: + if not mandatt_option_found: + mandatt_option_found = True + else: + raise exception.MutuallyExclusiveMandatoryAttribute(mandatt) + + if not mandatt_option_found: + raise exception.MissingMandatoryAttribute(mandatt) + + # Add default attributes if not specified in input file. + for defattr in self.DefaultAttributes(): + if not defattr in self.attrs: + self.attrs[defattr] = self.DefaultAttributes()[defattr] + + def GetCdata(self): + '''Returns all CDATA of this element, concatenated into a single + string. Note that this ignores any elements embedded in CDATA.''' + return ''.join(filter(lambda c: isinstance(c, types.StringTypes), + self.mixed_content)) + + def __unicode__(self): + '''Returns this node and all nodes below it as an XML document in a Unicode + string.''' + header = u'<?xml version="1.0" encoding="UTF-8"?>\n' + return header + self.FormatXml() + + # Compliance with ItemFormatter interface. + def Format(self, item, lang_re = None, begin_item=True): + if not begin_item: + return '' + else: + return item.FormatXml() + + def FormatXml(self, indent = u'', one_line = False): + '''Returns this node and all nodes below it as an XML + element in a Unicode string. This differs from __unicode__ in that it does + not include the <?xml> stuff at the top of the string. If one_line is true, + children and CDATA are layed out in a way that preserves internal + whitespace. + ''' + assert isinstance(indent, types.StringTypes) + + content_one_line = (one_line or + self._ContentType() == self._CONTENT_TYPE_MIXED) + inside_content = self.ContentsAsXml(indent, content_one_line) + + # Then the attributes for this node. + attribs = u' ' + for (attrib, value) in self.attrs.iteritems(): + # Only print an attribute if it is other than the default value. + if (not self.DefaultAttributes().has_key(attrib) or + value != self.DefaultAttributes()[attrib]): + attribs += u'%s=%s ' % (attrib, saxutils.quoteattr(value)) + attribs = attribs.rstrip() # if no attribs, we end up with '', otherwise + # we end up with a space-prefixed string + + # Finally build the XML for our node and return it + if len(inside_content) > 0: + if one_line: + return u'<%s%s>%s</%s>' % (self.name, attribs, inside_content, self.name) + elif content_one_line: + return u'%s<%s%s>\n%s %s\n%s</%s>' % ( + indent, self.name, attribs, + indent, inside_content, + indent, self.name) + else: + return u'%s<%s%s>\n%s\n%s</%s>' % ( + indent, self.name, attribs, + inside_content, + indent, self.name) + else: + return u'%s<%s%s />' % (indent, self.name, attribs) + + def ContentsAsXml(self, indent, one_line): + '''Returns the contents of this node (CDATA and child elements) in XML + format. If 'one_line' is true, the content will be laid out on one line.''' + assert isinstance(indent, types.StringTypes) + + # Build the contents of the element. + inside_parts = [] + last_item = None + for mixed_item in self.mixed_content: + if isinstance(mixed_item, Node): + inside_parts.append(mixed_item.FormatXml(indent + u' ', one_line)) + if not one_line: + inside_parts.append(u'\n') + else: + message = mixed_item + # If this is the first item and it starts with whitespace, we add + # the ''' delimiter. + if not last_item and message.lstrip() != message: + message = u"'''" + message + inside_parts.append(util.EncodeCdata(message)) + last_item = mixed_item + + # If there are only child nodes and no cdata, there will be a spurious + # trailing \n + if len(inside_parts) and inside_parts[-1] == '\n': + inside_parts = inside_parts[:-1] + + # If the last item is a string (not a node) and ends with whitespace, + # we need to add the ''' delimiter. + if (isinstance(last_item, types.StringTypes) and + last_item.rstrip() != last_item): + inside_parts[-1] = inside_parts[-1] + u"'''" + + return u''.join(inside_parts) + + def RunGatherers(self, recursive=0, debug=False): + '''Runs all gatherers on this object, which may add to the data stored + by the object. If 'recursive' is true, will call RunGatherers() recursively + on all child nodes first. If 'debug' is True, will print out information + as it is running each nodes' gatherers. + + Gatherers for <translations> child nodes will always be run after all other + child nodes have been gathered. + ''' + if recursive: + process_last = [] + for child in self.children: + if child.name == 'translations': + process_last.append(child) + else: + child.RunGatherers(recursive=recursive, debug=debug) + for child in process_last: + child.RunGatherers(recursive=recursive, debug=debug) + + def ItemFormatter(self, type): + '''Returns an instance of the item formatter for this object of the + specified type, or None if not supported. + + Args: + type: 'rc-header' + + Return: + (object RcHeaderItemFormatter) + ''' + if type == 'xml': + return self + else: + return None + + def SatisfiesOutputCondition(self): + '''Returns true if this node is either not a child of an <if> element + or if it is a child of an <if> element and the conditions for it being + output are satisfied. + + Used to determine whether to return item formatters for formats that + obey conditional output of resources (e.g. the RC formatters). + ''' + from grit.node import misc + if not self.parent or not isinstance(self.parent, misc.IfNode): + return True + else: + return self.parent.IsConditionSatisfied() + + def _IsValidChild(self, child): + '''Returns true if 'child' is a valid child of this node. + Overridden by subclasses.''' + return False + + def _IsValidAttribute(self, name, value): + '''Returns true if 'name' is the name of a valid attribute of this element + and 'value' is a valid value for that attribute. Overriden by + subclasses unless they have only mandatory attributes.''' + return (name in self.MandatoryAttributes() or + name in self.DefaultAttributes()) + + def _ContentType(self): + '''Returns the type of content this element can have. Overridden by + subclasses. The content type can be one of the _CONTENT_TYPE_XXX constants + above.''' + return self._CONTENT_TYPE_NONE + + def MandatoryAttributes(self): + '''Returns a list of attribute names that are mandatory (non-optional) + on the current element. One can specify a list of + "mutually exclusive mandatory" attributes by specifying them as one + element in the list, separated by a "|" character. + ''' + return [] + + def DefaultAttributes(self): + '''Returns a dictionary of attribute names that have defaults, mapped to + the default value. Overridden by subclasses.''' + return {} + + def GetCliques(self): + '''Returns all MessageClique objects belonging to this node. Overridden + by subclasses. + + Return: + [clique1, clique2] or [] + ''' + return [] + + def ToRealPath(self, path_from_basedir): + '''Returns a real path (which can be absolute or relative to the current + working directory), given a path that is relative to the base directory + set for the GRIT input file. + + Args: + path_from_basedir: '..' + + Return: + 'resource' + ''' + return util.normpath(os.path.join(self.GetRoot().GetBaseDir(), + path_from_basedir)) + + def FilenameToOpen(self): + '''Returns a path, either absolute or relative to the current working + directory, that points to the file the node refers to. This is only valid + for nodes that have a 'file' or 'path' attribute. Note that the attribute + is a path to the file relative to the 'base-dir' of the .grd file, whereas + this function returns a path that can be used to open the file.''' + file_attribute = 'file' + if not file_attribute in self.attrs: + file_attribute = 'path' + return self.ToRealPath(self.attrs[file_attribute]) + + def UberClique(self): + '''Returns the uberclique that should be used for messages originating in + a given node. If the node itself has its uberclique set, that is what we + use, otherwise we search upwards until we find one. If we do not find one + even at the root node, we set the root node's uberclique to a new + uberclique instance. + ''' + node = self + while not node.uberclique and node.parent: + node = node.parent + if not node.uberclique: + node.uberclique = clique.UberClique() + return node.uberclique + + def IsTranslateable(self): + '''Returns false if the node has contents that should not be translated, + otherwise returns false (even if the node has no contents). + ''' + if not 'translateable' in self.attrs: + return True + else: + return self.attrs['translateable'] == 'true' + + def GetNodeById(self, id): + '''Returns the node in the subtree parented by this node that has a 'name' + attribute matching 'id'. Returns None if no such node is found. + ''' + for node in self: + if 'name' in node.attrs and node.attrs['name'] == id: + return node + return None + + def GetTextualIds(self): + '''Returns the textual ids of this node, if it has some. + Otherwise it just returns None. + ''' + if 'name' in self.attrs: + return [self.attrs['name']] + return None + + def EvaluateCondition(self, expr): + '''Returns true if and only if the Python expression 'expr' evaluates + to true. + + The expression is given a few local variables: + - 'lang' is the language currently being output + - 'defs' is a map of C preprocessor-style define names to their values + - 'pp_ifdef(define)' which behaves just like the C preprocessors #ifdef, + i.e. it is shorthand for "define in defs" + - 'pp_if(define)' which behaves just like the C preprocessor's #if, i.e. + it is shorthand for "define in defs and defs[define]". + ''' + root = self.GetRoot() + lang = '' + defs = {} + def pp_ifdef(define): + return define in defs + def pp_if(define): + return define in defs and defs[define] + if hasattr(root, 'output_language'): + lang = root.output_language + if hasattr(root, 'defines'): + defs = root.defines + return eval(expr, {}, + {'lang' : lang, + 'defs' : defs, + 'pp_ifdef' : pp_ifdef, + 'pp_if' : pp_if}) + + def OnlyTheseTranslations(self, languages): + '''Turns off loading of translations for languages not in the provided list. + + Attrs: + languages: ['fr', 'zh_cn'] + ''' + for node in self: + if (hasattr(node, 'IsTranslation') and + node.IsTranslation() and + node.GetLang() not in languages): + node.DisableLoading() + + def PseudoIsAllowed(self): + '''Returns true if this node is allowed to use pseudo-translations. This + is true by default, unless this node is within a <release> node that has + the allow_pseudo attribute set to false. + ''' + p = self.parent + while p: + if 'allow_pseudo' in p.attrs: + return (p.attrs['allow_pseudo'].lower() == 'true') + p = p.parent + return True + + def ShouldFallbackToEnglish(self): + '''Returns true iff this node should fall back to English when + pseudotranslations are disabled and no translation is available for a + given message. + ''' + p = self.parent + while p: + if 'fallback_to_english' in p.attrs: + return (p.attrs['fallback_to_english'].lower() == 'true') + p = p.parent + return False + +class ContentNode(Node): + '''Convenience baseclass for nodes that can have content.''' + def _ContentType(self): + return self._CONTENT_TYPE_MIXED diff --git a/tools/grit/grit/node/base_unittest.py b/tools/grit/grit/node/base_unittest.py new file mode 100644 index 0000000..8e13d0f --- /dev/null +++ b/tools/grit/grit/node/base_unittest.py @@ -0,0 +1,193 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for base.Node functionality (as used in various subclasses)''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest + +from grit.node import base +from grit.node import message +from grit.node import structure +from grit.node import variant + +def MakePlaceholder(phname='BINGO'): + ph = message.PhNode() + ph.StartParsing(u'ph', None) + ph.HandleAttribute(u'name', phname) + ph.AppendContent(u'bongo') + ph.EndParsing() + return ph + + +class NodeUnittest(unittest.TestCase): + def testWhitespaceHandling(self): + # We test using the Message node type. + node = message.MessageNode() + node.StartParsing(u'hello', None) + node.HandleAttribute(u'name', u'bla') + node.AppendContent(u" ''' two spaces ") + node.EndParsing() + self.failUnless(node.GetCdata() == u' two spaces') + + node = message.MessageNode() + node.StartParsing(u'message', None) + node.HandleAttribute(u'name', u'bla') + node.AppendContent(u" two spaces ''' ") + node.EndParsing() + self.failUnless(node.GetCdata() == u'two spaces ') + + def testWhitespaceHandlingWithChildren(self): + # We test using the Message node type. + node = message.MessageNode() + node.StartParsing(u'message', None) + node.HandleAttribute(u'name', u'bla') + node.AppendContent(u" ''' two spaces ") + node.AddChild(MakePlaceholder()) + node.AppendContent(u' space before and after ') + node.AddChild(MakePlaceholder('BONGO')) + node.AppendContent(u" space before two after '''") + node.EndParsing() + self.failUnless(node.mixed_content[0] == u' two spaces ') + self.failUnless(node.mixed_content[2] == u' space before and after ') + self.failUnless(node.mixed_content[-1] == u' space before two after ') + + def testXmlFormatMixedContent(self): + # Again test using the Message node type, because it is the only mixed + # content node. + node = message.MessageNode() + node.StartParsing(u'message', None) + node.HandleAttribute(u'name', u'name') + node.AppendContent(u'Hello <young> ') + + ph = message.PhNode() + ph.StartParsing(u'ph', None) + ph.HandleAttribute(u'name', u'USERNAME') + ph.AppendContent(u'$1') + ex = message.ExNode() + ex.StartParsing(u'ex', None) + ex.AppendContent(u'Joi') + ex.EndParsing() + ph.AddChild(ex) + ph.EndParsing() + + node.AddChild(ph) + node.EndParsing() + + non_indented_xml = node.Format(node) + self.failUnless(non_indented_xml == u'<message name="name">\n Hello ' + u'<young> <ph name="USERNAME">$1<ex>Joi</ex></ph>' + u'\n</message>') + + indented_xml = node.FormatXml(u' ') + self.failUnless(indented_xml == u' <message name="name">\n Hello ' + u'<young> <ph name="USERNAME">$1<ex>Joi</ex></ph>' + u'\n </message>') + + def testXmlFormatMixedContentWithLeadingWhitespace(self): + # Again test using the Message node type, because it is the only mixed + # content node. + node = message.MessageNode() + node.StartParsing(u'message', None) + node.HandleAttribute(u'name', u'name') + node.AppendContent(u"''' Hello <young> ") + + ph = message.PhNode() + ph.StartParsing(u'ph', None) + ph.HandleAttribute(u'name', u'USERNAME') + ph.AppendContent(u'$1') + ex = message.ExNode() + ex.StartParsing(u'ex', None) + ex.AppendContent(u'Joi') + ex.EndParsing() + ph.AddChild(ex) + ph.EndParsing() + + node.AddChild(ph) + node.AppendContent(u" yessiree '''") + node.EndParsing() + + non_indented_xml = node.Format(node) + self.failUnless(non_indented_xml == + u"<message name=\"name\">\n ''' Hello" + u' <young> <ph name="USERNAME">$1<ex>Joi</ex></ph>' + u" yessiree '''\n</message>") + + indented_xml = node.FormatXml(u' ') + self.failUnless(indented_xml == + u" <message name=\"name\">\n ''' Hello" + u' <young> <ph name="USERNAME">$1<ex>Joi</ex></ph>' + u" yessiree '''\n </message>") + + self.failUnless(node.GetNodeById('name')) + + def testXmlFormatContentWithEntities(self): + '''Tests a bug where would not be escaped correctly.''' + from grit import tclib + msg_node = message.MessageNode.Construct(None, tclib.Message( + text = 'BEGIN_BOLDHelloWHITESPACEthere!END_BOLD Bingo!', + placeholders = [ + tclib.Placeholder('BEGIN_BOLD', '<b>', 'bla'), + tclib.Placeholder('WHITESPACE', ' ', 'bla'), + tclib.Placeholder('END_BOLD', '</b>', 'bla')]), + 'BINGOBONGO') + xml = msg_node.FormatXml() + self.failUnless(xml.find(' ') == -1, 'should have no entities') + + def testIter(self): + # First build a little tree of message and ph nodes. + node = message.MessageNode() + node.StartParsing(u'message', None) + node.HandleAttribute(u'name', u'bla') + node.AppendContent(u" ''' two spaces ") + node.AppendContent(u' space before and after ') + ph = message.PhNode() + ph.StartParsing(u'ph', None) + ph.AddChild(message.ExNode()) + ph.HandleAttribute(u'name', u'BINGO') + ph.AppendContent(u'bongo') + node.AddChild(ph) + node.AddChild(message.PhNode()) + node.AppendContent(u" space before two after '''") + + order = [message.MessageNode, message.PhNode, message.ExNode, message.PhNode] + for n in node: + self.failUnless(type(n) == order[0]) + order = order[1:] + self.failUnless(len(order) == 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/node/custom/__init__.py b/tools/grit/grit/node/custom/__init__.py new file mode 100644 index 0000000..0a30448 --- /dev/null +++ b/tools/grit/grit/node/custom/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/python2.4 +# Copyright 2004 Google Inc. +# All Rights Reserved. +# Author: Joi Sigurdsson <joi@google.com> + +'''Package 'grit.node.custom' +''' + +pass
\ No newline at end of file diff --git a/tools/grit/grit/node/custom/filename.py b/tools/grit/grit/node/custom/filename.py new file mode 100644 index 0000000..1f89b9c --- /dev/null +++ b/tools/grit/grit/node/custom/filename.py @@ -0,0 +1,54 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''A CustomType for filenames.''' + +import re + +from grit import clique + + +class WindowsFilename(clique.CustomType): + '''Validates that messages can be used as Windows filenames, and strips + illegal characters out of translations. + ''' + + BANNED = re.compile('\+|:|\/|\\\\|\*|\?|\"|\<|\>|\|') + + def Validate(self, message): + return not self.BANNED.search(message.GetPresentableContent()) + + def ValidateAndModify(self, lang, translation): + is_ok = self.Validate(translation) + self.ModifyEachTextPart(lang, translation) + return is_ok + + def ModifyTextPart(self, lang, text): + return self.BANNED.sub(' ', text)
\ No newline at end of file diff --git a/tools/grit/grit/node/custom/filename_unittest.py b/tools/grit/grit/node/custom/filename_unittest.py new file mode 100644 index 0000000..5c7cba9 --- /dev/null +++ b/tools/grit/grit/node/custom/filename_unittest.py @@ -0,0 +1,58 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.node.custom.filename''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) + +import unittest +from grit.node.custom import filename +from grit import clique +from grit import tclib + + +class WindowsFilenameUnittest(unittest.TestCase): + + def testValidate(self): + factory = clique.UberClique() + msg = tclib.Message(text='Bingo bongo') + c = factory.MakeClique(msg) + c.SetCustomType(filename.WindowsFilename()) + translation = tclib.Translation(id=msg.GetId(), text='Bilingo bolongo:') + c.AddTranslation(translation, 'fr') + self.failUnless(c.MessageForLanguage('fr').GetRealContent() == 'Bilingo bolongo ') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/node/empty.py b/tools/grit/grit/node/empty.py new file mode 100644 index 0000000..a2aee54 --- /dev/null +++ b/tools/grit/grit/node/empty.py @@ -0,0 +1,94 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Container nodes that don't have any logic. +''' + + +from grit.node import base +from grit.node import include +from grit.node import structure +from grit.node import message +from grit.node import io +from grit.node import misc + + +class GroupingNode(base.Node): + '''Base class for all the grouping elements (<structures>, <includes>, + <messages> and <identifiers>).''' + def DefaultAttributes(self): + return { + 'first_id' : '', + 'comment' : '', + 'fallback_to_english' : 'false', + } + + +class IncludesNode(GroupingNode): + '''The <includes> element.''' + def _IsValidChild(self, child): + return isinstance(child, (include.IncludeNode, misc.IfNode)) + + +class MessagesNode(GroupingNode): + '''The <messages> element.''' + def _IsValidChild(self, child): + return isinstance(child, (message.MessageNode, misc.IfNode)) + + def ItemFormatter(self, t): + '''Return the stringtable itemformatter if an RC is being formatted.''' + if t in ['rc_all', 'rc_translateable', 'rc_nontranslateable']: + from grit.format import rc # avoid circular dep by importing here + return rc.StringTable() + + +class StructuresNode(GroupingNode): + '''The <structures> element.''' + def _IsValidChild(self, child): + return isinstance(child, (structure.StructureNode, misc.IfNode)) + + +class TranslationsNode(base.Node): + '''The <translations> element.''' + def _IsValidChild(self, child): + return isinstance(child, io.FileNode) + + +class OutputsNode(base.Node): + '''The <outputs> element.''' + def _IsValidChild(self, child): + return isinstance(child, io.OutputNode) + + +class IdentifiersNode(GroupingNode): + '''The <identifiers> element.''' + def _IsValidChild(self, child): + from grit.node import misc + return isinstance(child, misc.IdentifierNode) diff --git a/tools/grit/grit/node/include.py b/tools/grit/grit/node/include.py new file mode 100644 index 0000000..0e74865 --- /dev/null +++ b/tools/grit/grit/node/include.py @@ -0,0 +1,95 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Handling of the <include> element. +''' + + +import grit.format.rc_header +import grit.format.rc + +from grit.node import base +from grit import util + +class IncludeNode(base.Node): + '''An <include> element.''' + + def _IsValidChild(self, child): + return False + + def MandatoryAttributes(self): + return ['name', 'type', 'file'] + + def DefaultAttributes(self): + return {'translateable' : 'true', + 'generateid': 'true', + 'filenameonly': 'false', + 'relativepath': 'false', + } + + def ItemFormatter(self, t): + if t == 'rc_header': + return grit.format.rc_header.Item() + elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and + self.SatisfiesOutputCondition()): + return grit.format.rc.RcInclude(self.attrs['type'].upper(), + self.attrs['filenameonly'] == 'true', + self.attrs['relativepath'] == 'true') + else: + return super(type(self), self).ItemFormatter(t) + + def FileForLanguage(self, lang, output_dir): + '''Returns the file for the specified language. This allows us to return + different files for different language variants of the include file. + ''' + return self.FilenameToOpen() + + # static method + def Construct(parent, name, type, file, translateable=True, + filenameonly=False, relativepath=False): + '''Creates a new node which is a child of 'parent', with attributes set + by parameters of the same name. + ''' + # Convert types to appropriate strings + translateable = util.BoolToString(translateable) + filenameonly = util.BoolToString(filenameonly) + relativepath = util.BoolToString(relativepath) + + node = IncludeNode() + node.StartParsing('include', parent) + node.HandleAttribute('name', name) + node.HandleAttribute('type', type) + node.HandleAttribute('file', file) + node.HandleAttribute('translateable', translateable) + node.HandleAttribute('filenameonly', filenameonly) + node.HandleAttribute('relativepath', relativepath) + node.EndParsing() + return node + Construct = staticmethod(Construct) diff --git a/tools/grit/grit/node/io.py b/tools/grit/grit/node/io.py new file mode 100644 index 0000000..b038185 --- /dev/null +++ b/tools/grit/grit/node/io.py @@ -0,0 +1,130 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The <output> and <file> elements. +''' + +import os +import re +import grit.format.rc_header + +from grit.node import base +from grit import exception +from grit import util +from grit import xtb_reader + + +class FileNode(base.Node): + '''A <file> element.''' + + def __init__(self): + super(type(self), self).__init__() + self.re = None + self.should_load_ = True + + def IsTranslation(self): + return True + + def GetLang(self): + return self.attrs['lang'] + + def DisableLoading(self): + self.should_load_ = False + + def MandatoryAttributes(self): + return ['path', 'lang'] + + def RunGatherers(self, recursive=False, debug=False): + if not self.should_load_: + return + + xtb_file = file(self.GetFilePath()) + try: + lang = xtb_reader.Parse(xtb_file, + self.UberClique().GenerateXtbParserCallback( + self.attrs['lang'], debug=debug)) + except: + print "Exception during parsing of %s" % self.GetFilePath() + raise + assert (lang == self.attrs['lang'], 'The XTB file you ' + 'reference must contain messages in the language specified\n' + 'by the \'lang\' attribute.') + + def GetFilePath(self): + return self.ToRealPath(os.path.expandvars(self.attrs['path'])) + + +class OutputNode(base.Node): + '''An <output> element.''' + + def MandatoryAttributes(self): + return ['filename', 'type'] + + def DefaultAttributes(self): + return { 'lang' : '', # empty lang indicates all languages + 'language_section' : 'neutral' # defines a language neutral section + } + + def GetType(self): + return self.attrs['type'] + + def GetLanguage(self): + '''Returns the language ID, default 'en'.''' + return self.attrs['lang'] + + def GetFilename(self): + return self.attrs['filename'] + + def GetOutputFilename(self): + if hasattr(self, 'output_filename'): + return self.output_filename + else: + return self.attrs['filename'] + + def _IsValidChild(self, child): + return isinstance(child, EmitNode) + +class EmitNode(base.ContentNode): + ''' An <emit> element.''' + + def DefaultAttributes(self): + return { 'emit_type' : 'prepend'} + + def GetEmitType(self): + '''Returns the emit_type for this node. Default is 'append'.''' + return self.attrs['emit_type'] + + def ItemFormatter(self, t): + if t == 'rc_header': + return grit.format.rc_header.EmitAppender() + else: + return super(type(self), self).ItemFormatter(t) + + diff --git a/tools/grit/grit/node/io_unittest.py b/tools/grit/grit/node/io_unittest.py new file mode 100644 index 0000000..0122adf --- /dev/null +++ b/tools/grit/grit/node/io_unittest.py @@ -0,0 +1,87 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for io.FileNode''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import os +import StringIO +import unittest + +from grit.node import misc +from grit.node import io +from grit.node import empty +from grit import grd_reader +from grit import util + + +class FileNodeUnittest(unittest.TestCase): + def testGetPath(self): + root = misc.GritNode() + root.StartParsing(u'grit', None) + root.HandleAttribute(u'latest_public_release', u'0') + root.HandleAttribute(u'current_release', u'1') + root.HandleAttribute(u'base_dir', ur'..\resource') + translations = empty.TranslationsNode() + translations.StartParsing(u'translations', root) + root.AddChild(translations) + file_node = io.FileNode() + file_node.StartParsing(u'file', translations) + file_node.HandleAttribute(u'path', ur'flugel\kugel.pdf') + translations.AddChild(file_node) + root.EndParsing() + + self.failUnless(file_node.GetFilePath() == + util.normpath( + os.path.join(ur'../resource', ur'flugel/kugel.pdf'))) + + def testLoadTranslations(self): + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <translations> + <file path="fr.xtb" lang="fr" /> + </translations> + <release seq="3"> + <messages> + <message name="ID_HELLO">Hello!</message> + <message name="ID_HELLO_USER">Hello <ph name="USERNAME">%s<ex>Joi</ex></ph></message> + </messages> + </release> + </grit>'''), util.PathFromRoot('grit/test/data')) + grd.RunGatherers(recursive=True) + self.failUnless(True) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/node/mapping.py b/tools/grit/grit/node/mapping.py new file mode 100644 index 0000000..c866b9a --- /dev/null +++ b/tools/grit/grit/node/mapping.py @@ -0,0 +1,82 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Maps each node type to an implementation class. +When adding a new node type, you add to this mapping. +''' + + +from grit import exception + +from grit.node import empty +from grit.node import message +from grit.node import misc +from grit.node import variant +from grit.node import structure +from grit.node import include +from grit.node import io + + +_ELEMENT_TO_CLASS = { + 'includes' : empty.IncludesNode, + 'messages' : empty.MessagesNode, + 'structures' : empty.StructuresNode, + 'translations' : empty.TranslationsNode, + 'outputs' : empty.OutputsNode, + 'message' : message.MessageNode, + 'ph' : message.PhNode, + 'ex' : message.ExNode, + 'grit' : misc.GritNode, + 'include' : include.IncludeNode, + 'structure' : structure.StructureNode, + 'skeleton' : variant.SkeletonNode, + 'release' : misc.ReleaseNode, + 'file' : io.FileNode, + 'output' : io.OutputNode, + 'emit' : io.EmitNode, + 'identifiers' : empty.IdentifiersNode, + 'identifier' : misc.IdentifierNode, + 'if' : misc.IfNode, +} + + +def ElementToClass(name, typeattr): + '''Maps an element to a class that handles the element. + + Args: + name: 'element' (the name of the element) + typeattr: 'type' (the value of the type attribute, if present, else None) + + Return: + type + ''' + if not _ELEMENT_TO_CLASS.has_key(name): + raise exception.UnknownElement() + return _ELEMENT_TO_CLASS[name] diff --git a/tools/grit/grit/node/message.py b/tools/grit/grit/node/message.py new file mode 100644 index 0000000..0e5373d --- /dev/null +++ b/tools/grit/grit/node/message.py @@ -0,0 +1,271 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Handling of the <message> element. +''' + +import re +import types + +from grit.node import base + +import grit.format.rc_header +import grit.format.rc + +from grit import clique +from grit import exception +from grit import tclib +from grit import util + + +# Finds whitespace at the start and end of a string which can be multiline. +_WHITESPACE = re.compile('(?P<start>\s*)(?P<body>.+?)(?P<end>\s*)\Z', + re.DOTALL | re.MULTILINE) + + +class MessageNode(base.ContentNode): + '''A <message> element.''' + + # For splitting a list of things that can be separated by commas or + # whitespace + _SPLIT_RE = re.compile('\s*,\s*|\s+') + + def __init__(self): + super(type(self), self).__init__() + # Valid after EndParsing, this is the MessageClique that contains the + # source message and any translations of it that have been loaded. + self.clique = None + + # We don't send leading and trailing whitespace into the translation + # console, but rather tack it onto the source message and any + # translations when formatting them into RC files or what have you. + self.ws_at_start = '' # Any whitespace characters at the start of the text + self.ws_at_end = '' # --"-- at the end of the text + + # A list of "shortcut groups" this message is in. We check to make sure + # that shortcut keys (e.g. &J) within each shortcut group are unique. + self.shortcut_groups_ = [] + + def _IsValidChild(self, child): + return isinstance(child, (PhNode)) + + def _IsValidAttribute(self, name, value): + if name not in ['name', 'offset', 'translateable', 'desc', 'meaning', + 'internal_comment', 'shortcut_groups', 'custom_type', + 'validation_expr']: + return False + if name == 'translateable' and value not in ['true', 'false']: + return False + return True + + def MandatoryAttributes(self): + return ['name|offset'] + + def DefaultAttributes(self): + return { + 'translateable' : 'true', + 'desc' : '', + 'meaning' : '', + 'internal_comment' : '', + 'shortcut_groups' : '', + 'custom_type' : '', + 'validation_expr' : '', + } + + def GetTextualIds(self): + ''' + Returns the concatenation of the parent's node first_id and + this node's offset if it has one, otherwise just call the + superclass' implementation + ''' + if 'offset' in self.attrs: + # we search for the first grouping node in the parents' list + # to take care of the case where the first parent is an <if> node + grouping_parent = self.parent + import grit.node.empty + while grouping_parent and not isinstance(grouping_parent, + grit.node.empty.GroupingNode): + grouping_parent = grouping_parent.parent + + assert 'first_id' in grouping_parent.attrs + return [grouping_parent.attrs['first_id'] + '_' + self.attrs['offset']] + else: + return super(type(self), self).GetTextualIds() + + def IsTranslateable(self): + return self.attrs['translateable'] == 'true' + + def ItemFormatter(self, t): + if t == 'rc_header': + return grit.format.rc_header.Item() + elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and + self.SatisfiesOutputCondition()): + return grit.format.rc.Message() + else: + return super(type(self), self).ItemFormatter(t) + + def EndParsing(self): + super(type(self), self).EndParsing() + + # Make the text (including placeholder references) and list of placeholders, + # then strip and store leading and trailing whitespace and create the + # tclib.Message() and a clique to contain it. + + text = '' + placeholders = [] + for item in self.mixed_content: + if isinstance(item, types.StringTypes): + text += item + else: + presentation = item.attrs['name'].upper() + text += presentation + ex = ' ' + if len(item.children): + ex = item.children[0].GetCdata() + original = item.GetCdata() + placeholders.append(tclib.Placeholder(presentation, original, ex)) + + m = _WHITESPACE.match(text) + if m: + self.ws_at_start = m.group('start') + self.ws_at_end = m.group('end') + text = m.group('body') + + self.shortcut_groups_ = self._SPLIT_RE.split(self.attrs['shortcut_groups']) + self.shortcut_groups_ = [i for i in self.shortcut_groups_ if i != ''] + + description_or_id = self.attrs['desc'] + if description_or_id == '' and 'name' in self.attrs: + description_or_id = 'ID: %s' % self.attrs['name'] + + message = tclib.Message(text=text, placeholders=placeholders, + description=description_or_id, + meaning=self.attrs['meaning']) + self.clique = self.UberClique().MakeClique(message, self.IsTranslateable()) + for group in self.shortcut_groups_: + self.clique.AddToShortcutGroup(group) + if self.attrs['custom_type'] != '': + self.clique.SetCustomType(util.NewClassInstance(self.attrs['custom_type'], + clique.CustomType)) + elif self.attrs['validation_expr'] != '': + self.clique.SetCustomType( + clique.OneOffCustomType(self.attrs['validation_expr'])) + + def GetCliques(self): + if self.clique: + return [self.clique] + else: + return [] + + def Translate(self, lang): + '''Returns a translated version of this message. + ''' + assert self.clique + return self.clique.MessageForLanguage(lang, + self.PseudoIsAllowed(), + self.ShouldFallbackToEnglish() + ).GetRealContent() + + def NameOrOffset(self): + if 'name' in self.attrs: + return self.attrs['name'] + else: + return self.attrs['offset'] + + # static method + def Construct(parent, message, name, desc='', meaning='', translateable=True): + '''Constructs a new message node that is a child of 'parent', with the + name, desc, meaning and translateable attributes set using the same-named + parameters and the text of the message and any placeholders taken from + 'message', which must be a tclib.Message() object.''' + # Convert type to appropriate string + if translateable: + translateable = 'true' + else: + translateable = 'false' + + node = MessageNode() + node.StartParsing('message', parent) + node.HandleAttribute('name', name) + node.HandleAttribute('desc', desc) + node.HandleAttribute('meaning', meaning) + node.HandleAttribute('translateable', translateable) + + items = message.GetContent() + for ix in range(len(items)): + if isinstance(items[ix], types.StringTypes): + text = items[ix] + + # Ensure whitespace at front and back of message is correctly handled. + if ix == 0: + text = "'''" + text + if ix == len(items) - 1: + text = text + "'''" + + node.AppendContent(text) + else: + phnode = PhNode() + phnode.StartParsing('ph', node) + phnode.HandleAttribute('name', items[ix].GetPresentation()) + phnode.AppendContent(items[ix].GetOriginal()) + + if len(items[ix].GetExample()) and items[ix].GetExample() != ' ': + exnode = ExNode() + exnode.StartParsing('ex', phnode) + exnode.AppendContent(items[ix].GetExample()) + exnode.EndParsing() + phnode.AddChild(exnode) + + phnode.EndParsing() + node.AddChild(phnode) + + node.EndParsing() + return node + Construct = staticmethod(Construct) + +class PhNode(base.ContentNode): + '''A <ph> element.''' + + def _IsValidChild(self, child): + return isinstance(child, ExNode) + + def MandatoryAttributes(self): + return ['name'] + + def EndParsing(self): + super(type(self), self).EndParsing() + # We only allow a single example for each placeholder + if len(self.children) > 1: + raise exception.TooManyExamples() + + +class ExNode(base.ContentNode): + '''An <ex> element.''' + pass diff --git a/tools/grit/grit/node/message_unittest.py b/tools/grit/grit/node/message_unittest.py new file mode 100644 index 0000000..e9e0939 --- /dev/null +++ b/tools/grit/grit/node/message_unittest.py @@ -0,0 +1,87 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.node.message''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest +import StringIO + +from grit.node import message +from grit import grd_reader +from grit import tclib + +class MessageUnittest(unittest.TestCase): + def testMessage(self): + buf = StringIO.StringIO('''<message name="IDS_GREETING" + desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message>''') + res = grd_reader.Parse(buf, flexible_root = True) + cliques = res.GetCliques() + content = cliques[0].GetMessage().GetPresentableContent() + self.failUnless(content == 'Hello USERNAME, how are you doing today?') + + def testMessageWithWhitespace(self): + buf = StringIO.StringIO('<message name="IDS_BLA" desc="">' + '\'\'\' Hello there <ph name="USERNAME">%s</ph> \'\'\'' + '</message>') + res = grd_reader.Parse(buf, flexible_root = True) + content = res.GetCliques()[0].GetMessage().GetPresentableContent() + self.failUnless(content == 'Hello there USERNAME') + self.failUnless(res.ws_at_start == ' ') + self.failUnless(res.ws_at_end == ' ') + + def testConstruct(self): + msg = tclib.Message(text=" Hello USERNAME, how are you? BINGO\t\t", + placeholders=[tclib.Placeholder('USERNAME', '%s', 'Joi'), + tclib.Placeholder('BINGO', '%d', '11')]) + msg_node = message.MessageNode.Construct(None, msg, 'BINGOBONGO') + self.failUnless(msg_node.children[0].name == 'ph') + self.failUnless(msg_node.children[0].children[0].name == 'ex') + self.failUnless(msg_node.children[0].children[0].GetCdata() == 'Joi') + self.failUnless(msg_node.children[1].children[0].GetCdata() == '11') + self.failUnless(msg_node.ws_at_start == ' ') + self.failUnless(msg_node.ws_at_end == '\t\t') + + def testUnicodeConstruct(self): + text = u'Howdie \u00fe' + msg = tclib.Message(text=text) + msg_node = message.MessageNode.Construct(None, msg, 'BINGOBONGO') + msg_from_node = msg_node.GetCdata() + self.failUnless(msg_from_node == text) + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/node/misc.py b/tools/grit/grit/node/misc.py new file mode 100644 index 0000000..749cfed --- /dev/null +++ b/tools/grit/grit/node/misc.py @@ -0,0 +1,284 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Miscellaneous node types. +''' + +import os.path + +from grit.node import base +from grit.node import message + +from grit import exception +from grit import constants +from grit import util + +import grit.format.rc_header + + + +class IfNode(base.Node): + '''A node for conditional inclusion of resources. + ''' + + def _IsValidChild(self, child): + from grit.node import empty + assert self.parent, '<if> node should never be root.' + if isinstance(self.parent, empty.IncludesNode): + from grit.node import include + return isinstance(child, include.IncludeNode) + elif isinstance(self.parent, empty.MessagesNode): + from grit.node import message + return isinstance(child, message.MessageNode) + elif isinstance(self.parent, empty.StructuresNode): + from grit.node import structure + return isinstance(child, structure.StructureNode) + else: + return False + + def MandatoryAttributes(self): + return ['expr'] + + def IsConditionSatisfied(self): + '''Returns true if and only if the Python expression stored in attribute + 'expr' evaluates to true. + ''' + return self.EvaluateCondition(self.attrs['expr']) + + +class ReleaseNode(base.Node): + '''The <release> element.''' + + def _IsValidChild(self, child): + from grit.node import empty + return isinstance(child, (empty.IncludesNode, empty.MessagesNode, + empty.StructuresNode, empty.IdentifiersNode)) + + def _IsValidAttribute(self, name, value): + return ( + (name == 'seq' and int(value) <= self.GetRoot().GetCurrentRelease()) or + name == 'allow_pseudo' + ) + + def MandatoryAttributes(self): + return ['seq'] + + def DefaultAttributes(self): + return { 'allow_pseudo' : 'true' } + + def GetReleaseNumber(): + '''Returns the sequence number of this release.''' + return self.attribs['seq'] + + +class GritNode(base.Node): + '''The <grit> root element.''' + + def __init__(self): + base.Node.__init__(self) + self.output_language = '' + self.defines = {} + + def _IsValidChild(self, child): + from grit.node import empty + return isinstance(child, (ReleaseNode, empty.TranslationsNode, + empty.OutputsNode)) + + def _IsValidAttribute(self, name, value): + if name not in ['base_dir', 'source_lang_id', + 'latest_public_release', 'current_release', + 'enc_check', 'tc_project']: + return False + if name in ['latest_public_release', 'current_release'] and value.strip( + '0123456789') != '': + return False + return True + + def MandatoryAttributes(self): + return ['latest_public_release', 'current_release'] + + def DefaultAttributes(self): + return { + 'base_dir' : '.', + 'source_lang_id' : 'en', + 'enc_check' : constants.ENCODING_CHECK, + 'tc_project' : 'NEED_TO_SET_tc_project_ATTRIBUTE', + } + + def EndParsing(self): + base.Node.EndParsing(self) + if (int(self.attrs['latest_public_release']) + > int(self.attrs['current_release'])): + raise exception.Parsing('latest_public_release cannot have a greater ' + 'value than current_release') + + self.ValidateUniqueIds() + + # Add the encoding check if it's not present (should ensure that it's always + # present in all .grd files generated by GRIT). If it's present, assert if + # it's not correct. + if 'enc_check' not in self.attrs or self.attrs['enc_check'] == '': + self.attrs['enc_check'] = constants.ENCODING_CHECK + else: + assert self.attrs['enc_check'] == constants.ENCODING_CHECK, ( + 'Are you sure your .grd file is in the correct encoding (UTF-8)?') + + def ValidateUniqueIds(self): + '''Validate that 'name' attribute is unique in all nodes in this tree + except for nodes that are children of <if> nodes. + ''' + unique_names = {} + duplicate_names = [] + for node in self: + if isinstance(node, message.PhNode): + continue # PhNode objects have a 'name' attribute which is not an ID + + node_ids = node.GetTextualIds() + if node_ids: + for node_id in node_ids: + if util.SYSTEM_IDENTIFIERS.match(node_id): + continue # predefined IDs are sometimes used more than once + + # Don't complain about duplicate IDs if they occur in a node that is + # inside an <if> node. + if (node_id in unique_names and node_id not in duplicate_names and + (not node.parent or not isinstance(node.parent, IfNode))): + duplicate_names.append(node_id) + unique_names[node_id] = 1 + + if len(duplicate_names): + raise exception.DuplicateKey(', '.join(duplicate_names)) + + + def GetCurrentRelease(self): + '''Returns the current release number.''' + return int(self.attrs['current_release']) + + def GetLatestPublicRelease(self): + '''Returns the latest public release number.''' + return int(self.attrs['latest_public_release']) + + def GetSourceLanguage(self): + '''Returns the language code of the source language.''' + return self.attrs['source_lang_id'] + + def GetTcProject(self): + '''Returns the name of this project in the TranslationConsole, or + 'NEED_TO_SET_tc_project_ATTRIBUTE' if it is not defined.''' + return self.attrs['tc_project'] + + def SetOwnDir(self, dir): + '''Informs the 'grit' element of the directory the file it is in resides. + This allows it to calculate relative paths from the input file, which is + what we desire (rather than from the current path). + + Args: + dir: r'c:\bla' + + Return: + None + ''' + assert dir + self.base_dir = os.path.normpath(os.path.join(dir, self.attrs['base_dir'])) + + def GetBaseDir(self): + '''Returns the base directory, relative to the working directory. To get + the base directory as set in the .grd file, use GetOriginalBaseDir() + ''' + if hasattr(self, 'base_dir'): + return self.base_dir + else: + return self.GetOriginalBaseDir() + + def GetOriginalBaseDir(self): + '''Returns the base directory, as set in the .grd file. + ''' + return self.attrs['base_dir'] + + def GetOutputFiles(self): + '''Returns the list of <file> nodes that are children of this node's + <outputs> child.''' + for child in self.children: + if child.name == 'outputs': + return child.children + raise exception.MissingElement() + + def ItemFormatter(self, t): + if t == 'rc_header': + from grit.format import rc_header # import here to avoid circular dep + return rc_header.TopLevel() + elif t in ['rc_all', 'rc_translateable', 'rc_nontranslateable']: + from grit.format import rc # avoid circular dep + return rc.TopLevel() + else: + return super(type(self), self).ItemFormatter(t) + + def SetOutputContext(self, output_language, defines): + self.output_language = output_language + self.defines = defines + + +class IdentifierNode(base.Node): + '''A node for specifying identifiers that should appear in the resource + header file, and be unique amongst all other resource identifiers, but don't + have any other attributes or reference any resources. + ''' + + def MandatoryAttributes(self): + return ['name'] + + def DefaultAttributes(self): + return { 'comment' : '', 'id' : '' } + + def ItemFormatter(self, t): + if t == 'rc_header': + return grit.format.rc_header.Item() + + def GetId(self): + '''Returns the id of this identifier if it has one, None otherwise + ''' + if 'id' in self.attrs: + return self.attrs['id'] + return None + + # static method + def Construct(parent, name, id, comment): + '''Creates a new node which is a child of 'parent', with attributes set + by parameters of the same name. + ''' + node = IdentifierNode() + node.StartParsing('identifier', parent) + node.HandleAttribute('name', name) + node.HandleAttribute('id', id) + node.HandleAttribute('comment', comment) + node.EndParsing() + return node + Construct = staticmethod(Construct) + diff --git a/tools/grit/grit/node/misc_unittest.py b/tools/grit/grit/node/misc_unittest.py new file mode 100644 index 0000000..791be25 --- /dev/null +++ b/tools/grit/grit/node/misc_unittest.py @@ -0,0 +1,162 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for misc.GritNode''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest +import StringIO + +from grit import grd_reader +import grit.exception +from grit import util +from grit.node import misc + + +class GritNodeUnittest(unittest.TestCase): + def testUniqueNameAttribute(self): + try: + restree = grd_reader.Parse( + util.PathFromRoot('grit/test/data/duplicate-name-input.xml')) + self.fail('Expected parsing exception because of duplicate names.') + except grit.exception.Parsing: + pass # Expected case + + +class IfNodeUnittest(unittest.TestCase): + def testIffyness(self): + grd = grd_reader.Parse(StringIO.StringIO(''' + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <messages> + <if expr="'bingo' in defs"> + <message name="IDS_BINGO"> + Bingo! + </message> + </if> + <if expr="'hello' in defs"> + <message name="IDS_HELLO"> + Hello! + </message> + </if> + <if expr="lang == 'fr' or 'FORCE_FRENCH' in defs"> + <message name="IDS_HELLO" internal_comment="French version"> + Good morning + </message> + </if> + </messages> + </release> + </grit>'''), dir='.') + + messages_node = grd.children[0].children[0] + bingo_message = messages_node.children[0].children[0] + hello_message = messages_node.children[1].children[0] + french_message = messages_node.children[2].children[0] + assert bingo_message.name == 'message' + assert hello_message.name == 'message' + assert french_message.name == 'message' + + grd.SetOutputContext('fr', {'hello' : '1'}) + self.failUnless(not bingo_message.SatisfiesOutputCondition()) + self.failUnless(hello_message.SatisfiesOutputCondition()) + self.failUnless(french_message.SatisfiesOutputCondition()) + + grd.SetOutputContext('en', {'bingo' : 1}) + self.failUnless(bingo_message.SatisfiesOutputCondition()) + self.failUnless(not hello_message.SatisfiesOutputCondition()) + self.failUnless(not french_message.SatisfiesOutputCondition()) + + grd.SetOutputContext('en', {'FORCE_FRENCH' : '1', 'bingo' : '1'}) + self.failUnless(bingo_message.SatisfiesOutputCondition()) + self.failUnless(not hello_message.SatisfiesOutputCondition()) + self.failUnless(french_message.SatisfiesOutputCondition()) + + +class ReleaseNodeUnittest(unittest.TestCase): + def testPseudoControl(self): + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="1" source_lang_id="en-US" current_release="2" base_dir="."> + <release seq="1" allow_pseudo="false"> + <messages> + <message name="IDS_HELLO"> + Hello + </message> + </messages> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" encoding="utf-16" file="klonk.rc" /> + </structures> + </release> + <release seq="2"> + <messages> + <message name="IDS_BINGO"> + Bingo + </message> + </messages> + <structures> + <structure type="menu" name="IDC_KLONKMENU" encoding="utf-16" file="klonk.rc" /> + </structures> + </release> + </grit>'''), util.PathFromRoot('grit/test/data')) + grd.RunGatherers(recursive=True) + + hello = grd.GetNodeById('IDS_HELLO') + aboutbox = grd.GetNodeById('IDD_ABOUTBOX') + bingo = grd.GetNodeById('IDS_BINGO') + menu = grd.GetNodeById('IDC_KLONKMENU') + + for node in [hello, aboutbox]: + self.failUnless(not node.PseudoIsAllowed()) + + for node in [bingo, menu]: + self.failUnless(node.PseudoIsAllowed()) + + for node in [hello, aboutbox]: + try: + formatter = node.ItemFormatter('rc_all') + formatter.Format(node, 'xyz-pseudo') + self.fail('Should have failed during Format since pseudo is not allowed') + except: + pass # expected case + + for node in [bingo, menu]: + try: + formatter = node.ItemFormatter('rc_all') + formatter.Format(node, 'xyz-pseudo') + except: + self.fail('Should not have gotten exception since pseudo is allowed') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/node/structure.py b/tools/grit/grit/node/structure.py new file mode 100644 index 0000000..9bf92244 --- /dev/null +++ b/tools/grit/grit/node/structure.py @@ -0,0 +1,284 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The <structure> element. +''' + +import os + +from grit.node import base +from grit.node import variant + +from grit import constants +from grit import exception +from grit import util + +import grit.gather.rc +import grit.gather.tr_html +import grit.gather.admin_template +import grit.gather.txt +import grit.gather.muppet_strings + +import grit.format.rc +import grit.format.rc_header + +# RTL languages +# TODO(jennyz): remove this fixed set of RTL language array +# when generic expand_variable code is added by grit team. +_RTL_LANGS = [ + 'ar', + 'iw', + 'ur', +] + +# Type of the gatherer to use for each type attribute +_GATHERERS = { + 'accelerators' : grit.gather.rc.Accelerators, + 'admin_template' : grit.gather.admin_template.AdmGatherer, + 'dialog' : grit.gather.rc.Dialog, + 'menu' : grit.gather.rc.Menu, + 'muppet' : grit.gather.muppet_strings.MuppetStrings, + 'rcdata' : grit.gather.rc.RCData, + 'tr_html' : grit.gather.tr_html.TrHtml, + 'txt' : grit.gather.txt.TxtFile, + 'version' : grit.gather.rc.Version, +} + + +# Formatter instance to use for each type attribute +# when formatting .rc files. +_RC_FORMATTERS = { + 'accelerators' : grit.format.rc.RcSection(), + 'admin_template' : grit.format.rc.RcInclude('ADM'), + 'dialog' : grit.format.rc.RcSection(), + 'menu' : grit.format.rc.RcSection(), + 'muppet' : grit.format.rc.RcInclude('XML'), + 'rcdata' : grit.format.rc.RcSection(), + 'tr_html' : grit.format.rc.RcInclude('HTML'), + 'txt' : grit.format.rc.RcInclude('TXT'), + 'version' : grit.format.rc.RcSection(), +} + + +# TODO(joi) Print a warning if the 'variant_of_revision' attribute indicates +# that a skeleton variant is older than the original file. + + +class StructureNode(base.Node): + '''A <structure> element.''' + + def __init__(self): + base.Node.__init__(self) + self.gatherer = None + self.skeletons = {} # expressions to skeleton gatherers + + def _IsValidChild(self, child): + return isinstance(child, variant.SkeletonNode) + + def MandatoryAttributes(self): + return ['type', 'name', 'file'] + + def DefaultAttributes(self): + return { 'encoding' : 'cp1252', + 'exclude_from_rc' : 'false', + 'line_end' : 'unix', + 'output_encoding' : 'utf-8', + 'generateid': 'true', + 'expand_variables' : 'false', + 'output_filename' : '', + # TODO(joi) this is a hack - should output all generated files + # as SCons dependencies; however, for now there is a bug I can't + # find where GRIT doesn't build the matching fileset, therefore + # this hack so that only the files you really need are marked as + # dependencies. + 'sconsdep' : 'false', + } + + def IsExcludedFromRc(self): + return self.attrs['exclude_from_rc'] == 'true' + + def GetLineEnd(self): + '''Returns the end-of-line character or characters for files output because + of this node ('\r\n', '\n', or '\r' depending on the 'line_end' attribute). + ''' + if self.attrs['line_end'] == 'unix': + return '\n' + elif self.attrs['line_end'] == 'windows': + return '\r\n' + elif self.attrs['line_end'] == 'mac': + return '\r' + else: + raise exception.UnexpectedAttribute( + "Attribute 'line_end' must be one of 'linux' (default), 'windows' or 'mac'") + + def GetCliques(self): + if self.gatherer: + return self.gatherer.GetCliques() + else: + return [] + + def GetTextualIds(self): + if self.gatherer and self.attrs['type'] not in ['tr_html', 'admin_template', 'txt']: + return self.gatherer.GetTextualIds() + else: + return [self.attrs['name']] + + def ItemFormatter(self, t): + if t == 'rc_header': + return grit.format.rc_header.Item() + elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and + self.SatisfiesOutputCondition()): + return _RC_FORMATTERS[self.attrs['type']] + else: + return super(type(self), self).ItemFormatter(t) + + def RunGatherers(self, recursive=False, debug=False): + if self.gatherer: + return # idempotent + + gathertype = _GATHERERS[self.attrs['type']] + + if debug: + print 'Running gatherer %s for file %s' % (str(gathertype), self.FilenameToOpen()) + + self.gatherer = gathertype.FromFile(self.FilenameToOpen(), + self.attrs['name'], + self.attrs['encoding']) + self.gatherer.SetUberClique(self.UberClique()) + self.gatherer.Parse() + + for child in self.children: + assert isinstance(child, variant.SkeletonNode) + skel = gathertype.FromFile(child.FilenameToOpen(), + self.attrs['name'], + child.GetEncodingToUse()) + skel.SetUberClique(self.UberClique()) + skel.SetSkeleton(True) + skel.Parse() + self.skeletons[child.attrs['expr']] = skel + + def GetSkeletonGatherer(self): + '''Returns the gatherer for the alternate skeleton that should be used, + based on the expressions for selecting skeletons, or None if the skeleton + from the English version of the structure should be used. + ''' + for expr in self.skeletons: + if self.EvaluateCondition(expr): + return self.skeletons[expr] + return None + + def GetFilePath(self): + return self.ToRealPath(self.attrs['file']) + + def HasFileForLanguage(self): + return self.attrs['type'] in ['tr_html', 'admin_template', 'txt', 'muppet'] + + def FileForLanguage(self, lang, output_dir, create_file=True, + return_if_not_generated=True): + '''Returns the filename of the file associated with this structure, + for the specified language. + + Args: + lang: 'fr' + output_dir: 'c:\temp' + create_file: True + ''' + assert self.HasFileForLanguage() + if (lang == self.GetRoot().GetSourceLanguage() and + self.attrs['expand_variables'] != 'true'): + if return_if_not_generated: + return self.GetFilePath() + else: + return None + else: + if self.attrs['output_filename'] != '': + filename = self.attrs['output_filename'] + else: + filename = os.path.basename(self.attrs['file']) + assert len(filename) + filename = '%s_%s' % (lang, filename) + filename = os.path.join(output_dir, filename) + + if create_file: + text = self.gatherer.Translate( + lang, + pseudo_if_not_available=self.PseudoIsAllowed(), + fallback_to_english=self.ShouldFallbackToEnglish(), + skeleton_gatherer=self.GetSkeletonGatherer()) + + file_object = util.WrapOutputStream(file(filename, 'wb'), + self._GetOutputEncoding()) + file_contents = util.FixLineEnd(text, self.GetLineEnd()) + if self.attrs['expand_variables'] == 'true': + file_contents = file_contents.replace('[GRITLANGCODE]', lang) + # TODO(jennyz): remove this hard coded logic for expanding + # [GRITDIR] variable for RTL languages when the generic + # expand_variable code is added by grit team. + if lang in _RTL_LANGS : + file_contents = file_contents.replace('[GRITDIR]', 'dir="RTL"') + else : + file_contents = file_contents.replace('[GRITDIR]', 'dir="LTR"') + if self._ShouldAddBom(): + file_object.write(constants.BOM) + file_object.write(file_contents) + file_object.close() + + return filename + + def _GetOutputEncoding(self): + '''Python doesn't natively support UTF encodings with a BOM signature, + so we add support by allowing you to append '-sig' to the encoding name. + This function returns the specified output encoding minus that part. + ''' + enc = self.attrs['output_encoding'] + if enc.endswith('-sig'): + return enc[0:len(enc) - len('-sig')] + else: + return enc + + def _ShouldAddBom(self): + '''Returns true if output files should have the Unicode BOM prepended. + ''' + return self.attrs['output_encoding'].endswith('-sig') + + # static method + def Construct(parent, name, type, file, encoding='cp1252'): + '''Creates a new node which is a child of 'parent', with attributes set + by parameters of the same name. + ''' + node = StructureNode() + node.StartParsing('structure', parent) + node.HandleAttribute('name', name) + node.HandleAttribute('type', type) + node.HandleAttribute('file', file) + node.HandleAttribute('encoding', encoding) + node.EndParsing() + return node + Construct = staticmethod(Construct) diff --git a/tools/grit/grit/node/structure_unittest.py b/tools/grit/grit/node/structure_unittest.py new file mode 100644 index 0000000..5fcd678 --- /dev/null +++ b/tools/grit/grit/node/structure_unittest.py @@ -0,0 +1,86 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for <structure> nodes. +''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest +import StringIO + +from grit.node import structure +from grit import grd_reader +from grit import util + + +class StructureUnittest(unittest.TestCase): + def testSkeleton(self): + grd = grd_reader.Parse(StringIO.StringIO( + '''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" file="klonk.rc" encoding="utf-16-le"> + <skeleton expr="lang == 'fr'" variant_of_revision="1" file="klonk-alternate-skeleton.rc" /> + </structure> + </structures> + </release> + </grit>'''), dir=util.PathFromRoot('grit\\test\\data')) + grd.RunGatherers(recursive=True) + grd.output_language = 'fr' + + node = grd.GetNodeById('IDD_ABOUTBOX') + formatter = node.ItemFormatter('rc_all') + self.failUnless(formatter) + transl = formatter.Format(node, 'fr') + + self.failUnless(transl.count('040704') and transl.count('110978')) + self.failUnless(transl.count('2005",IDC_STATIC')) + + def testOutputEncoding(self): + grd = grd_reader.Parse(StringIO.StringIO( + '''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" file="klonk.rc" encoding="utf-16-le" output_encoding="utf-8-sig" /> + </structures> + </release> + </grit>'''), dir=util.PathFromRoot('grit\\test\\data')) + node = grd.GetNodeById('IDD_ABOUTBOX') + self.failUnless(node._GetOutputEncoding() == 'utf-8') + self.failUnless(node._ShouldAddBom()) + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/node/variant.py b/tools/grit/grit/node/variant.py new file mode 100644 index 0000000..e5da2f9 --- /dev/null +++ b/tools/grit/grit/node/variant.py @@ -0,0 +1,66 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The <skeleton> element. +''' + + +from grit.node import base + + +class SkeletonNode(base.Node): + '''A <skeleton> element.''' + + # TODO(joi) Support inline skeleton variants as CDATA instead of requiring + # a 'file' attribute. + + def MandatoryAttributes(self): + return ['expr', 'variant_of_revision', 'file'] + + def DefaultAttributes(self): + '''If not specified, 'encoding' will actually default to the parent node's + encoding. + ''' + return {'encoding' : ''} + + def _ContentType(self): + if self.attrs.has_key('file'): + return self._CONTENT_TYPE_NONE + else: + return self._CONTENT_TYPE_CDATA + + def GetEncodingToUse(self): + if self.attrs['encoding'] == '': + return self.parent.attrs['encoding'] + else: + return self.attrs['encoding'] + + def GetFilePath(self): + return self.ToRealPath(self.attrs['file']) diff --git a/tools/grit/grit/pseudo.py b/tools/grit/grit/pseudo.py new file mode 100644 index 0000000..53ee9fa --- /dev/null +++ b/tools/grit/grit/pseudo.py @@ -0,0 +1,154 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Pseudotranslation support. Our pseudotranslations are based on the +P-language, which is a simple vowel-extending language. Examples of P: + - "hello" becomes "hepellopo" + - "howdie" becomes "hopowdiepie" + - "because" becomes "bepecaupause" (but in our implementation we don't + handle the silent e at the end so it actually would return "bepecaupausepe" + +The P-language has the excellent quality of increasing the length of text +by around 30-50% which is great for pseudotranslations, to stress test any +GUI layouts etc. + +To make the pseudotranslations more obviously "not a translation" and to make +them exercise any code that deals with encodings, we also transform all English +vowels into equivalent vowels with diacriticals on them (rings, acutes, +diaresis, and circumflex), and we write the "p" in the P-language as a Hebrew +character Qof. It looks sort of like a latin character "p" but it is outside +the latin-1 character set which will stress character encoding bugs. +''' + +import re +import types + +from grit import tclib + + +# An RFC language code for the P pseudolanguage. +PSEUDO_LANG = 'x-P-pseudo' + +# Hebrew character Qof. It looks kind of like a 'p' but is outside +# the latin-1 character set which is good for our purposes. +# TODO(joi) For now using P instead of Qof, because of some bugs it used. Find +# a better solution, i.e. one that introduces a non-latin1 character into the +# pseudotranslation. +#_QOF = u'\u05e7' +_QOF = u'P' + +# How we map each vowel. +_VOWELS = { + u'a' : u'\u00e5', # a with ring + u'e' : u'\u00e9', # e acute + u'i' : u'\u00ef', # i diaresis + u'o' : u'\u00f4', # o circumflex + u'u' : u'\u00fc', # u diaresis + u'y' : u'\u00fd', # y acute + u'A' : u'\u00c5', # A with ring + u'E' : u'\u00c9', # E acute + u'I' : u'\u00cf', # I diaresis + u'O' : u'\u00d4', # O circumflex + u'U' : u'\u00dc', # U diaresis + u'Y' : u'\u00dd', # Y acute +} + +# Matches vowels and P +_PSUB_RE = re.compile("(%s)" % '|'.join(_VOWELS.keys() + ['P'])) + + +# Pseudotranslations previously created. This is important for performance +# reasons, especially since we routinely pseudotranslate the whole project +# several or many different times for each build. +_existing_translations = {} + + +def MapVowels(str, also_p = False): + '''Returns a copy of 'str' where characters that exist as keys in _VOWELS + have been replaced with the corresponding value. If also_p is true, this + function will also change capital P characters into a Hebrew character Qof. + ''' + def Repl(match): + if match.group() == 'p': + if also_p: + return _QOF + else: + return 'p' + else: + return _VOWELS[match.group()] + return _PSUB_RE.sub(Repl, str) + + +def PseudoString(str): + '''Returns a pseudotranslation of the provided string, in our enhanced + P-language.''' + if str in _existing_translations: + return _existing_translations[str] + + outstr = u'' + ix = 0 + while ix < len(str): + if str[ix] not in _VOWELS.keys(): + outstr += str[ix] + ix += 1 + else: + # We want to treat consecutive vowels as one composite vowel. This is not + # always accurate e.g. in composite words but good enough. + consecutive_vowels = u'' + while ix < len(str) and str[ix] in _VOWELS.keys(): + consecutive_vowels += str[ix] + ix += 1 + changed_vowels = MapVowels(consecutive_vowels) + outstr += changed_vowels + outstr += _QOF + outstr += changed_vowels + + _existing_translations[str] = outstr + return outstr + + +def PseudoMessage(message): + '''Returns a pseudotranslation of the provided message. + + Args: + message: tclib.Message() + + Return: + tclib.Translation() + ''' + transl = tclib.Translation() + + for part in message.GetContent(): + if isinstance(part, tclib.Placeholder): + transl.AppendPlaceholder(part) + else: + transl.AppendText(PseudoString(part)) + + return transl diff --git a/tools/grit/grit/pseudo_unittest.py b/tools/grit/grit/pseudo_unittest.py new file mode 100644 index 0000000..6ce5f46 --- /dev/null +++ b/tools/grit/grit/pseudo_unittest.py @@ -0,0 +1,78 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.pseudo''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + +from grit import pseudo +from grit import tclib + + +class PseudoUnittest(unittest.TestCase): + def testVowelMapping(self): + self.failUnless(pseudo.MapVowels('abebibobuby') == + u'\u00e5b\u00e9b\u00efb\u00f4b\u00fcb\u00fd') + self.failUnless(pseudo.MapVowels('ABEBIBOBUBY') == + u'\u00c5B\u00c9B\u00cfB\u00d4B\u00dcB\u00dd') + + def testPseudoString(self): + out = pseudo.PseudoString('hello') + self.failUnless(out == pseudo.MapVowels(u'hePelloPo', True)) + + def testConsecutiveVowels(self): + out = pseudo.PseudoString("beautiful weather, ain't it?") + self.failUnless(out == pseudo.MapVowels( + u"beauPeautiPifuPul weaPeathePer, aiPain't iPit?", 1)) + + def testCapitals(self): + out = pseudo.PseudoString("HOWDIE DOODIE, DR. JONES") + self.failUnless(out == pseudo.MapVowels( + u"HOPOWDIEPIE DOOPOODIEPIE, DR. JOPONEPES", 1)) + + def testPseudoMessage(self): + msg = tclib.Message(text='Hello USERNAME, how are you?', + placeholders=[ + tclib.Placeholder('USERNAME', '%s', 'Joi')]) + trans = pseudo.PseudoMessage(msg) + # TODO(joi) It would be nicer if 'you' -> 'youPou' instead of + # 'you' -> 'youPyou' and if we handled the silent e in 'are' + self.failUnless(trans.GetPresentableContent() == + pseudo.MapVowels( + u'HePelloPo USERNAME, hoPow aParePe youPyou?', 1)) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/scons.py b/tools/grit/grit/scons.py new file mode 100644 index 0000000..1b960a6 --- /dev/null +++ b/tools/grit/grit/scons.py @@ -0,0 +1,167 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''SCons integration for GRIT. +''' + +# NOTE: DO NOT IMPORT ANY GRIT STUFF HERE - we import lazily so that grit and +# its dependencies aren't imported until actually needed. + +import os +import types + +def _IsDebugEnabled(): + return 'GRIT_DEBUG' in os.environ and os.environ['GRIT_DEBUG'] == '1' + +def _SourceToFile(source): + '''Return the path to the source file, given the 'source' argument as provided + by SCons to the _Builder or _Emitter functions. + ''' + # Get the filename of the source. The 'source' parameter can be a string, + # a "node", or a list of strings or nodes. + if isinstance(source, types.ListType): + source = str(source[0]) + else: + source = str(source) + return source + + +def _Builder(target, source, env): + from grit import grit_runner + from grit.tool import build + options = grit_runner.Options() + # This sets options to default values TODO(joi) Remove verbose + options.ReadOptions(['-v']) + options.input = _SourceToFile(source) + + # TODO(joi) Check if we can get the 'verbose' option from the environment. + + builder = build.RcBuilder() + + # Get the CPP defines from the environment. + for flag in env['RCFLAGS']: + if flag.startswith('/D'): + flag = flag[2:] + name, val = build.ParseDefine(flag) + # Only apply to first instance of a given define + if name not in builder.defines: + builder.defines[name] = val + + # To ensure that our output files match what we promised SCons, we + # use the list of targets provided by SCons and update the file paths in + # our .grd input file with the targets. + builder.scons_targets = [str(t) for t in target] + builder.Run(options, []) + return None # success + + +def _Emitter(target, source, env): + '''A SCons emitter for .grd files, which modifies the list of targes to + include all files in the <outputs> section of the .grd file as well as + any other files output by 'grit build' for the .grd file. + ''' + from grit import util + from grit import grd_reader + + base_dir = util.dirname(str(target[0])) + + grd = grd_reader.Parse(_SourceToFile(source), debug=_IsDebugEnabled()) + + target = [] + lang_folders = {} + # Add all explicitly-specified output files + for output in grd.GetOutputFiles(): + path = os.path.join(base_dir, output.GetFilename()) + target.append(path) + if _IsDebugEnabled(): + print "GRIT: Added target %s" % path + if output.attrs['lang'] != '': + lang_folders[output.attrs['lang']] = os.path.dirname(path) + + # Add all generated files, once for each output language. + for node in grd: + if node.name == 'structure': + # TODO(joi) Should remove the "if sconsdep is true" thing as it is a + # hack - see grit/node/structure.py + if node.HasFileForLanguage() and node.attrs['sconsdep'] == 'true': + for lang in lang_folders: + path = node.FileForLanguage(lang, lang_folders[lang], + create_file=False, + return_if_not_generated=False) + if path: + target.append(path) + if _IsDebugEnabled(): + print "GRIT: Added target %s" % path + + # return target and source lists + return (target, source) + + +def _Scanner(file_node, env, path): + '''A SCons scanner function for .grd files, which outputs the list of files + that changes in could change the output of building the .grd file. + ''' + from grit import grd_reader + + grd = grd_reader.Parse(str(file_node), debug=_IsDebugEnabled()) + files = [] + for node in grd: + if (node.name == 'structure' or node.name == 'skeleton' or + (node.name == 'file' and node.parent and + node.parent.name == 'translations')): + files.append(os.path.abspath(node.GetFilePath())) + return files + + +# Function name is mandated by newer versions of SCons. +def generate(env): + # Importing this module should be possible whenever this function is invoked + # since it should only be invoked by SCons. + import SCons.Builder + import SCons.Action + + # The varlist parameter tells SCons that GRIT needs to be invoked again + # if RCFLAGS has changed since last compilation. + action = SCons.Action.FunctionAction(_Builder, varlist=['RCFLAGS']) + + builder = SCons.Builder.Builder(action=action, + emitter=_Emitter, + src_suffix='.grd') + + scanner = env.Scanner(function=_Scanner, name='GRIT', skeys=['.grd']) + + # add our builder and scanner to the environment + env.Append(BUILDERS = {'GRIT': builder}) + env.Prepend(SCANNERS = scanner) + + +# Function name is mandated by newer versions of SCons. +def exists(env): + return 1 diff --git a/tools/grit/grit/shortcuts.py b/tools/grit/grit/shortcuts.py new file mode 100644 index 0000000..43da40b --- /dev/null +++ b/tools/grit/grit/shortcuts.py @@ -0,0 +1,119 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Stuff to prevent conflicting shortcuts. +''' + +import re + +from grit import util + + +class ShortcutGroup(object): + '''Manages a list of cliques that belong together in a single shortcut + group. Knows how to detect conflicting shortcut keys. + ''' + + # Matches shortcut keys, e.g. &J + SHORTCUT_RE = re.compile('([^&]|^)(&[A-Za-z])') + + def __init__(self, name): + self.name = name + # Map of language codes to shortcut keys used (which is a map of + # shortcut keys to counts). + self.keys_by_lang = {} + # List of cliques in this group + self.cliques = [] + + def AddClique(self, c): + for existing_clique in self.cliques: + if existing_clique.GetId() == c.GetId(): + # This happens e.g. when we have e.g. + # <if expr1><structure 1></if> <if expr2><structure 2></if> + # where only one will really be included in the output. + return + + self.cliques.append(c) + for (lang, msg) in c.clique.items(): + if lang not in self.keys_by_lang: + self.keys_by_lang[lang] = {} + keymap = self.keys_by_lang[lang] + + content = msg.GetRealContent() + keys = [groups[1] for groups in self.SHORTCUT_RE.findall(content)] + for key in keys: + key = key.upper() + if key in keymap: + keymap[key] += 1 + else: + keymap[key] = 1 + + def GenerateWarnings(self, tc_project): + # For any language that has more than one occurrence of any shortcut, + # make a list of the conflicting shortcuts. + problem_langs = {} + for (lang, keys) in self.keys_by_lang.items(): + for (key, count) in keys.items(): + if count > 1: + if lang not in problem_langs: + problem_langs[lang] = [] + problem_langs[lang].append(key) + + warnings = [] + if len(problem_langs): + warnings.append("WARNING - duplicate keys exist in shortcut group %s" % + self.name) + for (lang,keys) in problem_langs.items(): + warnings.append(" %6s duplicates: %s" % (lang, ', '.join(keys))) + return warnings + + +def GenerateDuplicateShortcutsWarnings(uberclique, tc_project): + '''Given an UberClique and a project name, will print out helpful warnings + if there are conflicting shortcuts within shortcut groups in the provided + UberClique. + + Args: + uberclique: clique.UberClique() + tc_project: 'MyProjectNameInTheTranslationConsole' + + Returns: + ['warning line 1', 'warning line 2', ...] + ''' + warnings = [] + groups = {} + for c in uberclique.AllCliques(): + for group in c.shortcut_groups: + if group not in groups: + groups[group] = ShortcutGroup(group) + groups[group].AddClique(c) + for group in groups.values(): + warnings += group.GenerateWarnings(tc_project) + return warnings diff --git a/tools/grit/grit/shortcuts_unittests.py b/tools/grit/grit/shortcuts_unittests.py new file mode 100644 index 0000000..634177d --- /dev/null +++ b/tools/grit/grit/shortcuts_unittests.py @@ -0,0 +1,103 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.shortcuts +''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + +from grit import shortcuts +from grit import clique +from grit import tclib +from grit.gather import rc + +class ShortcutsUnittest(unittest.TestCase): + + def setUp(self): + self.uq = clique.UberClique() + + def testFunctionality(self): + c = self.uq.MakeClique(tclib.Message(text="Hello &there")) + c.AddToShortcutGroup('group_name') + c = self.uq.MakeClique(tclib.Message(text="Howdie &there partner")) + c.AddToShortcutGroup('group_name') + + warnings = shortcuts.GenerateDuplicateShortcutsWarnings(self.uq, 'PROJECT') + self.failUnless(warnings) + + def testAmpersandEscaping(self): + c = self.uq.MakeClique(tclib.Message(text="Hello &there")) + c.AddToShortcutGroup('group_name') + c = self.uq.MakeClique(tclib.Message(text="S&&T are the &letters S and T")) + c.AddToShortcutGroup('group_name') + + warnings = shortcuts.GenerateDuplicateShortcutsWarnings(self.uq, 'PROJECT') + self.failUnless(len(warnings) == 0) + + def testDialog(self): + dlg = rc.Dialog('''\ +IDD_SIDEBAR_RSS_PANEL_PROPPAGE DIALOGEX 0, 0, 239, 221 +STYLE DS_SETFONT | DS_FIXEDSYS | WS_CHILD +FONT 8, "MS Shell Dlg", 400, 0, 0x1 +BEGIN + PUSHBUTTON "Add &URL",IDC_SIDEBAR_RSS_ADD_URL,182,53,57,14 + EDITTEXT IDC_SIDEBAR_RSS_NEW_URL,0,53,178,15,ES_AUTOHSCROLL + PUSHBUTTON "&Remove",IDC_SIDEBAR_RSS_REMOVE,183,200,56,14 + PUSHBUTTON "&Edit",IDC_SIDEBAR_RSS_EDIT,123,200,56,14 + CONTROL "&Automatically add commonly viewed clips", + IDC_SIDEBAR_RSS_AUTO_ADD,"Button",BS_AUTOCHECKBOX | + BS_MULTILINE | WS_TABSTOP,0,200,120,17 + PUSHBUTTON "",IDC_SIDEBAR_RSS_HIDDEN,179,208,6,6,NOT WS_VISIBLE + LTEXT "You can display clips from blogs, news sites, and other online sources.", + IDC_STATIC,0,0,239,10 + LISTBOX IDC_SIDEBAR_DISPLAYED_FEED_LIST,0,69,239,127,LBS_SORT | + LBS_OWNERDRAWFIXED | LBS_HASSTRINGS | + LBS_NOINTEGRALHEIGHT | WS_VSCROLL | WS_HSCROLL | + WS_TABSTOP + LTEXT "Add a clip from a recently viewed website by clicking Add Recent Clips.", + IDC_STATIC,0,13,141,19 + LTEXT "Or, if you know a site supports RSS or Atom, you can enter the RSS or Atom URL below and add it to your list of Web Clips.", + IDC_STATIC,0,33,239,18 + PUSHBUTTON "Add Recent &Clips (10)...", + IDC_SIDEBAR_RSS_ADD_RECENT_CLIPS,146,14,93,14 +END''') + dlg.SetUberClique(self.uq) + dlg.Parse() + + warnings = shortcuts.GenerateDuplicateShortcutsWarnings(self.uq, 'PROJECT') + self.failUnless(len(warnings) == 0) + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/tclib.py b/tools/grit/grit/tclib.py new file mode 100644 index 0000000..0623a5a --- /dev/null +++ b/tools/grit/grit/tclib.py @@ -0,0 +1,233 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Adaptation of the extern.tclib classes for our needs. +''' + + +import re +import types + +from grit import exception +import grit.extern.tclib + +def Identity(i): + return i + + +class BaseMessage(object): + '''Base class with methods shared by Message and Translation. + ''' + + def __init__(self, text='', placeholders=[], description='', meaning=''): + self.parts = [] + self.placeholders = [] + self.description = description + self.meaning = meaning + self.dirty = True # True if self.id is (or might be) wrong + self.id = 0 + + if text != '': + if not placeholders or placeholders == []: + self.AppendText(text) + else: + tag_map = {} + for placeholder in placeholders: + tag_map[placeholder.GetPresentation()] = [placeholder, 0] + tag_re = '(' + '|'.join(tag_map.keys()) + ')' + # This creates a regexp like '(TAG1|TAG2|TAG3)' + chunked_text = re.split(tag_re, text) + for chunk in chunked_text: + if chunk: # ignore empty chunk + if tag_map.has_key(chunk): + self.AppendPlaceholder(tag_map[chunk][0]) + tag_map[chunk][1] += 1 # increase placeholder use count + else: + self.AppendText(chunk) + for key in tag_map.keys(): + assert tag_map[key][1] != 0 + + def GetRealContent(self, escaping_function=Identity): + '''Returns the original content, i.e. what your application and users + will see. + + Specify a function to escape each translateable bit, if you like. + ''' + bits = [] + for item in self.parts: + if isinstance(item, types.StringTypes): + bits.append(escaping_function(item)) + else: + bits.append(item.GetOriginal()) + return ''.join(bits) + + def GetPresentableContent(self): + presentable_content = [] + for part in self.parts: + if isinstance(part, Placeholder): + presentable_content.append(part.GetPresentation()) + else: + presentable_content.append(part) + return ''.join(presentable_content) + + def AppendPlaceholder(self, placeholder): + assert isinstance(placeholder, Placeholder) + dup = False + for other in self.GetPlaceholders(): + if (other.presentation.find(placeholder.presentation) != -1 or + placeholder.presentation.find(other.presentation) != -1): + assert(False, "Placeholder names must be unique and must not overlap") + if other.presentation == placeholder.presentation: + assert other.original == placeholder.original + dup = True + + if not dup: + self.placeholders.append(placeholder) + self.parts.append(placeholder) + self.dirty = True + + def AppendText(self, text): + assert isinstance(text, types.StringTypes) + assert text != '' + + self.parts.append(text) + self.dirty = True + + def GetContent(self): + '''Returns the parts of the message. You may modify parts if you wish. + Note that you must not call GetId() on this object until you have finished + modifying the contents. + ''' + self.dirty = True # user might modify content + return self.parts + + def GetDescription(self): + return self.description + + def SetDescription(self, description): + self.description = description + + def GetMeaning(self): + return self.meaning + + def GetId(self): + if self.dirty: + self.id = self.GenerateId() + self.dirty = False + return self.id + + def GenerateId(self): + # Must use a UTF-8 encoded version of the presentable content, along with + # the meaning attribute, to match the TC. + return grit.extern.tclib.GenerateMessageId( + self.GetPresentableContent().encode('utf-8'), self.meaning) + + def GetPlaceholders(self): + return self.placeholders + + def FillTclibBaseMessage(self, msg): + msg.SetDescription(self.description.encode('utf-8')) + + for part in self.parts: + if isinstance(part, Placeholder): + ph = grit.extern.tclib.Placeholder( + part.presentation.encode('utf-8'), + part.original.encode('utf-8'), + part.example.encode('utf-8')) + msg.AppendPlaceholder(ph) + else: + msg.AppendText(part.encode('utf-8')) + + +class Message(BaseMessage): + '''A message.''' + + def __init__(self, text='', placeholders=[], description='', meaning=''): + BaseMessage.__init__(self, text, placeholders, description, meaning) + + def ToTclibMessage(self): + msg = grit.extern.tclib.Message('utf-8', meaning=self.meaning) + self.FillTclibBaseMessage(msg) + return msg + +class Translation(BaseMessage): + '''A translation.''' + + def __init__(self, text='', id='', placeholders=[], description='', meaning=''): + BaseMessage.__init__(self, text, placeholders, description, meaning) + self.id = id + + def GetId(self): + assert id != '', "ID has not been set." + return self.id + + def SetId(self, id): + self.id = id + + def ToTclibMessage(self): + msg = grit.extern.tclib.Message( + 'utf-8', id=self.id, meaning=self.meaning) + self.FillTclibBaseMessage(msg) + return msg + + +class Placeholder(grit.extern.tclib.Placeholder): + '''Modifies constructor to accept a Unicode string + ''' + + # Must match placeholder presentation names + _NAME_RE = re.compile('[A-Za-z0-9_]+') + + def __init__(self, presentation, original, example): + '''Creates a new placeholder. + + Args: + presentation: 'USERNAME' + original: '%s' + example: 'Joi' + ''' + assert presentation != '' + assert original != '' + assert example != '' + if not self._NAME_RE.match(presentation): + raise exception.InvalidPlaceholderName(presentation) + self.presentation = presentation + self.original = original + self.example = example + + def GetPresentation(self): + return self.presentation + + def GetOriginal(self): + return self.original + + def GetExample(self): + return self.example + diff --git a/tools/grit/grit/tclib_unittest.py b/tools/grit/grit/tclib_unittest.py new file mode 100644 index 0000000..c73dd4d --- /dev/null +++ b/tools/grit/grit/tclib_unittest.py @@ -0,0 +1,189 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.tclib''' + + +import sys +import os.path +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import types +import unittest + +from grit import tclib + +import grit.extern.tclib + + +class TclibUnittest(unittest.TestCase): + def testInit(self): + msg = tclib.Message(text=u'Hello Earthlings') + self.failUnless(msg.GetPresentableContent() == 'Hello Earthlings') + self.failUnless(isinstance(msg.GetPresentableContent(), types.StringTypes)) + + def testGetAttr(self): + msg = tclib.Message() + msg.AppendText(u'Hello') # Tests __getattr__ + self.failUnless(msg.GetPresentableContent() == 'Hello') + self.failUnless(isinstance(msg.GetPresentableContent(), types.StringTypes)) + + def testAll(self): + text = u'Howdie USERNAME' + phs = [tclib.Placeholder(u'USERNAME', u'%s', 'Joi')] + msg = tclib.Message(text=text, placeholders=phs) + self.failUnless(msg.GetPresentableContent() == 'Howdie USERNAME') + + trans = tclib.Translation(text=text, placeholders=phs) + self.failUnless(trans.GetPresentableContent() == 'Howdie USERNAME') + self.failUnless(isinstance(trans.GetPresentableContent(), types.StringTypes)) + + def testUnicodeReturn(self): + text = u'\u00fe' + msg = tclib.Message(text=text) + self.failUnless(msg.GetPresentableContent() == text) + from_list = msg.GetContent()[0] + self.failUnless(from_list == text) + + def testRegressionTranslationInherited(self): + '''Regression tests a bug that was caused by grit.tclib.Translation + inheriting from the translation console's Translation object + instead of only owning an instance of it. + ''' + msg = tclib.Message(text=u"BLA1\r\nFrom: BLA2 \u00fe BLA3", + placeholders=[ + tclib.Placeholder('BLA1', '%s', '%s'), + tclib.Placeholder('BLA2', '%s', '%s'), + tclib.Placeholder('BLA3', '%s', '%s')]) + transl = tclib.Translation(text=msg.GetPresentableContent(), + placeholders=msg.GetPlaceholders()) + content = transl.GetContent() + self.failUnless(isinstance(content[3], types.UnicodeType)) + + def testFingerprint(self): + # This has Windows line endings. That is on purpose. + id = grit.extern.tclib.GenerateMessageId( + 'Google Desktop for Enterprise\r\n' + 'Copyright (C) 2006 Google Inc.\r\n' + 'All Rights Reserved\r\n' + '\r\n' + '---------\r\n' + 'Contents\r\n' + '---------\r\n' + 'This distribution contains the following files:\r\n' + '\r\n' + 'GoogleDesktopSetup.msi - Installation and setup program\r\n' + 'GoogleDesktop.adm - Group Policy administrative template file\r\n' + 'AdminGuide.pdf - Google Desktop for Enterprise administrative guide\r\n' + '\r\n' + '\r\n' + '--------------\r\n' + 'Documentation\r\n' + '--------------\r\n' + 'Full documentation and installation instructions are in the \r\n' + 'administrative guide, and also online at \r\n' + 'http://desktop.google.com/enterprise/adminguide.html.\r\n' + '\r\n' + '\r\n' + '------------------------\r\n' + 'IBM Lotus Notes Plug-In\r\n' + '------------------------\r\n' + 'The Lotus Notes plug-in is included in the release of Google \r\n' + 'Desktop for Enterprise. The IBM Lotus Notes Plug-in for Google \r\n' + 'Desktop indexes mail, calendar, task, contact and journal \r\n' + 'documents from Notes. Discussion documents including those from \r\n' + 'the discussion and team room templates can also be indexed by \r\n' + 'selecting an option from the preferences. Once indexed, this data\r\n' + 'will be returned in Google Desktop searches. The corresponding\r\n' + 'document can be opened in Lotus Notes from the Google Desktop \r\n' + 'results page.\r\n' + '\r\n' + 'Install: The plug-in will install automatically during the Google \r\n' + 'Desktop setup process if Lotus Notes is already installed. Lotus \r\n' + 'Notes must not be running in order for the install to occur. \r\n' + '\r\n' + 'Preferences: Preferences and selection of databases to index are\r\n' + 'set in the \'Google Desktop for Notes\' dialog reached through the \r\n' + '\'Actions\' menu.\r\n' + '\r\n' + 'Reindexing: Selecting \'Reindex all databases\' will index all the \r\n' + 'documents in each database again.\r\n' + '\r\n' + '\r\n' + 'Notes Plug-in Known Issues\r\n' + '---------------------------\r\n' + '\r\n' + 'If the \'Google Desktop for Notes\' item is not available from the \r\n' + 'Lotus Notes Actions menu, then installation was not successful. \r\n' + 'Installation consists of writing one file, notesgdsplugin.dll, to \r\n' + 'the Notes application directory and a setting to the notes.ini \r\n' + 'configuration file. The most likely cause of an unsuccessful \r\n' + 'installation is that the installer was not able to locate the \r\n' + 'notes.ini file. Installation will complete if the user closes Notes\r\n' + 'and manually adds the following setting to this file on a new line:\r\n' + 'AddinMenus=notegdsplugin.dll\r\n' + '\r\n' + 'If the notesgdsplugin.dll file is not in the application directory\r\n' + '(e.g., C:\Program Files\Lotus\Notes) after Google Desktop \r\n' + 'installation, it is likely that Notes was not installed correctly. \r\n' + '\r\n' + 'Only local databases can be indexed. If they can be determined, \r\n' + 'the user\'s local mail file and address book will be included in the\r\n' + 'list automatically. Mail archives and other databases must be \r\n' + 'added with the \'Add\' button.\r\n' + '\r\n' + 'Some users may experience performance issues during the initial \r\n' + 'indexing of a database. The \'Perform the initial index of a \r\n' + 'database only when I\'m idle\' option will limit the indexing process\r\n' + 'to times when the user is not using the machine. If this does not \r\n' + 'alleviate the problem or the user would like to continually index \r\n' + 'but just do so more slowly or quickly, the GoogleWaitTime notes.ini\r\n' + 'value can be set. Increasing the GoogleWaitTime value will slow \r\n' + 'down the indexing process, and lowering the value will speed it up.\r\n' + 'A value of zero causes the fastest possible indexing. Removing the\r\n' + 'ini parameter altogether returns it to the default (20).\r\n' + '\r\n' + 'Crashes have been known to occur with certain types of history \r\n' + 'bookmarks. If the Notes client seems to crash randomly, try \r\n' + 'disabling the \'Index note history\' option. If it crashes before,\r\n' + 'you can get to the preferences, add the following line to your \r\n' + 'notes.ini file:\r\n' + 'GDSNoIndexHistory=1\r\n') + self.failUnless(id == '8961534701379422820') + + def testPlaceholderNameChecking(self): + try: + ph = tclib.Placeholder('BINGO BONGO', 'bla', 'bla') + except exception.InvalidPlaceholderName: + pass # Expect exception to be thrown because presentation contained space + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/test_suite_all.py b/tools/grit/grit/test_suite_all.py new file mode 100644 index 0000000..c102102 --- /dev/null +++ b/tools/grit/grit/test_suite_all.py @@ -0,0 +1,107 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit test suite that collects all test cases for GRIT.''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + + +# TODO(joi) Use unittest.defaultTestLoader to automatically load tests +# from modules. Iterating over the directory and importing could then +# automate this all the way, if desired. + + +class TestSuiteAll(unittest.TestSuite): + def __init__(self): + super(type(self), self).__init__() + # Imports placed here to prevent circular imports. + from grit import grd_reader_unittest + from grit import grit_runner_unittest + from grit.node import base_unittest + from grit.node import io_unittest + from grit import clique_unittest + from grit.node import misc_unittest + from grit.gather import rc_unittest + from grit.gather import tr_html_unittest + from grit.node import message_unittest + from grit import tclib_unittest + import grit.format.rc_unittest + from grit.tool import rc2grd_unittest + from grit.tool import transl2tc_unittest + from grit.gather import txt_unittest + from grit.gather import admin_template_unittest + from grit import xtb_reader_unittest + from grit import util_unittest + from grit.tool import preprocess_unittest + from grit.tool import postprocess_unittest + from grit import shortcuts_unittests + from grit.gather import muppet_strings_unittest + from grit.node.custom import filename_unittest + + test_classes = [ + base_unittest.NodeUnittest, + io_unittest.FileNodeUnittest, + grit_runner_unittest.OptionArgsUnittest, + grd_reader_unittest.GrdReaderUnittest, + clique_unittest.MessageCliqueUnittest, + misc_unittest.GritNodeUnittest, + rc_unittest.RcUnittest, + tr_html_unittest.ParserUnittest, + tr_html_unittest.TrHtmlUnittest, + message_unittest.MessageUnittest, + tclib_unittest.TclibUnittest, + grit.format.rc_unittest.FormatRcUnittest, + rc2grd_unittest.Rc2GrdUnittest, + transl2tc_unittest.TranslationToTcUnittest, + txt_unittest.TxtUnittest, + admin_template_unittest.AdmGathererUnittest, + xtb_reader_unittest.XtbReaderUnittest, + misc_unittest.IfNodeUnittest, + util_unittest.UtilUnittest, + preprocess_unittest.PreProcessingUnittest, + postprocess_unittest.PostProcessingUnittest, + misc_unittest.ReleaseNodeUnittest, + shortcuts_unittests.ShortcutsUnittest, + muppet_strings_unittest.MuppetStringsUnittest, + filename_unittest.WindowsFilenameUnittest, + # add test classes here... + ] + + for test_class in test_classes: + self.addTest(unittest.makeSuite(test_class)) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(TestSuiteAll()) diff --git a/tools/grit/grit/tool/__init__.py b/tools/grit/grit/tool/__init__.py new file mode 100644 index 0000000..378cbaa --- /dev/null +++ b/tools/grit/grit/tool/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Package grit.tool +''' + +pass diff --git a/tools/grit/grit/tool/build.py b/tools/grit/grit/tool/build.py new file mode 100644 index 0000000..9059af4 --- /dev/null +++ b/tools/grit/grit/tool/build.py @@ -0,0 +1,232 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit build' tool along with integration for this tool with the +SCons build system. +''' + +import os +import getopt +import types +import sys + +from grit import grd_reader +from grit import util +from grit.tool import interface +from grit import shortcuts + + +def ParseDefine(define): + '''Parses a define that is either like "NAME" or "NAME=VAL" and + returns its components, using True as the default value. Values of + "1" and "0" are transformed to True and False respectively. + ''' + parts = [part.strip() for part in define.split('=')] + assert len(parts) >= 1 + name = parts[0] + val = True + if len(parts) > 1: + val = parts[1] + if val == "1": val = True + elif val == "0": val = False + return (name, val) + + +class RcBuilder(interface.Tool): + '''A tool that builds RC files and resource header files for compilation. + +Usage: grit build [-o OUTPUTDIR] [-D NAME[=VAL]]* + +All output options for this tool are specified in the input file (see +'grit help' for details on how to specify the input file - it is a global +option). + +Options: + + -o OUTPUTDIR Specify what directory output paths are relative to. + Defaults to the current directory. + + -D NAME[=VAL] Specify a C-preprocessor-like define NAME with optional + value VAL (defaults to 1) which will be used to control + conditional inclusion of resources. + +Conditional inclusion of resources only affects the output of files which +control which resources get linked into a binary, e.g. it affects .rc files +meant for compilation but it does not affect resource header files (that define +IDs). This helps ensure that values of IDs stay the same, that all messages +are exported to translation interchange files (e.g. XMB files), etc. +''' + + def ShortDescription(self): + return 'A tool that builds RC files for compilation.' + + def Run(self, opts, args): + self.output_directory = '.' + (own_opts, args) = getopt.getopt(args, 'o:D:') + for (key, val) in own_opts: + if key == '-o': + self.output_directory = val + elif key == '-D': + name, val = ParseDefine(val) + self.defines[name] = val + if len(args): + print "This tool takes no tool-specific arguments." + return 2 + self.SetOptions(opts) + if self.scons_targets: + self.VerboseOut('Using SCons targets to identify files to output.\n') + else: + self.VerboseOut('Output directory: %s (absolute path: %s)\n' % + (self.output_directory, + os.path.abspath(self.output_directory))) + self.res = grd_reader.Parse(opts.input, debug=opts.extra_verbose) + self.res.RunGatherers(recursive = True) + self.Process() + return 0 + + def __init__(self): + # Default file-creation function is built-in file(). Only done to allow + # overriding by unit test. + self.fo_create = file + + # key/value pairs of C-preprocessor like defines that are used for + # conditional output of resources + self.defines = {} + + # self.res is a fully-populated resource tree if Run() + # has been called, otherwise None. + self.res = None + + # Set to a list of filenames for the output nodes that are relative + # to the current working directory. They are in the same order as the + # output nodes in the file. + self.scons_targets = None + + # static method + def ProcessNode(node, output_node, outfile): + '''Processes a node in-order, calling its formatter before and after + recursing to its children. + + Args: + node: grit.node.base.Node subclass + output_node: grit.node.io.File + outfile: open filehandle + ''' + base_dir = util.dirname(output_node.GetOutputFilename()) + + try: + formatter = node.ItemFormatter(output_node.GetType()) + if formatter: + outfile.write(formatter.Format(node, output_node.GetLanguage(), + begin_item=True, output_dir=base_dir)) + except: + print u"Error processing node %s" % unicode(node) + raise + + for child in node.children: + RcBuilder.ProcessNode(child, output_node, outfile) + + try: + if formatter: + outfile.write(formatter.Format(node, output_node.GetLanguage(), + begin_item=False, output_dir=base_dir)) + except: + print u"Error processing node %s" % unicode(node) + raise + ProcessNode = staticmethod(ProcessNode) + + + def Process(self): + # Update filenames with those provided by SCons if we're being invoked + # from SCons. The list of SCons targets also includes all <structure> + # node outputs, but it starts with our output files, in the order they + # occur in the .grd + if self.scons_targets: + assert len(self.scons_targets) >= len(self.res.GetOutputFiles()) + outfiles = self.res.GetOutputFiles() + for ix in range(len(outfiles)): + outfiles[ix].output_filename = os.path.abspath( + self.scons_targets[ix]) + else: + for output in self.res.GetOutputFiles(): + output.output_filename = os.path.abspath(os.path.join( + self.output_directory, output.GetFilename())) + + for output in self.res.GetOutputFiles(): + self.VerboseOut('Creating %s...' % output.GetFilename()) + # Microsoft's RC compiler can only deal with single-byte or double-byte + # files (no UTF-8), so we make all RC files UTF-16 to support all + # character sets. + if output.GetType() in ['rc_header']: + encoding = 'cp1252' + outname = output.GetOutputFilename() + oldname = outname + '.tmp' + if os.access(oldname, os.F_OK): + os.remove(oldname) + try: + os.rename(outname, oldname) + except OSError: + oldname = None + else: + encoding = 'utf_16' + outfile = util.WrapOutputStream( + self.fo_create(output.GetOutputFilename(), 'wb'), + encoding) + + # Set the context, for conditional inclusion of resources + self.res.SetOutputContext(output.GetLanguage(), self.defines) + + # TODO(joi) Handle this more gracefully + import grit.format.rc_header + grit.format.rc_header.Item.ids_ = {} + + # Iterate in-order through entire resource tree, calling formatters on + # the entry into a node and on exit out of it. + self.ProcessNode(self.res, output, outfile) + + outfile.close() + if output.GetType() in ['rc_header'] and oldname: + if open(oldname).read() != open(outname).read(): + os.remove(oldname) + else: + os.remove(outname) + os.rename(oldname, outname) + self.VerboseOut(' done.\n') + + # Print warnings if there are any duplicate shortcuts. + print '\n'.join(shortcuts.GenerateDuplicateShortcutsWarnings( + self.res.UberClique(), self.res.GetTcProject())) + + # Print out any fallback warnings, and missing translation errors, and + # exit with an error code if there are missing translations in a non-pseudo + # build + print self.res.UberClique().MissingTranslationsReport() + if self.res.UberClique().HasMissingTranslations(): + sys.exit(-1) diff --git a/tools/grit/grit/tool/count.py b/tools/grit/grit/tool/count.py new file mode 100644 index 0000000..c5ede8c --- /dev/null +++ b/tools/grit/grit/tool/count.py @@ -0,0 +1,68 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Count number of occurrences of a given message ID +''' + +import getopt +import os +import types + +from grit.tool import interface +from grit import grd_reader +from grit import util + +from grit.extern import tclib + + +class CountMessage(interface.Tool): + '''Count the number of times a given message ID is used. +''' + + def __init__(self): + pass + + def ShortDescription(self): + return 'Exports all translateable messages into an XMB file.' + + def Run(self, opts, args): + self.SetOptions(opts) + + id = args[0] + res_tree = grd_reader.Parse(opts.input, debug=opts.extra_verbose) + res_tree.OnlyTheseTranslations([]) + res_tree.RunGatherers(True) + + count = 0 + for c in res_tree.UberClique().AllCliques(): + if c.GetId() == id: + count += 1 + + print "There are %d occurrences of message %s." % (count, id) diff --git a/tools/grit/grit/tool/diff_structures.py b/tools/grit/grit/tool/diff_structures.py new file mode 100644 index 0000000..bef5d56 --- /dev/null +++ b/tools/grit/grit/tool/diff_structures.py @@ -0,0 +1,139 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit sdiff' tool. +''' + +import os +import getopt +import tempfile + +from grit.node import structure +from grit.tool import interface + +from grit import constants +from grit import util + +# Builds the description for the tool (used as the __doc__ +# for the DiffStructures class). +_class_doc = """\ +Allows you to view the differences in the structure of two files, +disregarding their translateable content. Translateable portions of +each file are changed to the string "TTTTTT" before invoking the diff program +specified by the P4DIFF environment variable. + +Usage: grit sdiff [-t TYPE] [-s SECTION] [-e ENCODING] LEFT RIGHT + +LEFT and RIGHT are the files you want to diff. SECTION is required +for structure types like 'dialog' to identify the part of the file to look at. +ENCODING indicates the encoding of the left and right files (default 'cp1252'). +TYPE can be one of the following, defaults to 'tr_html': +""" +for gatherer in structure._GATHERERS: + _class_doc += " - %s\n" % gatherer + + +class DiffStructures(interface.Tool): + __doc__ = _class_doc + + def __init__(self): + self.section = None + self.left_encoding = 'cp1252' + self.right_encoding = 'cp1252' + self.structure_type = 'tr_html' + + def ShortDescription(self): + return 'View differences without regard for translateable portions.' + + def Run(self, global_opts, args): + (opts, args) = getopt.getopt(args, 's:e:t:', + ['left_encoding=', 'right_encoding=']) + for key, val in opts: + if key == '-s': + self.section = val + elif key == '-e': + self.left_encoding = val + self.right_encoding = val + elif key == '-t': + self.structure_type = val + elif key == '--left_encoding': + self.left_encoding = val + elif key == '--right_encoding': + self.right_encoding == val + + if len(args) != 2: + print "Incorrect usage - 'grit help sdiff' for usage details." + return 2 + + if 'P4DIFF' not in os.environ: + print "Environment variable P4DIFF not set; defaulting to 'windiff'." + diff_program = 'windiff' + else: + diff_program = os.environ['P4DIFF'] + + left_trans = self.MakeStaticTranslation(args[0], self.left_encoding) + try: + try: + right_trans = self.MakeStaticTranslation(args[1], self.right_encoding) + + os.system('%s %s %s' % (diff_program, left_trans, right_trans)) + finally: + os.unlink(right_trans) + finally: + os.unlink(left_trans) + + def MakeStaticTranslation(self, original_filename, encoding): + """Given the name of the structure type (self.structure_type), the filename + of the file holding the original structure, and optionally the "section" key + identifying the part of the file to look at (self.section), creates a + temporary file holding a "static" translation of the original structure + (i.e. one where all translateable parts have been replaced with "TTTTTT") + and returns the temporary file name. It is the caller's responsibility to + delete the file when finished. + + Args: + original_filename: 'c:\\bingo\\bla.rc' + + Return: + 'c:\\temp\\werlkjsdf334.tmp' + """ + original = structure._GATHERERS[self.structure_type].FromFile( + original_filename, extkey=self.section, encoding=encoding) + original.Parse() + translated = original.Translate(constants.CONSTANT_LANGUAGE, False) + + fname = tempfile.mktemp() + fhandle = file(fname, 'w') + writer = util.WrapOutputStream(fhandle) + writer.write("Original filename: %s\n=============\n\n" % original_filename) + writer.write(translated) # write in UTF-8 + fhandle.close() + + return fname
\ No newline at end of file diff --git a/tools/grit/grit/tool/interface.py b/tools/grit/grit/tool/interface.py new file mode 100644 index 0000000..d56fac4 --- /dev/null +++ b/tools/grit/grit/tool/interface.py @@ -0,0 +1,84 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Base class and interface for tools. +''' + +import sys + +class Tool(object): + '''Base class for all tools. Tools should use their docstring (i.e. the + class-level docstring) for the help they want to have printed when they + are invoked.''' + + # + # Interface (abstract methods) + # + + def ShortDescription(self): + '''Returns a short description of the functionality of the tool.''' + raise NotImplementedError() + + def Run(self, global_options, my_arguments): + '''Runs the tool. + + Args: + global_options: object grit_runner.Options + my_arguments: [arg1 arg2 ...] + + Return: + 0 for success, non-0 for error + ''' + raise NotImplementedError() + + # + # Base class implementation + # + + def __init__(self): + self.o = None + + def SetOptions(self, opts): + self.o = opts + + def Out(self, text): + '''Always writes out 'text'.''' + self.o.output_stream.write(text) + + def VerboseOut(self, text): + '''Writes out 'text' if the verbose option is on.''' + if self.o.verbose: + self.o.output_stream.write(text) + + def ExtraVerboseOut(self, text): + '''Writes out 'text' if the extra-verbose option is on. + ''' + if self.o.extra_verbose: + self.o.output_stream.write(text)
\ No newline at end of file diff --git a/tools/grit/grit/tool/menu_from_parts.py b/tools/grit/grit/tool/menu_from_parts.py new file mode 100644 index 0000000..480335c --- /dev/null +++ b/tools/grit/grit/tool/menu_from_parts.py @@ -0,0 +1,108 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit menufromparts' tool.''' + +import os +import getopt +import types + +from grit.tool import interface +from grit.tool import transl2tc +from grit import grd_reader +from grit import tclib +from grit import util +from grit import xtb_reader + + +import grit.extern.tclib + + +class MenuTranslationsFromParts(interface.Tool): + '''One-off tool to generate translated menu messages (where each menu is kept +in a single message) based on existing translations of the individual menu +items. Was needed when changing menus from being one message per menu item +to being one message for the whole menu.''' + + def ShortDescription(self): + return ('Create translations of whole menus from existing translations of ' + 'menu items.') + + def Run(self, globopt, args): + self.SetOptions(globopt) + assert len(args) == 2, "Need exactly two arguments, the XTB file and the output file" + + xtb_file = args[0] + output_file = args[1] + + grd = grd_reader.Parse(self.o.input, debug=self.o.extra_verbose) + grd.OnlyTheseTranslations([]) # don't load translations + grd.RunGatherers(recursive = True) + + xtb = {} + def Callback(msg_id, parts): + msg = [] + for part in parts: + if part[0]: + msg = [] + break # it had a placeholder so ignore it + else: + msg.append(part[1]) + if len(msg): + xtb[msg_id] = ''.join(msg) + f = file(xtb_file) + xtb_reader.Parse(f, Callback) + f.close() + + translations = [] # list of translations as per transl2tc.WriteTranslations + for node in grd: + if node.name == 'structure' and node.attrs['type'] == 'menu': + assert len(node.GetCliques()) == 1 + message = node.GetCliques()[0].GetMessage() + translation = [] + + contents = message.GetContent() + for part in contents: + if isinstance(part, types.StringTypes): + id = grit.extern.tclib.GenerateMessageId(part) + if id not in xtb: + print "WARNING didn't find all translations for menu %s" % node.attrs['name'] + translation = [] + break + translation.append(xtb[id]) + else: + translation.append(part.GetPresentation()) + + if len(translation): + translations.append([message.GetId(), ''.join(translation)]) + + f = util.WrapOutputStream(file(output_file, 'w')) + transl2tc.TranslationToTc.WriteTranslations(f, translations) + f.close() diff --git a/tools/grit/grit/tool/newgrd.py b/tools/grit/grit/tool/newgrd.py new file mode 100644 index 0000000..29e284c --- /dev/null +++ b/tools/grit/grit/tool/newgrd.py @@ -0,0 +1,96 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Tool to create a new, empty .grd file with all the basic sections. +''' + +from grit.tool import interface +from grit import constants +from grit import util + +# The contents of the new .grd file +_FILE_CONTENTS = '''\ +<?xml version="1.0" encoding="UTF-8"?> +<grit base_dir="." latest_public_release="0" current_release="1" + source_lang_id="en" enc_check="%s"> + <outputs> + <!-- TODO add each of your output files. Modify the three below, and add + your own for your various languages. See the user's guide for more + details. + Note that all output references are relative to the output directory + which is specified at build time. --> + <output filename="resource.h" type="rc_header" /> + <output filename="en_resource.rc" type="rc_all" /> + <output filename="fr_resource.rc" type="rc_all" /> + </outputs> + <translations> + <!-- TODO add references to each of the XTB files (from the Translation + Console) that contain translations of messages in your project. Each + takes a form like <file path="english.xtb" />. Remember that all file + references are relative to this .grd file. --> + </translations> + <release seq="1"> + <includes> + <!-- TODO add a list of your included resources here, e.g. BMP and GIF + resources. --> + </includes> + <structures> + <!-- TODO add a list of all your structured resources here, e.g. HTML + templates, menus, dialogs etc. Note that for menus, dialogs and version + information resources you reference an .rc file containing them.--> + </structures> + <messages> + <!-- TODO add all of your "string table" messages here. Remember to + change nontranslateable parts of the messages into placeholders (using the + <ph> element). You can also use the 'grit add' tool to help you identify + nontranslateable parts and create placeholders for them. --> + </messages> + </release> +</grit>''' % constants.ENCODING_CHECK + + +class NewGrd(interface.Tool): + '''Usage: grit newgrd OUTPUT_FILE + +Creates a new, empty .grd file OUTPUT_FILE with comments about what to put +where in the file.''' + + def ShortDescription(self): + return 'Create a new empty .grd file.' + + def Run(self, global_options, my_arguments): + if not len(my_arguments) == 1: + print 'This tool requires exactly one argument, the name of the output file.' + return 2 + filename = my_arguments[0] + out = util.WrapOutputStream(file(filename, 'w'), 'utf-8') + out.write(_FILE_CONTENTS) + out.close() + print "Wrote file %s" % filename
\ No newline at end of file diff --git a/tools/grit/grit/tool/postprocess_interface.py b/tools/grit/grit/tool/postprocess_interface.py new file mode 100644 index 0000000..afcb12b --- /dev/null +++ b/tools/grit/grit/tool/postprocess_interface.py @@ -0,0 +1,57 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' Base class for postprocessing of RC files. +''' + +import sys + +class PostProcessor(object): + ''' Base class for postprocessing of the RC file data before being + output through the RC2GRD tool. You should implement this class if + you want GRIT to do specific things to the RC files after it has + converted the data into GRD format, i.e. change the content of the + RC file, and put it into a P4 changelist, etc.''' + + + def Process(self, rctext, rcpath, grdnode): + ''' Processes the data in rctext and grdnode. + Args: + rctext: string containing the contents of the RC file being processed. + rcpath: the path used to access the file. + grdtext: the root node of the grd xml data generated by + the rc2grd tool. + + Return: + The root node of the processed GRD tree. + ''' + raise NotImplementedError() + + diff --git a/tools/grit/grit/tool/postprocess_unittest.py b/tools/grit/grit/tool/postprocess_unittest.py new file mode 100644 index 0000000..16b31e6 --- /dev/null +++ b/tools/grit/grit/tool/postprocess_unittest.py @@ -0,0 +1,87 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit test that checks postprocessing of files. + Tests postprocessing by having the postprocessor + modify the grd data tree, changing the message name attributes. +''' + +import os +import re +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + +import grit.tool.postprocess_interface +from grit.tool import rc2grd + + +class PostProcessingUnittest(unittest.TestCase): + + def testPostProcessing(self): + rctext = '''STRINGTABLE +BEGIN + DUMMY_STRING_1 "String 1" + // Some random description + DUMMY_STRING_2 "This text was added during preprocessing" +END + ''' + tool = rc2grd.Rc2Grd() + class DummyOpts(object): + verbose = False + extra_verbose = False + tool.o = DummyOpts() + tool.post_process = 'grit.tool.postprocess_unittest.DummyPostProcessor' + result = tool.Process(rctext, '.\resource.rc') + + self.failUnless( + result.children[2].children[2].children[0].attrs['name'] == 'SMART_STRING_1') + self.failUnless( + result.children[2].children[2].children[1].attrs['name'] == 'SMART_STRING_2') + +class DummyPostProcessor(grit.tool.postprocess_interface.PostProcessor): + ''' + Post processing replaces all message name attributes containing "DUMMY" to + "SMART". + ''' + def Process(self, rctext, rcpath, grdnode): + smarter = re.compile(r'(DUMMY)(.*)') + messages = grdnode.children[2].children[2] + for node in messages.children: + name_attr = node.attrs['name'] + m = smarter.search(name_attr) + if m: + node.attrs['name'] = 'SMART' + m.group(2) + return grdnode + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/tool/preprocess_interface.py b/tools/grit/grit/tool/preprocess_interface.py new file mode 100644 index 0000000..a6a7f4b --- /dev/null +++ b/tools/grit/grit/tool/preprocess_interface.py @@ -0,0 +1,53 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' Base class for preprocessing of RC files. +''' + +import sys + +class PreProcessor(object): + ''' Base class for preprocessing of the RC file data before being + output through the RC2GRD tool. You should implement this class if + you have specific constructs in your RC files that GRIT cannot handle.''' + + + def Process(self, rctext, rcpath): + ''' Processes the data in rctext. + Args: + rctext: string containing the contents of the RC file being processed + rcpath: the path used to access the file. + + Return: + The processed text. + ''' + raise NotImplementedError() + + diff --git a/tools/grit/grit/tool/preprocess_unittest.py b/tools/grit/grit/tool/preprocess_unittest.py new file mode 100644 index 0000000..ef5a5f9 --- /dev/null +++ b/tools/grit/grit/tool/preprocess_unittest.py @@ -0,0 +1,73 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit test that checks preprocessing of files. + Tests preprocessing by adding having the preprocessor + provide the actual rctext data. +''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + +import grit.tool.preprocess_interface +from grit.tool import rc2grd + + +class PreProcessingUnittest(unittest.TestCase): + + def testPreProcessing(self): + tool = rc2grd.Rc2Grd() + class DummyOpts(object): + verbose = False + extra_verbose = False + tool.o = DummyOpts() + tool.pre_process = 'grit.tool.preprocess_unittest.DummyPreProcessor' + result = tool.Process('', '.\resource.rc') + + self.failUnless( + result.children[2].children[2].children[0].attrs['name'] == 'DUMMY_STRING_1') + +class DummyPreProcessor(grit.tool.preprocess_interface.PreProcessor): + def Process(self, rctext, rcpath): + rctext = '''STRINGTABLE +BEGIN + DUMMY_STRING_1 "String 1" + // Some random description + DUMMY_STRING_2 "This text was added during preprocessing" +END + ''' + return rctext + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/tool/rc2grd.py b/tools/grit/grit/tool/rc2grd.py new file mode 100644 index 0000000..24080eb --- /dev/null +++ b/tools/grit/grit/tool/rc2grd.py @@ -0,0 +1,427 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit rc2grd' tool.''' + + +import os.path +import getopt +import re +import StringIO +import types + +import grit.node.empty +from grit.node import include +from grit.node import structure +from grit.node import message + +from grit.gather import rc +from grit.gather import tr_html + +from grit.tool import interface +from grit.tool import postprocess_interface +from grit.tool import preprocess_interface + +from grit import grd_reader +from grit import tclib +from grit import util + + +# Matches files referenced from an .rc file +_FILE_REF = re.compile(''' + ^(?P<id>[A-Z_0-9.]+)[ \t]+ + (?P<type>[A-Z_0-9]+)[ \t]+ + "(?P<file>.*?([^"]|""))"[ \t]*$''', re.VERBOSE | re.MULTILINE) + + +# Matches a dialog section +_DIALOG = re.compile('^(?P<id>[A-Z0-9_]+)\s+DIALOG(EX)?\s.+?^BEGIN\s*$.+?^END\s*$', + re.MULTILINE | re.DOTALL) + + +# Matches a menu section +_MENU = re.compile('^(?P<id>[A-Z0-9_]+)\s+MENU.+?^BEGIN\s*$.+?^END\s*$', + re.MULTILINE | re.DOTALL) + + +# Matches a versioninfo section +_VERSIONINFO = re.compile('^(?P<id>[A-Z0-9_]+)\s+VERSIONINFO\s.+?^BEGIN\s*$.+?^END\s*$', + re.MULTILINE | re.DOTALL) + + +# Matches a stringtable +_STRING_TABLE = re.compile('^STRINGTABLE(\s+(PRELOAD|DISCARDABLE|CHARACTERISTICS.+|LANGUAGE.+|VERSION.+))*\s*\nBEGIN\s*$(?P<body>.+?)^END\s*$', + re.MULTILINE | re.DOTALL) + + +# Matches each message inside a stringtable, breaking it up into comments, +# the ID of the message, and the (RC-escaped) message text. +_MESSAGE = re.compile(''' + (?P<comment>(^\s+//.+?)*) # 0 or more lines of comments preceding the message + ^\s* + (?P<id>[A-Za-z0-9_]+) # id + \s+ + "(?P<text>.*?([^"]|""))"([^"]|$) # The message itself + ''', re.MULTILINE | re.DOTALL | re.VERBOSE) + + +# Matches each line of comment text in a multi-line comment. +_COMMENT_TEXT = re.compile('^\s*//\s*(?P<text>.+?)$', re.MULTILINE) + + +# Matches a string that is empty or all whitespace +_WHITESPACE_ONLY = re.compile('\A\s*\Z', re.MULTILINE) + + +# Finds printf and FormatMessage style format specifiers +# Uses non-capturing groups except for the outermost group, so the output of +# re.split() should include both the normal text and what we intend to +# replace with placeholders. +# TODO(joi) Check documentation for printf (and Windows variants) and FormatMessage +_FORMAT_SPECIFIER = re.compile( + '(%[-# +]?(?:[0-9]*|\*)(?:\.(?:[0-9]+|\*))?(?:h|l|L)?' # printf up to last char + '(?:d|i|o|u|x|X|e|E|f|F|g|G|c|r|s|ls|ws)' # printf last char + '|\$[1-9][0-9]*)') # FormatMessage + + +class Rc2Grd(interface.Tool): + '''A tool for converting .rc files to .grd files. This tool is only for +converting the source (nontranslated) .rc file to a .grd file. For importing +existing translations, use the rc2xtb tool. + +Usage: grit [global options] rc2grd [OPTIONS] RCFILE + +The tool takes a single argument, which is the path to the .rc file to convert. +It outputs a .grd file with the same name in the same directory as the .rc file. +The .grd file may have one or more TODO comments for things that have to be +cleaned up manually. + +OPTIONS may be any of the following: + + -e ENCODING Specify the ENCODING of the .rc file. Default is 'cp1252'. + + -h TYPE Specify the TYPE attribute for HTML structures. + Default is 'tr_html'. + + -u ENCODING Specify the ENCODING of HTML files. Default is 'utf-8'. + + -n MATCH Specify the regular expression to match in comments that will + indicate that the resource the comment belongs to is not + translateable. Default is 'Not locali(s|z)able'. + + -r GRDFILE Specify that GRDFILE should be used as a "role model" for + any placeholders that otherwise would have had TODO names. + This attempts to find an identical message in the GRDFILE + and uses that instead of the automatically placeholderized + message. + + --pre CLASS Specify an optional, fully qualified classname, which + has to be a subclass of grit.tool.PreProcessor, to + run on the text of the RC file before conversion occurs. + This can be used to support constructs in the RC files + that GRIT cannot handle on its own. + + --post CLASS Specify an optional, fully qualified classname, which + has to be a subclass of grit.tool.PostProcessor, to + run on the text of the converted RC file. + This can be used to alter the content of the RC file + based on the conversion that occured. + +For menus, dialogs and version info, the .grd file will refer to the original +.rc file. Once conversion is complete, you can strip the original .rc file +of its string table and all comments as these will be available in the .grd +file. + +Note that this tool WILL NOT obey C preprocessor rules, so even if something +is #if 0-ed out it will still be included in the output of this tool +Therefore, if your .rc file contains sections like this, you should run the +C preprocessor on the .rc file or manually edit it before using this tool. +''' + + def ShortDescription(self): + return 'A tool for converting .rc source files to .grd files.' + + def __init__(self): + self.input_encoding = 'cp1252' + self.html_type = 'tr_html' + self.html_encoding = 'utf-8' + self.not_localizable_re = re.compile('Not locali(s|z)able') + self.role_model = None + self.pre_process = None + self.post_process = None + + def ParseOptions(self, args): + '''Given a list of arguments, set this object's options and return + all non-option arguments. + ''' + (own_opts, args) = getopt.getopt(args, 'e:h:u:n:r', ['pre=', 'post=']) + for (key, val) in own_opts: + if key == '-e': + self.input_encoding = val + elif key == '-h': + self.html_type = val + elif key == '-u': + self.html_encoding = val + elif key == '-n': + self.not_localizable_re = re.compile(val) + elif key == '-r': + self.role_model = grd_reader.Parse(val) + elif key == '--pre': + self.pre_process = val + elif key == '--post': + self.post_process = val + return args + + def Run(self, opts, args): + args = self.ParseOptions(args) + if len(args) != 1: + print ('This tool takes a single tool-specific argument, the path to the\n' + '.rc file to process.') + return 2 + self.SetOptions(opts) + + path = args[0] + out_path = os.path.join(util.dirname(path), + os.path.splitext(os.path.basename(path))[0] + '.grd') + + rcfile = util.WrapInputStream(file(path, 'r'), self.input_encoding) + rctext = rcfile.read() + + grd_text = unicode(self.Process(rctext, path)) + + rcfile.close() + + outfile = util.WrapOutputStream(file(out_path, 'w'), 'utf-8') + outfile.write(grd_text) + outfile.close() + + print 'Wrote output file %s.\nPlease check for TODO items in the file.' % out_path + + + def Process(self, rctext, rc_path): + '''Processes 'rctext' and returns a resource tree corresponding to it. + + Args: + rctext: complete text of the rc file + rc_path: 'resource\resource.rc' + + Return: + grit.node.base.Node subclass + ''' + + if self.pre_process: + preprocess_class = util.NewClassInstance(self.pre_process, + preprocess_interface.PreProcessor) + if preprocess_class: + rctext = preprocess_class.Process(rctext, rc_path) + else: + self.Out( + 'PreProcessing class could not be found. Skipping preprocessing.\n') + + # Start with a basic skeleton for the .grd file + root = grd_reader.Parse(StringIO.StringIO( + '''<?xml version="1.0" encoding="UTF-8"?> + <grit base_dir="." latest_public_release="0" + current_release="1" source_lang_id="en"> + <outputs /> + <translations /> + <release seq="1"> + <includes /> + <structures /> + <messages /> + </release> + </grit>'''), util.dirname(rc_path)) + includes = root.children[2].children[0] + structures = root.children[2].children[1] + messages = root.children[2].children[2] + assert (isinstance(includes, grit.node.empty.IncludesNode) and + isinstance(structures, grit.node.empty.StructuresNode) and + isinstance(messages, grit.node.empty.MessagesNode)) + + self.AddIncludes(rctext, includes) + self.AddStructures(rctext, structures, os.path.basename(rc_path)) + self.AddMessages(rctext, messages) + + self.VerboseOut('Validating that all IDs are unique...\n') + root.ValidateUniqueIds() + self.ExtraVerboseOut('Done validating that all IDs are unique.\n') + + if self.post_process: + postprocess_class = util.NewClassInstance(self.post_process, + postprocess_interface.PostProcessor) + if postprocess_class: + root = postprocess_class.Process(rctext, rc_path, root) + else: + self.Out( + 'PostProcessing class could not be found. Skipping postprocessing.\n') + + return root + + + def AddIncludes(self, rctext, node): + '''Scans 'rctext' for included resources (e.g. BITMAP, ICON) and + adds each included resource as an <include> child node of 'node'.''' + for m in _FILE_REF.finditer(rctext): + id = m.group('id') + type = m.group('type').upper() + fname = rc.Section.UnEscape(m.group('file')) + assert fname.find('\n') == -1 + if type != 'HTML': + self.VerboseOut('Processing %s with ID %s (filename: %s)\n' % (type, id, fname)) + node.AddChild(include.IncludeNode.Construct(node, id, type, fname)) + + + def AddStructures(self, rctext, node, rc_filename): + '''Scans 'rctext' for structured resources (e.g. menus, dialogs, version + information resources and HTML templates) and adds each as a <structure> + child of 'node'.''' + # First add HTML includes + for m in _FILE_REF.finditer(rctext): + id = m.group('id') + type = m.group('type').upper() + fname = rc.Section.UnEscape(m.group('file')) + if type == 'HTML': + node.AddChild(structure.StructureNode.Construct( + node, id, self.html_type, fname, self.html_encoding)) + + # Then add all RC includes + def AddStructure(type, id): + self.VerboseOut('Processing %s with ID %s\n' % (type, id)) + node.AddChild(structure.StructureNode.Construct(node, id, type, + rc_filename, + encoding=self.input_encoding)) + for m in _MENU.finditer(rctext): + AddStructure('menu', m.group('id')) + for m in _DIALOG.finditer(rctext): + AddStructure('dialog', m.group('id')) + for m in _VERSIONINFO.finditer(rctext): + AddStructure('version', m.group('id')) + + + def AddMessages(self, rctext, node): + '''Scans 'rctext' for all messages in string tables, preprocesses them as + much as possible for placeholders (e.g. messages containing $1, $2 or %s, %d + type format specifiers get those specifiers replaced with placeholders, and + HTML-formatted messages get run through the HTML-placeholderizer). Adds + each message as a <message> node child of 'node'.''' + for tm in _STRING_TABLE.finditer(rctext): + table = tm.group('body') + for mm in _MESSAGE.finditer(table): + comment_block = mm.group('comment') + comment_text = [] + for cm in _COMMENT_TEXT.finditer(comment_block): + comment_text.append(cm.group('text')) + comment_text = ' '.join(comment_text) + + id = mm.group('id') + text = rc.Section.UnEscape(mm.group('text')) + + self.VerboseOut('Processing message %s (text: "%s")\n' % (id, text)) + + msg_obj = self.Placeholderize(text) + + # Messages that contain only placeholders do not need translation. + is_translateable = False + for item in msg_obj.GetContent(): + if isinstance(item, types.StringTypes): + if not _WHITESPACE_ONLY.match(item): + is_translateable = True + + if self.not_localizable_re.search(comment_text): + is_translateable = False + + message_meaning = '' + internal_comment = '' + + # If we have a "role model" (existing GRD file) and this node exists + # in the role model, use the description, meaning and translateable + # attributes from the role model. + if self.role_model: + role_node = self.role_model.GetNodeById(id) + if role_node: + is_translateable = role_node.IsTranslateable() + message_meaning = role_node.attrs['meaning'] + comment_text = role_node.attrs['desc'] + internal_comment = role_node.attrs['internal_comment'] + + # For nontranslateable messages, we don't want the complexity of + # placeholderizing everything. + if not is_translateable: + msg_obj = tclib.Message(text=text) + + msg_node = message.MessageNode.Construct(node, msg_obj, id, + desc=comment_text, + translateable=is_translateable, + meaning=message_meaning) + msg_node.attrs['internal_comment'] = internal_comment + + node.AddChild(msg_node) + self.ExtraVerboseOut('Done processing message %s\n' % id) + + + def Placeholderize(self, text): + '''Creates a tclib.Message object from 'text', attempting to recognize + a few different formats of text that can be automatically placeholderized + (HTML code, printf-style format strings, and FormatMessage-style format + strings). + ''' + + try: + # First try HTML placeholderizing. + # TODO(joi) Allow use of non-TotalRecall flavors of HTML placeholderizing + msg = tr_html.HtmlToMessage(text, True) + for item in msg.GetContent(): + if not isinstance(item, types.StringTypes): + return msg # Contained at least one placeholder, so we're done + + # HTML placeholderization didn't do anything, so try to find printf or + # FormatMessage format specifiers and change them into placeholders. + msg = tclib.Message() + parts = _FORMAT_SPECIFIER.split(text) + todo_counter = 1 # We make placeholder IDs 'TODO_0001' etc. + for part in parts: + if _FORMAT_SPECIFIER.match(part): + msg.AppendPlaceholder(tclib.Placeholder( + 'TODO_%04d' % todo_counter, part, 'TODO')) + todo_counter += 1 + elif part != '': + msg.AppendText(part) + + if self.role_model and len(parts) > 1: # there are TODO placeholders + role_model_msg = self.role_model.UberClique().BestCliqueByOriginalText( + msg.GetRealContent(), '') + if role_model_msg: + # replace wholesale to get placeholder names and examples + msg = role_model_msg + + return msg + except: + print 'Exception processing message with text "%s"' % text + raise diff --git a/tools/grit/grit/tool/rc2grd_unittest.py b/tools/grit/grit/tool/rc2grd_unittest.py new file mode 100644 index 0000000..3e9cc3f --- /dev/null +++ b/tools/grit/grit/tool/rc2grd_unittest.py @@ -0,0 +1,162 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.tool.rc2grd''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import re +import StringIO +import unittest + +from grit.node import base +from grit.tool import rc2grd +from grit.gather import rc +from grit import grd_reader + + +class Rc2GrdUnittest(unittest.TestCase): + def testPlaceholderize(self): + tool = rc2grd.Rc2Grd() + original = "Hello %s, how are you? I'm $1 years old!" + msg = tool.Placeholderize(original) + self.failUnless(msg.GetPresentableContent() == "Hello TODO_0001, how are you? I'm TODO_0002 years old!") + self.failUnless(msg.GetRealContent() == original) + + def testHtmlPlaceholderize(self): + tool = rc2grd.Rc2Grd() + original = "Hello <b>[USERNAME]</b>, how are you? I'm [AGE] years old!" + msg = tool.Placeholderize(original) + self.failUnless(msg.GetPresentableContent() == + "Hello BEGIN_BOLDX_USERNAME_XEND_BOLD, how are you? I'm X_AGE_X years old!") + self.failUnless(msg.GetRealContent() == original) + + def testMenuWithoutWhitespaceRegression(self): + # There was a problem in the original regular expression for parsing out + # menu sections, that would parse the following block of text as a single + # menu instead of two. + two_menus = ''' +// Hyper context menus +IDR_HYPERMENU_FOLDER MENU +BEGIN + POPUP "HyperFolder" + BEGIN + MENUITEM "Open Containing Folder", IDM_OPENFOLDER + END +END + +IDR_HYPERMENU_FILE MENU +BEGIN + POPUP "HyperFile" + BEGIN + MENUITEM "Open Folder", IDM_OPENFOLDER + END +END + +''' + self.failUnless(len(rc2grd._MENU.findall(two_menus)) == 2) + + def testRegressionScriptWithTranslateable(self): + tool = rc2grd.Rc2Grd() + + # test rig + class DummyNode(base.Node): + def AddChild(self, item): + self.node = item + verbose = False + extra_verbose = False + tool.not_localizable_re = re.compile('') + tool.o = DummyNode() + + rc_text = '''STRINGTABLE\nBEGIN\nID_BINGO "<SPAN id=hp style='BEHAVIOR: url(#default#homepage)'></SPAN><script>if (!hp.isHomePage('[$~HOMEPAGE~$]')) {document.write(""<a href=\\""[$~SETHOMEPAGEURL~$]\\"" >Set As Homepage</a> - "");}</script>"\nEND\n''' + tool.AddMessages(rc_text, tool.o) + self.failUnless(tool.o.node.GetCdata().find('Set As Homepage') != -1) + + # TODO(joi) Improve the HTML parser to support translateables inside + # <script> blocks? + self.failUnless(tool.o.node.attrs['translateable'] == 'false') + + def testRoleModel(self): + rc_text = ('STRINGTABLE\n' + 'BEGIN\n' + ' // This should not show up\n' + ' IDS_BINGO "Hello %s, how are you?"\n' + ' // The first description\n' + ' IDS_BONGO "Hello %s, my name is %s, and yours?"\n' + ' IDS_PROGRAMS_SHUTDOWN_TEXT "Google Desktop Search needs to close the following programs:\\n\\n$1\\nThe installation will not proceed if you choose to cancel."\n' + 'END\n') + tool = rc2grd.Rc2Grd() + tool.role_model = grd_reader.Parse(StringIO.StringIO( + '''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <messages> + <message name="IDS_BINGO"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you? + </message> + <message name="IDS_BONGO" desc="The other description"> + Hello <ph name="USERNAME">%s<ex>Jakob</ex></ph>, my name is <ph name="ADMINNAME">%s<ex>Joi</ex></ph>, and yours? + </message> + <message name="IDS_PROGRAMS_SHUTDOWN_TEXT" desc="LIST_OF_PROGRAMS is replaced by a bulleted list of program names."> + Google Desktop Search needs to close the following programs: + +<ph name="LIST_OF_PROGRAMS">$1<ex>Program 1, Program 2</ex></ph> +The installation will not proceed if you choose to cancel. + </message> + </messages> + </release> + </grit>'''), dir='.') + + # test rig + class DummyOpts(object): + verbose = False + extra_verbose = False + tool.o = DummyOpts() + result = tool.Process(rc_text, '.\resource.rc') + self.failUnless( + result.children[2].children[2].children[0].attrs['desc'] == '') + self.failUnless( + result.children[2].children[2].children[0].children[0].attrs['name'] == 'USERNAME') + self.failUnless( + result.children[2].children[2].children[1].attrs['desc'] == 'The other description') + self.failUnless( + result.children[2].children[2].children[1].attrs['meaning'] == '') + self.failUnless( + result.children[2].children[2].children[1].children[0].attrs['name'] == 'USERNAME') + self.failUnless( + result.children[2].children[2].children[1].children[1].attrs['name'] == 'ADMINNAME') + self.failUnless( + result.children[2].children[2].children[2].children[0].attrs['name'] == 'LIST_OF_PROGRAMS') + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/tool/resize.py b/tools/grit/grit/tool/resize.py new file mode 100644 index 0000000..9081bff --- /dev/null +++ b/tools/grit/grit/tool/resize.py @@ -0,0 +1,325 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit resize' tool. +''' + +import getopt +import os +import types + +from grit.tool import interface +from grit.tool import build +from grit import grd_reader +from grit import pseudo +from grit import util + +from grit.node import include +from grit.node import structure +from grit.node import message + +from grit.format import rc_header + + +# Template for the .vcproj file, with a couple of [[REPLACEABLE]] parts. +PROJECT_TEMPLATE = '''\ +<?xml version="1.0" encoding="Windows-1252"?> +<VisualStudioProject + ProjectType="Visual C++" + Version="7.10" + Name="[[DIALOG_NAME]]" + ProjectGUID="[[PROJECT_GUID]]" + Keyword="Win32Proj"> + <Platforms> + <Platform + Name="Win32"/> + </Platforms> + <Configurations> + <Configuration + Name="Debug|Win32" + OutputDirectory="Debug" + IntermediateDirectory="Debug" + ConfigurationType="1" + CharacterSet="2"> + </Configuration> + </Configurations> + <References> + </References> + <Files> + <Filter + Name="Resource Files" + Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx" + UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"> + <File + RelativePath=".\[[DIALOG_NAME]].rc"> + </File> + </Filter> + </Files> + <Globals> + </Globals> +</VisualStudioProject>''' + + +# Template for the .rc file with a couple of [[REPLACEABLE]] parts. +# TODO(joi) Improve this (and the resource.h template) to allow saving and then +# reopening of the RC file in Visual Studio. Currently you can only open it +# once and change it, then after you close it you won't be able to reopen it. +RC_TEMPLATE = '''\ +// Copyright (c) Google Inc. 2005 +// All rights reserved. +// This file is automatically generated by GRIT and intended for editing +// the layout of the dialogs contained in it. Do not edit anything but the +// dialogs. Any changes made to translateable portions of the dialogs will +// be ignored by GRIT. + +#include "resource.h" +#include <winres.h> +#include <winresrc.h> + +LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL + +#pragma code_page([[CODEPAGE_NUM]]) + +[[INCLUDES]] + +[[DIALOGS]] +''' + + +# Template for the resource.h file with a couple of [[REPLACEABLE]] parts. +HEADER_TEMPLATE = '''\ +// Copyright (c) Google Inc. 2005 +// All rights reserved. +// This file is automatically generated by GRIT. Do not edit. + +#pragma once + +// Edit commands +#define ID_EDIT_CLEAR 0xE120 +#define ID_EDIT_CLEAR_ALL 0xE121 +#define ID_EDIT_COPY 0xE122 +#define ID_EDIT_CUT 0xE123 +#define ID_EDIT_FIND 0xE124 +#define ID_EDIT_PASTE 0xE125 +#define ID_EDIT_PASTE_LINK 0xE126 +#define ID_EDIT_PASTE_SPECIAL 0xE127 +#define ID_EDIT_REPEAT 0xE128 +#define ID_EDIT_REPLACE 0xE129 +#define ID_EDIT_SELECT_ALL 0xE12A +#define ID_EDIT_UNDO 0xE12B +#define ID_EDIT_REDO 0xE12C + + +[[DEFINES]] +''' + + +class ResizeDialog(interface.Tool): + '''Generates an RC file, header and Visual Studio project that you can use +with Visual Studio's GUI resource editor to modify the layout of dialogs for +the language of your choice. You then use the RC file, after you resize the +dialog, for the language or languages of your choice, using the <skeleton> child +of the <structure> node for the dialog. The translateable bits of the dialog +will be ignored when you use the <skeleton> node (GRIT will instead use the +translateable bits from the original dialog) but the layout changes you make +will be used. Note that your layout changes must preserve the order of the +translateable elements in the RC file. + +Usage: grit resize [-f BASEFOLDER] [-l LANG] [-e RCENCODING] DIALOGID* + +Arguments: + DIALOGID The 'name' attribute of a dialog to output for resizing. Zero + or more of these parameters can be used. If none are + specified, all dialogs from the input .grd file are output. + +Options: + + -f BASEFOLDER The project will be created in a subfolder of BASEFOLDER. + The name of the subfolder will be the first DIALOGID you + specify. Defaults to '.' + + -l LANG Specifies that the RC file should contain a dialog translated + into the language LANG. The default is a cp1252-representable + pseudotranslation, because Visual Studio's GUI RC editor only + supports single-byte encodings. + + -c CODEPAGE Code page number to indicate to the RC compiler the encoding + of the RC file, default is something reasonable for the + language you selected (but this does not work for every single + language). See details on codepages below. NOTE that you do + not need to specify the codepage unless the tool complains + that it's not sure which codepage to use. See the following + page for codepage numbers supported by Windows: + http://www.microsoft.com/globaldev/reference/wincp.mspx + + -D NAME[=VAL] Specify a C-preprocessor-like define NAME with optional + value VAL (defaults to 1) which will be used to control + conditional inclusion of resources. + + +IMPORTANT NOTE: For now, the tool outputs a UTF-8 encoded file for any language +that can not be represented in cp1252 (i.e. anything other than Western +European languages). You will need to open this file in a text editor and +save it using the codepage indicated in the #pragma code_page(XXXX) command +near the top of the file, before you open it in Visual Studio. + +''' + + # TODO(joi) It would be cool to have this tool note the Perforce revision + # of the original RC file somewhere, such that the <skeleton> node could warn + # if the original RC file gets updated without the skeleton file being updated. + + # TODO(joi) Would be cool to have option to add the files to Perforce + + def __init__(self): + self.lang = pseudo.PSEUDO_LANG + self.defines = {} + self.base_folder = '.' + self.codepage_number = 1252 + self.codepage_number_specified_explicitly = False + + def SetLanguage(self, lang): + '''Sets the language code to output things in. + ''' + self.lang = lang + if not self.codepage_number_specified_explicitly: + self.codepage_number = util.LanguageToCodepage(lang) + + def GetEncoding(self): + if self.codepage_number == 1200: + return 'utf_16' + if self.codepage_number == 65001: + return 'utf_8' + return 'cp%d' % self.codepage_number + + def ShortDescription(self): + return 'Generate a file where you can resize a given dialog.' + + def Run(self, opts, args): + self.SetOptions(opts) + + own_opts, args = getopt.getopt(args, 'l:f:c:D:') + for key, val in own_opts: + if key == '-l': + self.SetLanguage(val) + if key == '-f': + self.base_folder = val + if key == '-c': + self.codepage_number = int(val) + self.codepage_number_specified_explicitly = True + if key == '-D': + name, val = build.ParseDefine(val) + self.defines[name] = val + + res_tree = grd_reader.Parse(opts.input, debug=opts.extra_verbose) + res_tree.OnlyTheseTranslations([self.lang]) + res_tree.RunGatherers(True) + + # Dialog IDs are either explicitly listed, or we output all dialogs from the + # .grd file + dialog_ids = args + if not len(dialog_ids): + for node in res_tree: + if node.name == 'structure' and node.attrs['type'] == 'dialog': + dialog_ids.append(node.attrs['name']) + + self.Process(res_tree, dialog_ids) + + def Process(self, grd, dialog_ids): + '''Outputs an RC file and header file for the dialog 'dialog_id' stored in + resource tree 'grd', to self.base_folder, as discussed in this class's + documentation. + + Arguments: + grd: grd = grd_reader.Parse(...); grd.RunGatherers() + dialog_ids: ['IDD_MYDIALOG', 'IDD_OTHERDIALOG'] + ''' + grd.SetOutputContext(self.lang, self.defines) + + project_name = dialog_ids[0] + + dir_path = os.path.join(self.base_folder, project_name) + if not os.path.isdir(dir_path): + os.mkdir(dir_path) + + # If this fails then we're not on Windows (or you don't have the required + # win32all Python libraries installed), so what are you doing mucking + # about with RC files anyway? :) + import pythoncom + + # Create the .vcproj file + project_text = PROJECT_TEMPLATE.replace( + '[[PROJECT_GUID]]', str(pythoncom.CreateGuid()) + ).replace('[[DIALOG_NAME]]', project_name) + fname = os.path.join(dir_path, '%s.vcproj' % project_name) + self.WriteFile(fname, project_text) + print "Wrote %s" % fname + + # Create the .rc file + # Output all <include> nodes since the dialogs might depend on them (e.g. + # for icons and bitmaps). + include_items = [] + for node in grd: + if isinstance(node, include.IncludeNode): + formatter = node.ItemFormatter('rc_all') + if formatter: + include_items.append(formatter.Format(node, self.lang)) + rc_text = RC_TEMPLATE.replace('[[CODEPAGE_NUM]]', + str(self.codepage_number)) + rc_text = rc_text.replace('[[INCLUDES]]', ''.join(include_items)) + + # Then output the dialogs we have been asked to output. + dialogs = [] + for dialog_id in dialog_ids: + node = grd.GetNodeById(dialog_id) + # TODO(joi) Add exception handling for better error reporting + formatter = node.ItemFormatter('rc_all') + dialogs.append(formatter.Format(node, self.lang)) + rc_text = rc_text.replace('[[DIALOGS]]', ''.join(dialogs)) + + fname = os.path.join(dir_path, '%s.rc' % project_name) + self.WriteFile(fname, rc_text, self.GetEncoding()) + print "Wrote %s" % fname + + # Create the resource.h file + header_defines = [] + for node in grd: + formatter = node.ItemFormatter('rc_header') + if formatter and not isinstance(formatter, rc_header.TopLevel): + header_defines.append(formatter.Format(node, self.lang)) + header_text = HEADER_TEMPLATE.replace('[[DEFINES]]', ''.join(header_defines)) + fname = os.path.join(dir_path, 'resource.h') + self.WriteFile(fname, header_text) + print "Wrote %s" % fname + + def WriteFile(self, filename, contents, encoding='cp1252'): + f = util.WrapOutputStream(file(filename, 'wb'), encoding) + f.write(contents) + f.close()
\ No newline at end of file diff --git a/tools/grit/grit/tool/test.py b/tools/grit/grit/tool/test.py new file mode 100644 index 0000000..35ffca5 --- /dev/null +++ b/tools/grit/grit/tool/test.py @@ -0,0 +1,48 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from grit.tool import interface + +class TestTool(interface.Tool): + '''This tool does nothing except print out the global options and +tool-specific arguments that it receives. It is intended only for testing, +hence the name :) +''' + + def ShortDescription(self): + return 'A do-nothing tool for testing command-line parsing.' + + def Run(self, global_options, my_arguments): + print 'NOTE This tool is only for testing the parsing of global options and' + print 'tool-specific arguments that it receives. You may have intended to' + print 'run "grit unit" which is the unit-test suite for GRIT.' + print 'Options: %s' % repr(global_options) + print 'Arguments: %s' % repr(my_arguments) + return 0 diff --git a/tools/grit/grit/tool/toolbar_postprocess.py b/tools/grit/grit/tool/toolbar_postprocess.py new file mode 100644 index 0000000..d79eb2f --- /dev/null +++ b/tools/grit/grit/tool/toolbar_postprocess.py @@ -0,0 +1,150 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' Toolbar postprocessing class. Modifies the previously processed GRD tree +by creating separate message groups for each of the IDS_COMMAND macros. +Also adds some identifiers nodes to declare specific ids to be included +in the generated grh file. +''' + +import sys +import re +import postprocess_interface +import grit.node.empty +from grit.node import misc + +class ToolbarPostProcessor(postprocess_interface.PostProcessor): + ''' Defines message groups within the grd file for each of the + IDS_COMMAND stuff. + ''' + + _IDS_COMMAND = re.compile(r'IDS_COMMAND_') + _GRAB_PARAMETERS = re.compile(r'(IDS_COMMAND_[a-zA-Z0-9]+)_([a-zA-z0-9]+)') + + def Process(self, rctext, rcpath, grdnode): + ''' Processes the data in rctext and grdnode. + Args: + rctext: string containing the contents of the RC file being processed. + rcpath: the path used to access the file. + grdnode: the root node of the grd xml data generated by + the rc2grd tool. + + Return: + The root node of the processed GRD tree. + ''' + + release = grdnode.children[2] + messages = release.children[2] + + identifiers = grit.node.empty.IdentifiersNode() + identifiers.StartParsing('identifiers', release) + identifiers.EndParsing() + release.AddChild(identifiers) + + + # + # Turn the IDS_COMMAND messages into separate message groups + # with ids that are offsetted to the message group's first id + # + previous_name_attr = '' + previous_prefix = '' + previous_node = '' + new_messages_node = self.ConstructNewMessages(release) + for node in messages.children[:]: + name_attr = node.attrs['name'] + if self._IDS_COMMAND.search(name_attr): + mo = self._GRAB_PARAMETERS.search(name_attr) + mp = self._GRAB_PARAMETERS.search(previous_name_attr) + if mo and mp: + prefix = mo.group(1) + previous_prefix = mp.group(1) + new_message_id = mp.group(2) + if prefix == previous_prefix: + messages.RemoveChild(previous_name_attr) + previous_node.attrs['offset'] = 'PCI_' + new_message_id + del previous_node.attrs['name'] + new_messages_node.AddChild(previous_node) + else: + messages.RemoveChild(previous_name_attr) + previous_node.attrs['offset'] = 'PCI_' + new_message_id + del previous_node.attrs['name'] + new_messages_node.AddChild(previous_node) + new_messages_node.attrs['first_id'] = previous_prefix + new_messages_node = self.ConstructNewMessages(release) + else: + if self._IDS_COMMAND.search(previous_name_attr): + messages.RemoveChild(previous_name_attr) + previous_prefix = mp.group(1) + new_message_id = mp.group(2) + previous_node.attrs['offset'] = 'PCI_' + new_message_id + del previous_node.attrs['name'] + new_messages_node.AddChild(previous_node) + new_messages_node.attrs['first_id'] = previous_prefix + new_messages_node = self.ConstructNewMessages(release) + else: + if self._IDS_COMMAND.search(previous_name_attr): + messages.RemoveChild(previous_name_attr) + mp = self._GRAB_PARAMETERS.search(previous_name_attr) + previous_prefix = mp.group(1) + new_message_id = mp.group(2) + previous_node.attrs['offset'] = 'PCI_' + new_message_id + del previous_node.attrs['name'] + new_messages_node.AddChild(previous_node) + new_messages_node.attrs['first_id'] = previous_prefix + new_messages_node = self.ConstructNewMessages(release) + previous_name_attr = name_attr + previous_node = node + + + self.AddIdentifiers(rctext, identifiers) + return grdnode + + def ConstructNewMessages(self, parent): + new_node = grit.node.empty.MessagesNode() + new_node.StartParsing('messages', parent) + new_node.EndParsing() + parent.AddChild(new_node) + return new_node + + def AddIdentifiers(self, rctext, node): + node.AddChild(misc.IdentifierNode.Construct(node, 'IDS_COMMAND_gcFirst', '12000', '')) + node.AddChild(misc.IdentifierNode.Construct(node, + 'IDS_COMMAND_PCI_SPACE', '16', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_BUTTON', '0', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_MENU', '1', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP', '2', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_OPTIONS_TEXT', '3', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_DISABLED', '4', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_MENU', '5', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_MENU_DISABLED', '6', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_OPTIONS', '7', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_OPTIONS_DISABLED', '8', '')) + node.AddChild(misc.IdentifierNode.Construct(node, + 'PCI_TIP_DISABLED_BY_POLICY', '9', '')) diff --git a/tools/grit/grit/tool/toolbar_preprocess.py b/tools/grit/grit/tool/toolbar_preprocess.py new file mode 100644 index 0000000..6523ee8 --- /dev/null +++ b/tools/grit/grit/tool/toolbar_preprocess.py @@ -0,0 +1,86 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' Toolbar preprocessing code. Turns all IDS_COMMAND macros in the RC file +into simpler constructs that can be understood by GRIT. Also deals with +expansion of $lf; placeholders into the correct linefeed character. +''' + +import preprocess_interface + +import re +import sys +import codecs + +class ToolbarPreProcessor(preprocess_interface.PreProcessor): + ''' Toolbar PreProcessing class. + ''' + + _IDS_COMMAND_MACRO = re.compile(r'(.*IDS_COMMAND)\s*\(([a-zA-Z0-9_]*)\s*,\s*([a-zA-Z0-9_]*)\)(.*)') + _LINE_FEED_PH = re.compile(r'\$lf;') + _PH_COMMENT = re.compile(r'PHRWR') + _COMMENT = re.compile(r'^(\s*)//.*') + + + def Process(self, rctext, rcpath): + ''' Processes the data in rctext. + Args: + rctext: string containing the contents of the RC file being processed + rcpath: the path used to access the file. + + Return: + The processed text. + ''' + + ret = '' + rclines = rctext.splitlines() + for line in rclines: + + if self._LINE_FEED_PH.search(line): + # Replace "$lf;" placeholder comments by an empty line. + # this will not be put into the processed result + if self._PH_COMMENT.search(line): + mm = self._COMMENT.search(line) + if mm: + line = '%s//' % mm.group(1) + + else: + # Replace $lf by the right linefeed character + line = self._LINE_FEED_PH.sub(r'\\n', line) + + # Deal with IDS_COMMAND_MACRO stuff + mo = self._IDS_COMMAND_MACRO.search(line) + if mo: + line = '%s_%s_%s%s' % (mo.group(1), mo.group(2), mo.group(3), mo.group(4)) + + ret += (line + '\n') + + return ret + diff --git a/tools/grit/grit/tool/transl2tc.py b/tools/grit/grit/tool/transl2tc.py new file mode 100644 index 0000000..69fe2e5 --- /dev/null +++ b/tools/grit/grit/tool/transl2tc.py @@ -0,0 +1,278 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit transl2tc' tool. +''' + + +import getopt + +from grit.tool import interface +from grit.tool import rc2grd +from grit import grd_reader +from grit import util + +from grit.extern import tclib + + +class TranslationToTc(interface.Tool): + '''A tool for importing existing translations in RC format into the +Translation Console. + +Usage: + +grit -i GRD transl2tc [-l LIMITS] [RCOPTS] SOURCE_RC TRANSLATED_RC OUT_FILE + +The tool needs a "source" RC file, i.e. in English, and an RC file that is a +translation of precisely the source RC file (not of an older or newer version). + +The tool also requires you to provide a .grd file (input file) e.g. using the +-i global option or the GRIT_INPUT environment variable. The tool uses +information from your .grd file to correct placeholder names in the +translations and ensure that only translatable items and translations still +being used are output. + +This tool will accept all the same RCOPTS as the 'grit rc2grd' tool. To get +a list of these options, run 'grit help rc2grd'. + +Additionally, you can use the -l option (which must be the first option to the +tool) to specify a file containing a list of message IDs to which output should +be limited. This is only useful if you are limiting the output to your XMB +files using the 'grit xmb' tool's -l option. See 'grit help xmb' for how to +generate a file containing a list of the message IDs in an XMB file. + +The tool will scan through both of the RC files as well as any HTML files they +refer to, and match together the source messages and translated messages. It +will output a file (OUTPUT_FILE) you can import directly into the TC using the +Bulk Translation Upload tool. +''' + + def ShortDescription(self): + return 'Import existing translations in RC format into the TC' + + def Setup(self, globopt, args): + '''Sets the instance up for use. + ''' + self.SetOptions(globopt) + self.rc2grd = rc2grd.Rc2Grd() + self.rc2grd.SetOptions(globopt) + self.limits = None + if len(args) and args[0] == '-l': + limit_file = file(args[1]) + self.limits = limit_file.read().split('\n') + limit_file.close() + args = args[2:] + return self.rc2grd.ParseOptions(args) + + def Run(self, globopt, args): + args = self.Setup(globopt, args) + + if len(args) != 3: + self.Out('This tool takes exactly three arguments:\n' + ' 1. The path to the original RC file\n' + ' 2. The path to the translated RC file\n' + ' 3. The output file path.\n') + return 2 + + grd = grd_reader.Parse(self.o.input, debug=self.o.extra_verbose) + grd.RunGatherers(recursive = True) + + source_rc = util.WrapInputStream(file(args[0], 'r'), self.rc2grd.input_encoding) + transl_rc = util.WrapInputStream(file(args[1], 'r'), self.rc2grd.input_encoding) + translations = self.ExtractTranslations(grd, + source_rc.read(), args[0], + transl_rc.read(), args[1]) + transl_rc.close() + source_rc.close() + + output_file = util.WrapOutputStream(file(args[2], 'w')) + self.WriteTranslations(output_file, translations.items()) + output_file.close() + + self.Out('Wrote output file %s' % args[2]) + + def ExtractTranslations(self, current_grd, source_rc, source_path, transl_rc, transl_path): + '''Extracts translations from the translated RC file, matching them with + translations in the source RC file to calculate their ID, and correcting + placeholders, limiting output to translateables, etc. using the supplied + .grd file which is the current .grd file for your project. + + If this object's 'limits' attribute is not None but a list, the output of + this function will be further limited to include only messages that have + message IDs in the 'limits' list. + + Args: + current_grd: grit.node.base.Node child, that has had RunGatherers(True) run on it + source_rc: Complete text of source RC file + source_path: Path to the source RC file + transl_rc: Complete text of translated RC file + transl_path: Path to the translated RC file + + Return: + { id1 : text1, '12345678' : 'Hello USERNAME, howzit?' } + ''' + source_grd = self.rc2grd.Process(source_rc, source_path) + self.VerboseOut('Read %s into GRIT format, running gatherers.\n' % source_path) + source_grd.RunGatherers(recursive=True, debug=self.o.extra_verbose) + transl_grd = self.rc2grd.Process(transl_rc, transl_path) + self.VerboseOut('Read %s into GRIT format, running gatherers.\n' % transl_path) + transl_grd.RunGatherers(recursive=True, debug=self.o.extra_verbose) + self.VerboseOut('Done running gatherers for %s.\n' % transl_path) + + # Proceed to create a map from ID to translation, getting the ID from the + # source GRD and the translation from the translated GRD. + id2transl = {} + for source_node in source_grd: + source_cliques = source_node.GetCliques() + if not len(source_cliques): + continue + + assert 'name' in source_node.attrs, 'All nodes with cliques should have an ID' + node_id = source_node.attrs['name'] + self.ExtraVerboseOut('Processing node %s\n' % node_id) + transl_node = transl_grd.GetNodeById(node_id) + + if transl_node: + transl_cliques = transl_node.GetCliques() + if not len(transl_cliques) == len(source_cliques): + self.Out( + 'Warning: Translation for %s has wrong # of cliques, skipping.\n' % + node_id) + continue + else: + self.Out('Warning: No translation for %s, skipping.\n' % node_id) + continue + + if source_node.name == 'message': + # Fixup placeholders as well as possible based on information from + # the current .grd file if they are 'TODO_XXXX' placeholders. We need + # to fixup placeholders in the translated message so that it looks right + # and we also need to fixup placeholders in the source message so that + # its calculated ID will match the current message. + current_node = current_grd.GetNodeById(node_id) + if current_node: + assert len(source_cliques) == 1 and len(current_node.GetCliques()) == 1 + + source_msg = source_cliques[0].GetMessage() + current_msg = current_node.GetCliques()[0].GetMessage() + + # Only do this for messages whose source version has not changed. + if (source_msg.GetRealContent() != current_msg.GetRealContent()): + self.VerboseOut('Info: Message %s has changed; skipping\n' % node_id) + else: + transl_msg = transl_cliques[0].GetMessage() + transl_content = transl_msg.GetContent() + current_content = current_msg.GetContent() + source_content = source_msg.GetContent() + + ok_to_fixup = True + if (len(transl_content) != len(current_content)): + # message structure of translation is different, don't try fixup + ok_to_fixup = False + if ok_to_fixup: + for ix in range(len(transl_content)): + if isinstance(transl_content[ix], tclib.Placeholder): + if not isinstance(current_content[ix], tclib.Placeholder): + ok_to_fixup = False # structure has changed + break + if (transl_content[ix].GetOriginal() != + current_content[ix].GetOriginal()): + ok_to_fixup = False # placeholders have likely been reordered + break + else: # translated part is not a placeholder but a string + if isinstance(current_content[ix], tclib.Placeholder): + ok_to_fixup = False # placeholders have likely been reordered + break + + if not ok_to_fixup: + self.VerboseOut( + 'Info: Structure of message %s has changed; skipping.\n' % node_id) + else: + def Fixup(content, ix): + if (isinstance(content[ix], tclib.Placeholder) and + content[ix].GetPresentation().startswith('TODO_')): + assert isinstance(current_content[ix], tclib.Placeholder) + # Get the placeholder ID and example from the current message + content[ix] = current_content[ix] + for ix in range(len(transl_content)): + Fixup(transl_content, ix) + Fixup(source_content, ix) + + # Only put each translation once into the map. Warn if translations + # for the same message are different. + for ix in range(len(transl_cliques)): + source_msg = source_cliques[ix].GetMessage() + source_msg.GenerateId() # needed to refresh ID based on new placeholders + message_id = source_msg.GetId() + translated_content = transl_cliques[ix].GetMessage().GetPresentableContent() + + if message_id in id2transl: + existing_translation = id2transl[message_id] + if existing_translation != translated_content: + original_text = source_cliques[ix].GetMessage().GetPresentableContent() + self.Out('Warning: Two different translations for "%s":\n' + ' Translation 1: "%s"\n' + ' Translation 2: "%s"\n' % + (original_text, existing_translation, translated_content)) + else: + id2transl[message_id] = translated_content + + # Remove translations for messages that do not occur in the current .grd + # or have been marked as not translateable, or do not occur in the 'limits' + # list (if it has been set). + current_message_ids = current_grd.UberClique().AllMessageIds() + for message_id in id2transl.keys(): + if (message_id not in current_message_ids or + not current_grd.UberClique().BestClique(message_id).IsTranslateable() or + (self.limits and message_id not in self.limits)): + del id2transl[message_id] + + return id2transl + + # static method + def WriteTranslations(output_file, translations): + '''Writes the provided list of translations to the provided output file + in the format used by the TC's Bulk Translation Upload tool. The file + must be UTF-8 encoded. + + Args: + output_file: util.WrapOutputStream(file('bingo.out', 'w')) + translations: [ [id1, text1], ['12345678', 'Hello USERNAME, howzit?'] ] + + Return: + None + ''' + for id, text in translations: + text = text.replace('<', '<').replace('>', '>') + output_file.write(id) + output_file.write(' ') + output_file.write(text) + output_file.write('\n') + WriteTranslations = staticmethod(WriteTranslations) diff --git a/tools/grit/grit/tool/transl2tc_unittest.py b/tools/grit/grit/tool/transl2tc_unittest.py new file mode 100644 index 0000000..f0b97d5 --- /dev/null +++ b/tools/grit/grit/tool/transl2tc_unittest.py @@ -0,0 +1,155 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for the 'grit transl2tc' tool.''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import StringIO +import unittest + +from grit.tool import transl2tc +from grit import grd_reader +from grit import util + + +def MakeOptions(): + from grit import grit_runner + return grit_runner.Options() + + +class TranslationToTcUnittest(unittest.TestCase): + + def testOutput(self): + buf = StringIO.StringIO() + tool = transl2tc.TranslationToTc() + translations = [ + ['1', 'Hello USERNAME, how are you?'], + ['12', 'Howdie doodie!'], + ['123', 'Hello\n\nthere\n\nhow are you?'], + ['1234', 'Hello is > goodbye but < howdie pardner'], + ] + tool.WriteTranslations(buf, translations) + output = buf.getvalue() + self.failUnless(output.strip() == ''' +1 Hello USERNAME, how are you? +12 Howdie doodie! +123 Hello + +there + +how are you? +1234 Hello is > goodbye but < howdie pardner +'''.strip()) + + def testExtractTranslations(self): + path = util.PathFromRoot('grit/test/data') + current_grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <messages> + <message name="IDS_SIMPLE"> + One + </message> + <message name="IDS_PLACEHOLDER"> + <ph name="NUMBIRDS">%s<ex>3</ex></ph> birds + </message> + <message name="IDS_PLACEHOLDERS"> + <ph name="ITEM">%d<ex>1</ex></ph> of <ph name="COUNT">%d<ex>3</ex></ph> + </message> + <message name="IDS_REORDERED_PLACEHOLDERS"> + <ph name="ITEM">$1<ex>1</ex></ph> of <ph name="COUNT">$2<ex>3</ex></ph> + </message> + <message name="IDS_CHANGED"> + This is the new version + </message> + <message name="IDS_TWIN_1">Hello</message> + <message name="IDS_TWIN_2">Hello</message> + <message name="IDS_NOT_TRANSLATEABLE" translateable="false">:</message> + <message name="IDS_LONGER_TRANSLATED"> + Removed document <ph name="FILENAME">$1<ex>c:\temp</ex></ph> + </message> + <message name="IDS_DIFFERENT_TWIN_1">Howdie</message> + <message name="IDS_DIFFERENT_TWIN_2">Howdie</message> + </messages> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" encoding="utf-16" file="klonk.rc" /> + <structure type="menu" name="IDC_KLONKMENU" encoding="utf-16" file="klonk.rc" /> + </structures> + </release> + </grit>'''), path) + current_grd.RunGatherers(recursive=True) + + source_rc_path = util.PathFromRoot('grit/test/data/source.rc') + source_rc = file(source_rc_path).read() + transl_rc_path = util.PathFromRoot('grit/test/data/transl.rc') + transl_rc = file(transl_rc_path).read() + + tool = transl2tc.TranslationToTc() + output_buf = StringIO.StringIO() + globopts = MakeOptions() + globopts.verbose = True + globopts.output_stream = output_buf + tool.Setup(globopts, []) + translations = tool.ExtractTranslations(current_grd, + source_rc, source_rc_path, + transl_rc, transl_rc_path) + + values = translations.values() + output = output_buf.getvalue() + + self.failUnless('Ein' in values) + self.failUnless('NUMBIRDS Vogeln' in values) + self.failUnless('ITEM von COUNT' in values) + self.failUnless(values.count('Hallo') == 1) + self.failIf('Dass war die alte Version' in values) + self.failIf(':' in values) + self.failIf('Dokument FILENAME ist entfernt worden' in values) + self.failIf('Nicht verwendet' in values) + self.failUnless(('Howdie' in values or 'Hallo sagt man' in values) and not + ('Howdie' in values and 'Hallo sagt man' in values)) + + self.failUnless('XX01XX&SkraXX02XX&HaettaXX03XXThetta er "Klonk" sem eg fylaXX04XXgonkurinnXX05XXKlonk && er "gott"XX06XX&HjalpXX07XX&Um...XX08XX' in values) + + self.failUnless('I lagi' in values) + + self.failUnless(output.count('Structure of message IDS_REORDERED_PLACEHOLDERS has changed')) + self.failUnless(output.count('Message IDS_CHANGED has changed')) + self.failUnless(output.count('Structure of message IDS_LONGER_TRANSLATED has changed')) + self.failUnless(output.count('Two different translations for "Howdie"')) + self.failUnless(output.count('IDD_DIFFERENT_LENGTH_IN_TRANSL has wrong # of cliques')) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/tool/unit.py b/tools/grit/grit/tool/unit.py new file mode 100644 index 0000000..b021293 --- /dev/null +++ b/tools/grit/grit/tool/unit.py @@ -0,0 +1,50 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''GRIT tool that runs the unit test suite for GRIT.''' + + +import unittest + +import grit.test_suite_all +from grit.tool import interface + + +class UnitTestTool(interface.Tool): + '''By using this tool (e.g. 'grit unit') you run all the unit tests for GRIT. +This happens in the environment that is set up by the basic GRIT runner, i.e. +whether to run disconnected has been specified, etc.''' + + def ShortDescription(self): + return 'Use this tool to run all the unit tests for GRIT.' + + def Run(self, opts, args): + return unittest.TextTestRunner(verbosity=2).run( + grit.test_suite_all.TestSuiteAll()) diff --git a/tools/grit/grit/util.py b/tools/grit/grit/util.py new file mode 100644 index 0000000..08b1cb1 --- /dev/null +++ b/tools/grit/grit/util.py @@ -0,0 +1,334 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Utilities used by GRIT. +''' + +import sys +import os.path +import codecs +import htmlentitydefs +import re +import time +from xml.sax import saxutils + +_root_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) + + +# Matches all of the resource IDs predefined by Windows. +# The '\b' before and after each word makes sure these match only whole words and +# not the beginning of any word.. eg. ID_FILE_NEW will not match ID_FILE_NEW_PROJECT +# see http://www.amk.ca/python/howto/regex/ (search for "\bclass\b" inside the html page) +SYSTEM_IDENTIFIERS = re.compile( + r'''\bIDOK\b | \bIDCANCEL\b | \bIDC_STATIC\b | \bIDYES\b | \bIDNO\b | + \bID_FILE_NEW\b | \bID_FILE_OPEN\b | \bID_FILE_CLOSE\b | \bID_FILE_SAVE\b | + \bID_FILE_SAVE_AS\b | \bID_FILE_PAGE_SETUP\b | \bID_FILE_PRINT_SETUP\b | + \bID_FILE_PRINT\b | \bID_FILE_PRINT_DIRECT\b | \bID_FILE_PRINT_PREVIEW\b | + \bID_FILE_UPDATE\b | \bID_FILE_SAVE_COPY_AS\b | \bID_FILE_SEND_MAIL\b | + \bID_FILE_MRU_FIRST\b | \bID_FILE_MRU_LAST\b | + \bID_EDIT_CLEAR\b | \bID_EDIT_CLEAR_ALL\b | \bID_EDIT_COPY\b | + \bID_EDIT_CUT\b | \bID_EDIT_FIND\b | \bID_EDIT_PASTE\b | \bID_EDIT_PASTE_LINK\b | + \bID_EDIT_PASTE_SPECIAL\b | \bID_EDIT_REPEAT\b | \bID_EDIT_REPLACE\b | + \bID_EDIT_SELECT_ALL\b | \bID_EDIT_UNDO\b | \bID_EDIT_REDO\b | + \bVS_VERSION_INFO\b | \bIDRETRY''', re.VERBOSE); + + +# Matches character entities, whether specified by name, decimal or hex. +_HTML_ENTITY = re.compile( + '&(#(?P<decimal>[0-9]+)|#x(?P<hex>[a-fA-F0-9]+)|(?P<named>[a-z0-9]+));', + re.IGNORECASE) + +# Matches characters that should be HTML-escaped. This is <, > and &, but only +# if the & is not the start of an HTML character entity. +_HTML_CHARS_TO_ESCAPE = re.compile('"|<|>|&(?!#[0-9]+|#x[0-9a-z]+|[a-z]+;)', + re.IGNORECASE | re.MULTILINE) + + +def WrapInputStream(stream, encoding = 'utf-8'): + '''Returns a stream that wraps the provided stream, making it read characters + using the specified encoding.''' + (e, d, sr, sw) = codecs.lookup(encoding) + return sr(stream) + + +def WrapOutputStream(stream, encoding = 'utf-8'): + '''Returns a stream that wraps the provided stream, making it write + characters using the specified encoding.''' + (e, d, sr, sw) = codecs.lookup(encoding) + return sw(stream) + + +def ChangeStdoutEncoding(encoding = 'utf-8'): + '''Changes STDOUT to print characters using the specified encoding.''' + sys.stdout = WrapOutputStream(sys.stdout, encoding) + + +def EscapeHtml(text, escape_quotes = False): + '''Returns 'text' with <, > and & (and optionally ") escaped to named HTML + entities. Any existing named entity or HTML entity defined by decimal or + hex code will be left untouched. This is appropriate for escaping text for + inclusion in HTML, but not for XML. + ''' + def Replace(match): + if match.group() == '&': return '&' + elif match.group() == '<': return '<' + elif match.group() == '>': return '>' + elif match.group() == '"': + if escape_quotes: return '"' + else: return match.group() + else: assert False + out = _HTML_CHARS_TO_ESCAPE.sub(Replace, text) + return out + + +def UnescapeHtml(text, replace_nbsp=True): + '''Returns 'text' with all HTML character entities (both named character + entities and those specified by decimal or hexadecimal Unicode ordinal) + replaced by their Unicode characters (or latin1 characters if possible). + + The only exception is that will not be escaped if 'replace_nbsp' is + False. + ''' + def Replace(match): + groups = match.groupdict() + if groups['hex']: + return unichr(int(groups['hex'], 16)) + elif groups['decimal']: + return unichr(int(groups['decimal'], 10)) + else: + name = groups['named'] + if name == 'nbsp' and not replace_nbsp: + return match.group() # Don't replace + assert name != None + if name in htmlentitydefs.name2codepoint.keys(): + return unichr(htmlentitydefs.name2codepoint[name]) + else: + return match.group() # Unknown HTML character entity - don't replace + + out = _HTML_ENTITY.sub(Replace, text) + return out + + +def EncodeCdata(cdata): + '''Returns the provided cdata in either escaped format or <![CDATA[xxx]]> + format, depending on which is more appropriate for easy editing. The data + is escaped for inclusion in an XML element's body. + + Args: + cdata: 'If x < y and y < z then x < z' + + Return: + '<![CDATA[If x < y and y < z then x < z]]>' + ''' + if cdata.count('<') > 1 or cdata.count('>') > 1 and cdata.count(']]>') == 0: + return '<![CDATA[%s]]>' % cdata + else: + return saxutils.escape(cdata) + + +def FixupNamedParam(function, param_name, param_value): + '''Returns a closure that is identical to 'function' but ensures that the + named parameter 'param_name' is always set to 'param_value' unless explicitly + set by the caller. + + Args: + function: callable + param_name: 'bingo' + param_value: 'bongo' (any type) + + Return: + callable + ''' + def FixupClosure(*args, **kw): + if not param_name in kw: + kw[param_name] = param_value + return function(*args, **kw) + return FixupClosure + + +def PathFromRoot(path): + '''Takes a path relative to the root directory for GRIT (the one that grit.py + resides in) and returns a path that is either absolute or relative to the + current working directory (i.e .a path you can use to open the file). + + Args: + path: 'rel_dir\file.ext' + + Return: + 'c:\src\tools\rel_dir\file.ext + ''' + return os.path.normpath(os.path.join(_root_dir, path)) + + +def FixRootForUnittest(root_node, dir=PathFromRoot('.')): + '''Adds a GetBaseDir() method to 'root_node', making unittesting easier.''' + def GetBaseDir(): + '''Returns a fake base directory.''' + return dir + def GetSourceLanguage(): + return 'en' + if not hasattr(root_node, 'GetBaseDir'): + setattr(root_node, 'GetBaseDir', GetBaseDir) + setattr(root_node, 'GetSourceLanguage', GetSourceLanguage) + + +def dirname(filename): + '''Version of os.path.dirname() that never returns empty paths (returns + '.' if the result of os.path.dirname() is empty). + ''' + ret = os.path.dirname(filename) + if ret == '': + ret = '.' + return ret + + +def normpath(path): + '''Version of os.path.normpath that also changes backward slashes to + forward slashes when not running on Windows. + ''' + # This is safe to always do because the Windows version of os.path.normpath + # will replace forward slashes with backward slashes. + path = path.replace('\\', '/') + return os.path.normpath(path) + + +_LANGUAGE_SPLIT_RE = re.compile('-|_|/') + + +def CanonicalLanguage(code): + '''Canonicalizes two-part language codes by using a dash and making the + second part upper case. Returns one-part language codes unchanged. + + Args: + code: 'zh_cn' + + Return: + code: 'zh-CN' + ''' + parts = _LANGUAGE_SPLIT_RE.split(code) + code = [ parts[0] ] + for part in parts[1:]: + code.append(part.upper()) + return '-'.join(code) + + +_LANG_TO_CODEPAGE = { + 'en' : 1252, + 'fr' : 1252, + 'it' : 1252, + 'de' : 1252, + 'es' : 1252, + 'nl' : 1252, + 'sv' : 1252, + 'no' : 1252, + 'da' : 1252, + 'fi' : 1252, + 'pt-BR' : 1252, + 'ru' : 1251, + 'ja' : 932, + 'zh-TW' : 950, + 'zh-CN' : 936, + 'ko' : 949, +} + + +def LanguageToCodepage(lang): + '''Returns the codepage _number_ that can be used to represent 'lang', which + may be either in formats such as 'en', 'pt_br', 'pt-BR', etc. + + The codepage returned will be one of the 'cpXXXX' codepage numbers. + + Args: + lang: 'de' + + Return: + 1252 + ''' + lang = CanonicalLanguage(lang) + if lang in _LANG_TO_CODEPAGE: + return _LANG_TO_CODEPAGE[lang] + else: + print "Not sure which codepage to use for %s, assuming cp1252" % lang + return 1252 + +def NewClassInstance(class_name, class_type): + '''Returns an instance of the class specified in classname + + Args: + class_name: the fully qualified, dot separated package + classname, + i.e. "my.package.name.MyClass". Short class names are not supported. + class_type: the class or superclass this object must implement + + Return: + An instance of the class, or None if none was found + ''' + lastdot = class_name.rfind('.') + module_name = '' + if lastdot >= 0: + module_name = class_name[0:lastdot] + if module_name: + class_name = class_name[lastdot+1:] + module = __import__(module_name, globals(), locals(), ['']) + if hasattr(module, class_name): + class_ = getattr(module, class_name) + class_instance = class_() + if isinstance(class_instance, class_type): + return class_instance + return None + + +def FixLineEnd(text, line_end): + # First normalize + text = text.replace('\r\n', '\n') + text = text.replace('\r', '\n') + # Then fix + text = text.replace('\n', line_end) + return text + + +def BoolToString(bool): + if bool: + return 'true' + else: + return 'false' + + +verbose = False +extra_verbose = False + +def IsVerbose(): + return verbose + +def IsExtraVerbose(): + return extra_verbose + +def GetCurrentYear(): + '''Returns the current 4-digit year as an integer.''' + return time.localtime()[0] diff --git a/tools/grit/grit/util_unittest.py b/tools/grit/grit/util_unittest.py new file mode 100644 index 0000000..eb11067 --- /dev/null +++ b/tools/grit/grit/util_unittest.py @@ -0,0 +1,94 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit test that checks some of util functions. +''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + +from grit import util + + +class UtilUnittest(unittest.TestCase): + ''' Tests functions from util + ''' + + def testNewClassInstance(self): + # Test short class name with no fully qualified package name + # Should fail, it is not supported by the function now (as documented) + cls = util.NewClassInstance('grit.util.TestClassToLoad', + TestBaseClassToLoad) + self.failUnless(cls == None) + + # Test non existent class name + cls = util.NewClassInstance('grit.util_unittest.NotExistingClass', + TestBaseClassToLoad) + self.failUnless(cls == None) + + # Test valid class name and valid base class + cls = util.NewClassInstance('grit.util_unittest.TestClassToLoad', + TestBaseClassToLoad) + self.failUnless(isinstance(cls, TestBaseClassToLoad)) + + # Test valid class name with wrong hierarchy + cls = util.NewClassInstance('grit.util_unittest.TestClassNoBase', + TestBaseClassToLoad) + self.failUnless(cls == None) + + def testCanonicalLanguage(self): + self.failUnless(util.CanonicalLanguage('en') == 'en') + self.failUnless(util.CanonicalLanguage('pt_br') == 'pt-BR') + self.failUnless(util.CanonicalLanguage('pt-br') == 'pt-BR') + self.failUnless(util.CanonicalLanguage('pt-BR') == 'pt-BR') + self.failUnless(util.CanonicalLanguage('pt/br') == 'pt-BR') + self.failUnless(util.CanonicalLanguage('pt/BR') == 'pt-BR') + self.failUnless(util.CanonicalLanguage('no_no_bokmal') == 'no-NO-BOKMAL') + + def testUnescapeHtml(self): + self.failUnless(util.UnescapeHtml('ϲ') == unichr(1010)) + self.failUnless(util.UnescapeHtml('ꯍ') == unichr(43981)) + +class TestBaseClassToLoad(object): + pass + +class TestClassToLoad(TestBaseClassToLoad): + pass + +class TestClassNoBase(object): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/xtb_reader.py b/tools/grit/grit/xtb_reader.py new file mode 100644 index 0000000..01db8d9 --- /dev/null +++ b/tools/grit/grit/xtb_reader.py @@ -0,0 +1,123 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Fast and efficient parser for XTB files. +''' + + +import xml.sax +import xml.sax.handler + + +class XtbContentHandler(xml.sax.handler.ContentHandler): + '''A content handler that calls a given callback function for each + translation in the XTB file. + ''' + + def __init__(self, callback, debug=False): + self.callback = callback + self.debug = debug + # 0 if we are not currently parsing a translation, otherwise the message + # ID of that translation. + self.current_id = 0 + # Empty if we are not currently parsing a translation, otherwise the + # parts we have for that translation - a list of tuples + # (is_placeholder, text) + self.current_structure = [] + # Set to the language ID when we see the <translationbundle> node. + self.language = '' + + def startElement(self, name, attrs): + if name == 'translation': + assert (self.current_id == 0 and len(self.current_structure) == 0, + "Didn't expect a <translation> element here.") + self.current_id = attrs.getValue('id') + elif name == 'ph': + assert self.current_id != 0, "Didn't expect a <ph> element here." + self.current_structure.append((True, attrs.getValue('name'))) + elif name == 'translationbundle': + self.language = attrs.getValue('lang') + + def endElement(self, name): + if name == 'translation': + assert self.current_id != 0 + self.callback(self.current_id, self.current_structure) + self.current_id = 0 + self.current_structure = [] + + def characters(self, content): + if self.current_id != 0: + # We are inside a <translation> node so just add the characters to our + # structure. + # + # This naive way of handling characters is OK because in the XTB format, + # <ph> nodes are always empty (always <ph name="XXX"/>) and whitespace + # inside the <translation> node should be preserved. + self.current_structure.append((False, content)) + + +class XtbErrorHandler(xml.sax.handler.ErrorHandler): + def error(self, exception): + pass + + def fatalError(self, exception): + raise exception + + def warning(self, exception): + pass + + +def Parse(xtb_file, callback_function, debug=False): + '''Parse xtb_file, making a call to callback_function for every translation + in the XTB file. + + The callback function must have the signature as described below. The 'parts' + parameter is a list of tuples (is_placeholder, text). The 'text' part is + either the raw text (if is_placeholder is False) or the name of the placeholder + (if is_placeholder is True). + + Args: + xtb_file: file('fr.xtb') + callback_function: def Callback(msg_id, parts): pass + + Return: + The language of the XTB, e.g. 'fr' + ''' + # Start by advancing the file pointer past the DOCTYPE thing, as the TC + # uses a path to the DTD that only works in Unix. + # TODO(joi) Remove this ugly hack by getting the TC gang to change the + # XTB files somehow? + front_of_file = xtb_file.read(1024) + xtb_file.seek(front_of_file.find('<translationbundle')) + + handler = XtbContentHandler(callback=callback_function, debug=debug) + xml.sax.parse(xtb_file, handler) + assert handler.language != '' + return handler.language diff --git a/tools/grit/grit/xtb_reader_unittest.py b/tools/grit/grit/xtb_reader_unittest.py new file mode 100644 index 0000000..c7beb1d --- /dev/null +++ b/tools/grit/grit/xtb_reader_unittest.py @@ -0,0 +1,108 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.xtb_reader''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import StringIO +import unittest + +from grit import xtb_reader +from grit import clique +from grit import grd_reader +from grit import tclib +from grit import util + + +class XtbReaderUnittest(unittest.TestCase): + def testParsing(self): + xtb_file = StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE translationbundle> + <translationbundle lang="fr"> + <translation id="5282608565720904145">Bingo.</translation> + <translation id="2955977306445326147">Bongo longo.</translation> + <translation id="238824332917605038">Hullo</translation> + <translation id="6629135689895381486"><ph name="PROBLEM_REPORT"/> peut <ph name="START_LINK"/>utilisation excessive de majuscules<ph name="END_LINK"/>.</translation> + <translation id="7729135689895381486">Hello +this is another line +and another + +and another after a blank line.</translation> + </translationbundle>''') + + messages = [] + def Callback(id, structure): + messages.append((id, structure)) + xtb_reader.Parse(xtb_file, Callback) + self.failUnless(len(messages[0][1]) == 1) + self.failUnless(messages[3][1][0]) # PROBLEM_REPORT placeholder + self.failUnless(messages[4][0] == '7729135689895381486') + self.failUnless(messages[4][1][7][1] == 'and another after a blank line.') + + def testParsingIntoMessages(self): + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <messages> + <message name="ID_MEGA">Fantastic!</message> + <message name="ID_HELLO_USER">Hello <ph name="USERNAME">%s<ex>Joi</ex></ph></message> + </messages>'''), dir='.', flexible_root=True) + + clique_mega = grd.children[0].GetCliques()[0] + msg_mega = clique_mega.GetMessage() + clique_hello_user = grd.children[1].GetCliques()[0] + msg_hello_user = clique_hello_user.GetMessage() + + xtb_file = StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE translationbundle> + <translationbundle lang="is"> + <translation id="%s">Meirihattar!</translation> + <translation id="%s">Saelir <ph name="USERNAME"/></translation> + </translationbundle>''' % (msg_mega.GetId(), msg_hello_user.GetId())) + + xtb_reader.Parse(xtb_file, grd.UberClique().GenerateXtbParserCallback('is')) + self.failUnless(clique_mega.MessageForLanguage('is').GetRealContent() == + 'Meirihattar!') + self.failUnless(clique_hello_user.MessageForLanguage('is').GetRealContent() == + 'Saelir %s') + + def testParseLargeFile(self): + def Callback(id, structure): + pass + xtb = file(util.PathFromRoot('grit/test/data/fr.xtb')) + xtb_reader.Parse(xtb, Callback) + xtb.close() + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/measure_page_load_time/ff_ext/chrome.manifest b/tools/measure_page_load_time/ff_ext/chrome.manifest new file mode 100644 index 0000000..9e1d73d4 --- /dev/null +++ b/tools/measure_page_load_time/ff_ext/chrome.manifest @@ -0,0 +1,2 @@ +content measurepageloadtimeextension content/
+overlay chrome://browser/content/browser.xul chrome://measurepageloadtimeextension/content/firefoxOverlay.xul
diff --git a/tools/measure_page_load_time/ff_ext/content/firefoxOverlay.xul b/tools/measure_page_load_time/ff_ext/content/firefoxOverlay.xul new file mode 100644 index 0000000..1c5529e --- /dev/null +++ b/tools/measure_page_load_time/ff_ext/content/firefoxOverlay.xul @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet href="chrome://measurepageloadtimeextension/skin/overlay.css" type="text/css"?>
+<!DOCTYPE overlay SYSTEM "chrome://measurepageloadtimeextension/locale/measurepageloadtimeextension.dtd">
+<overlay id="measurepageloadtimeextension-overlay"
+ xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
+ <script src="measure_page_load_time.js"/>
+</overlay>
diff --git a/tools/measure_page_load_time/ff_ext/content/measure_page_load_time.js b/tools/measure_page_load_time/ff_ext/content/measure_page_load_time.js new file mode 100644 index 0000000..d9a6ca6 --- /dev/null +++ b/tools/measure_page_load_time/ff_ext/content/measure_page_load_time.js @@ -0,0 +1,207 @@ +// Copyright 2008 Google Inc. +// All Rights Reserved. +/** + * @fileoverview measure_page_load_time.js implements a Firefox extension + * for measuring how long a page takes to load. It waits on TCP port + * 42492 for connections, then accepts URLs and returns strings of the + * form url,time, where "time" is the load time in milliseconds or the + * string "timeout" or "error". Load time is measured from the call to + * loadURI until the load event fires, or until the status changes to + * STATUS_STOP if the load event doesn't fire (there's an error.) + * @author jhaas@google.com (Jonathan Haas) */ + +// Shorthand reference to nsIWebProgress[Listener] interfaces +var IWP = Components.interfaces.nsIWebProgress; +var IWPL = Components.interfaces.nsIWebProgressListener; + + +var MPLT = { + /** + * Constants + */ + PORT_NUMBER : 42492, // port to listen for connections on + TIME_OUT : 4 * 60 * 1000, // timeout in 4 minutes + + /** + * Incoming URL buffer + * @type {string} + */ + textBuffer : '', + + /** + * URL we're currently visiting + * @type {string} + */ + URL : '', + + /** + * Listener to accept incoming connections + * @type {nsIServerSocketListener} + */ + acceptListener : + { + onSocketAccepted : function(serverSocket, transport) + { + MPLT.streamInput = transport.openInputStream(0,0,0); + MPLT.streamOutput = transport.openOutputStream(0,0,0); + + MPLT.scriptStream = Components.classes['@mozilla.org/scriptableinputstream;1'] + .createInstance(Components.interfaces.nsIScriptableInputStream); + MPLT.scriptStream.init(MPLT.streamInput); + MPLT.pump = Components.classes['@mozilla.org/network/input-stream-pump;1'] + .createInstance(Components.interfaces.nsIInputStreamPump); + MPLT.pump.init(MPLT.streamInput, -1, -1, 0, 0, false); + MPLT.pump.asyncRead(MPLT.dataListener,null); + }, + + onStopListening : function(){} + }, + + /** + * Listener for network input + * @type {nsIStreamListener} + */ + dataListener : + { + onStartRequest: function(){}, + onStopRequest: function(){}, + onDataAvailable: function(request, context, inputStream, offset, count){ + // Add the received data to the buffer, then process it + // Change CRLF to newline while we're at it + MPLT.textBuffer += MPLT.scriptStream.read(count).replace('\r\n', '\n'); + + MPLT.process(); + } + }, + + /** + * Process the incoming data buffer + */ + process : function() + { + // If we're waiting for a page to finish loading, wait + if (MPLT.timeLoadStarted) + return; + + // Look for a carriage return + var firstCR = MPLT.textBuffer.indexOf('\n'); + + // If we haven't received a carriage return yet, wait + if (firstCR < 0) + return; + + // If the first character was a carriage return, we're done! + if (firstCR == 0) { + MPLT.textBuffer = ''; + MPLT.streamInput.close(); + MPLT.streamOutput.close(); + + return; + } + + // Remove the URL from the buffer + MPLT.URL = MPLT.textBuffer.substr(0, firstCR); + MPLT.textBuffer = MPLT.textBuffer.substr(firstCR + 1); + + // Remember the current time and navigate to the new URL + MPLT.timeLoadStarted = new Date(); + gBrowser.loadURIWithFlags(MPLT.URL, gBrowser.LOAD_FLAGS_BYPASS_CACHE); + setTimeout('MPLT.onTimeOut()', MPLT.TIME_OUT); + }, + + /** + * Page load completion handler + */ + onPageLoad : function(e) { + // Ignore loads of non-HTML documents + if (!(e.originalTarget instanceof HTMLDocument)) + return; + + // Also ignore subframe loads + if (e.originalTarget.defaultView.frameElement) + return; + + clearTimeout(); + var timeElapsed = new Date() - MPLT.timeLoadStarted; + + MPLT.outputResult(timeElapsed); + }, + + /** + * Timeout handler + */ + onTimeOut : function() { + gBrowser.stop(); + + MPLT.outputResult('timeout'); + }, + + + /** + * Sends a properly-formatted result to the client + * @param {string} result The value to send along with the URL + */ + outputResult : function(result) { + + if (MPLT.URL) { + var outputString = MPLT.URL + ',' + result + '\n'; + MPLT.streamOutput.write(outputString, outputString.length); + MPLT.URL = ''; + } + + MPLT.timeLoadStarted = null; + MPLT.process(); + }, + + /** + * Time the page load started. If null, we're waiting for the + * initial page load, or otherwise don't care about the page + * that's currently loading + * @type {number} + */ + timeLoadStarted : null, + + /* + * TODO(jhaas): add support for nsIWebProgressListener + * If the URL being visited died as part of a network error + * (host not found, connection reset by peer, etc), the onload + * event doesn't fire. The only way to catch it would be in + * a web progress listener. However, nsIWebProgress is not + * behaving according to documentation. More research is needed. + * For now, omitting it means that if any of our URLs are "dirty" + * (do not point to real web servers with real responses), we'll log + * them as timeouts. This doesn't affect pages where the server + * exists but returns an error code. + */ + + /** + * Initialize the plugin, create the socket and listen + */ + initialize: function() { + // Register for page load events + gBrowser.addEventListener('load', this.onPageLoad, true); + + // Set a timeout to wait for the initial page to load + MPLT.timeLoadStarted = new Date(); + setTimeout('MPLT.onTimeOut()', MPLT.TIME_OUT); + + // Create the listening socket + MPLT.serverSocket = Components.classes['@mozilla.org/network/server-socket;1'] + .createInstance(Components.interfaces.nsIServerSocket); + + MPLT.serverSocket.init(MPLT.PORT_NUMBER, true, 1); + MPLT.serverSocket.asyncListen(this.acceptListener); + }, + + /** + * Close the socket(s) + */ + deinitialize: function() { + if (MPLT.streamInput) MPLT.streamInput.close(); + if (MPLT.streamOutput) MPLT.streamOutput.close(); + if (MPLT.serverSocket) MPLT.serverSocket.close(); + } +}; + +window.addEventListener('load', function(e) { MPLT.initialize(); }, false); +window.addEventListener('unload', function(e) { MPLT.deinitialize(); }, false); diff --git a/tools/measure_page_load_time/ff_ext/install.rdf b/tools/measure_page_load_time/ff_ext/install.rdf new file mode 100644 index 0000000..70d510a --- /dev/null +++ b/tools/measure_page_load_time/ff_ext/install.rdf @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+ <Description about="urn:mozilla:install-manifest">
+ <em:id>measurepageloadtimeextension@google.com</em:id>
+ <em:name>MeasurePageLoadTime</em:name>
+ <em:version>1.0</em:version>
+ <em:creator>Jonathan Haas</em:creator>
+ <em:targetApplication>
+ <Description>
+ <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id> <!-- firefox -->
+ <em:minVersion>1.5</em:minVersion>
+ <em:maxVersion>2.0.0.*</em:maxVersion>
+ </Description>
+ </em:targetApplication>
+ </Description>
+</RDF>
diff --git a/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.cpp b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.cpp new file mode 100644 index 0000000..55f4de2 --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.cpp @@ -0,0 +1,97 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// MeasurePageLoadTime.cpp : Implementation of DLL Exports. + +#include "stdafx.h" +#include "resource.h" +#include "MeasurePageLoadTime.h" + + +class CMeasurePageLoadTimeModule : public CAtlDllModuleT< CMeasurePageLoadTimeModule > +{ +public : + DECLARE_LIBID(LIBID_MeasurePageLoadTimeLib) + DECLARE_REGISTRY_APPID_RESOURCEID(IDR_MEASUREPAGELOADTIME, "{56C6D9F9-643C-4F6E-906C-5F7CECB23C24}") +}; + +CMeasurePageLoadTimeModule _AtlModule; + + +#ifdef _MANAGED +#pragma managed(push, off) +#endif + +// DLL Entry Point +extern "C" BOOL WINAPI DllMain(HINSTANCE hInstance, DWORD dwReason, LPVOID lpReserved) +{ + if (dwReason == DLL_PROCESS_ATTACH) + { + DisableThreadLibraryCalls(hInstance); + } + return _AtlModule.DllMain(dwReason, lpReserved); +} + +#ifdef _MANAGED +#pragma managed(pop) +#endif + + + + +// Used to determine whether the DLL can be unloaded by OLE +STDAPI DllCanUnloadNow(void) +{ + return _AtlModule.DllCanUnloadNow(); +} + + +// Returns a class factory to create an object of the requested type +STDAPI DllGetClassObject(REFCLSID rclsid, REFIID riid, LPVOID* ppv) +{ + return _AtlModule.DllGetClassObject(rclsid, riid, ppv); +} + + +// DllRegisterServer - Adds entries to the system registry +STDAPI DllRegisterServer(void) +{ + // registers object, typelib and all interfaces in typelib + HRESULT hr = _AtlModule.DllRegisterServer(); + return hr; +} + + +// DllUnregisterServer - Removes entries from the system registry +STDAPI DllUnregisterServer(void) +{ + HRESULT hr = _AtlModule.DllUnregisterServer(); + return hr; +} + diff --git a/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.def b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.def new file mode 100644 index 0000000..5552923 --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.def @@ -0,0 +1,9 @@ +; MeasurePageLoadTime.def : Declares the module parameters. + +LIBRARY "MeasurePageLoadTime.DLL" + +EXPORTS + DllCanUnloadNow PRIVATE + DllGetClassObject PRIVATE + DllRegisterServer PRIVATE + DllUnregisterServer PRIVATE diff --git a/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.idl b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.idl new file mode 100644 index 0000000..d0c8270 --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.idl @@ -0,0 +1,65 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// MeasurePageLoadTime.idl : IDL source for MeasurePageLoadTime +// + +// This file will be processed by the MIDL tool to +// produce the type library (MeasurePageLoadTime.tlb) and marshalling code. + +import "oaidl.idl"; +import "ocidl.idl"; + +[ + object, + uuid(019637EB-B865-485B-9A66-419477EE55A0), + dual, + nonextensible, + helpstring("IMeasurePageLoadTimeBHO Interface"), + pointer_default(unique) +] +interface IMeasurePageLoadTimeBHO : IDispatch{ +}; +[ + uuid(61AC7AC4-B715-4955-A238-5F9AEA80DF4B), + version(1.0), + helpstring("MeasurePageLoadTime 1.0 Type Library") +] +library MeasurePageLoadTimeLib +{ + importlib("stdole2.tlb"); + [ + uuid(807E68BC-238F-4163-AE4B-0A3604F3E145), + helpstring("MeasurePageLoadTimeBHO Class") + ] + coclass MeasurePageLoadTimeBHO + { + [default] interface IMeasurePageLoadTimeBHO; + }; +}; diff --git a/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.rc b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.rc new file mode 100644 index 0000000..9285a70 --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.rc @@ -0,0 +1,121 @@ +// Microsoft Visual C++ generated resource script. +// +#include "resource.h" + +#define APSTUDIO_READONLY_SYMBOLS +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 2 resource. +// +#include "winres.h" + +///////////////////////////////////////////////////////////////////////////// +#undef APSTUDIO_READONLY_SYMBOLS + +///////////////////////////////////////////////////////////////////////////// +// English (U.S.) resources + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +#ifdef _WIN32 +LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US +#pragma code_page(1252) +#endif //_WIN32 + +#ifdef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// TEXTINCLUDE +// + +1 TEXTINCLUDE +BEGIN + "resource.h\0" +END + +2 TEXTINCLUDE +BEGIN + "#include ""winres.h""\r\n" + "\0" +END + +3 TEXTINCLUDE +BEGIN + "1 TYPELIB ""MeasurePageLoadTime.tlb""\r\n" + "\0" +END + +#endif // APSTUDIO_INVOKED + + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +VS_VERSION_INFO VERSIONINFO + FILEVERSION 1,0,0,1 + PRODUCTVERSION 1,0,0,1 + FILEFLAGSMASK 0x3fL +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS 0x4L + FILETYPE 0x2L + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904e4" + BEGIN + VALUE "CompanyName", "Google" + VALUE "FileDescription", "Measures page load times" + VALUE "FileVersion", "1.0.0.1" + VALUE "LegalCopyright", "(c) 2008 Google. All rights reserved." + VALUE "InternalName", "MeasurePageLoadTime.dll" + VALUE "OriginalFilename", "MeasurePageLoadTime.dll" + VALUE "ProductName", "MeasurePageLoadTime" + VALUE "ProductVersion", "1.0.0.1" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1252 + END +END + + +///////////////////////////////////////////////////////////////////////////// +// +// REGISTRY +// + +IDR_MEASUREPAGELOADTIME REGISTRY "MeasurePageLoadTime.rgs" +IDR_MEASUREPAGELOADTIMEBHO REGISTRY "MeasurePageLoadTimeBHO.rgs" + +///////////////////////////////////////////////////////////////////////////// +// +// String Table +// + +STRINGTABLE +BEGIN + IDS_PROJNAME "MeasurePageLoadTime" +END + +#endif // English (U.S.) resources +///////////////////////////////////////////////////////////////////////////// + + + +#ifndef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 3 resource. +// +1 TYPELIB "MeasurePageLoadTime.tlb" + +///////////////////////////////////////////////////////////////////////////// +#endif // not APSTUDIO_INVOKED + diff --git a/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.rgs b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.rgs new file mode 100644 index 0000000..98e7f78 --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.rgs @@ -0,0 +1,29 @@ +HKCR
+{
+ NoRemove AppID
+ {
+ '%APPID%' = s 'MeasurePageLoadTime'
+ 'MeasurePageLoadTime.DLL'
+ {
+ val AppID = s '%APPID%'
+ }
+ }
+}
+
+HKLM {
+ NoRemove SOFTWARE {
+ NoRemove Microsoft {
+ NoRemove Windows {
+ NoRemove CurrentVersion {
+ NoRemove Explorer {
+ NoRemove 'Browser Helper Objects' {
+ ForceRemove '{807E68BC-238F-4163-AE4B-0A3604F3E145}' = s 'MeasurePageLoadTimeBHO' {
+ val 'NoExplorer' = d '1'
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.vcproj b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.vcproj new file mode 100644 index 0000000..9ed8327 --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.vcproj @@ -0,0 +1,320 @@ +<?xml version="1.0" encoding="Windows-1252"?> +<VisualStudioProject + ProjectType="Visual C++" + Version="8.00" + Name="MeasurePageLoadTime" + ProjectGUID="{151243DF-25BE-4A88-B566-8B7AE8970E86}" + RootNamespace="MeasurePageLoadTime" + Keyword="AtlProj" + > + <Platforms> + <Platform + Name="Win32" + /> + </Platforms> + <ToolFiles> + </ToolFiles> + <Configurations> + <Configuration + Name="Debug|Win32" + OutputDirectory="$(ConfigurationName)" + IntermediateDirectory="$(ConfigurationName)" + ConfigurationType="2" + UseOfATL="2" + ATLMinimizesCRunTimeLibraryUsage="false" + CharacterSet="1" + > + <Tool + Name="VCPreBuildEventTool" + /> + <Tool + Name="VCCustomBuildTool" + /> + <Tool + Name="VCXMLDataGeneratorTool" + /> + <Tool + Name="VCWebServiceProxyGeneratorTool" + /> + <Tool + Name="VCMIDLTool" + PreprocessorDefinitions="_DEBUG" + MkTypLibCompatible="false" + TargetEnvironment="1" + GenerateStublessProxies="true" + TypeLibraryName="$(IntDir)/MeasurePageLoadTime.tlb" + HeaderFileName="MeasurePageLoadTime.h" + DLLDataFileName="" + InterfaceIdentifierFileName="MeasurePageLoadTime_i.c" + ProxyFileName="MeasurePageLoadTime_p.c" + ValidateParameters="false" + /> + <Tool + Name="VCCLCompilerTool" + Optimization="0" + PreprocessorDefinitions="WIN32;_WINDOWS;_DEBUG;_USRDLL" + MinimalRebuild="true" + BasicRuntimeChecks="3" + RuntimeLibrary="3" + UsePrecompiledHeader="2" + WarningLevel="3" + Detect64BitPortabilityProblems="true" + DebugInformationFormat="4" + /> + <Tool + Name="VCManagedResourceCompilerTool" + /> + <Tool + Name="VCResourceCompilerTool" + PreprocessorDefinitions="_DEBUG" + Culture="1033" + AdditionalIncludeDirectories="$(IntDir)" + /> + <Tool + Name="VCPreLinkEventTool" + /> + <Tool + Name="VCLinkerTool" + RegisterOutput="true" + IgnoreImportLibrary="true" + AdditionalDependencies="ws2_32.lib" + LinkIncremental="2" + ModuleDefinitionFile=".\MeasurePageLoadTime.def" + GenerateDebugInformation="true" + SubSystem="2" + TargetMachine="1" + /> + <Tool + Name="VCALinkTool" + /> + <Tool + Name="VCManifestTool" + /> + <Tool + Name="VCXDCMakeTool" + /> + <Tool + Name="VCBscMakeTool" + /> + <Tool + Name="VCFxCopTool" + /> + <Tool + Name="VCAppVerifierTool" + /> + <Tool + Name="VCWebDeploymentTool" + /> + <Tool + Name="VCPostBuildEventTool" + /> + </Configuration> + <Configuration + Name="Release|Win32" + OutputDirectory="$(ConfigurationName)" + IntermediateDirectory="$(ConfigurationName)" + ConfigurationType="2" + UseOfATL="1" + ATLMinimizesCRunTimeLibraryUsage="false" + CharacterSet="1" + > + <Tool + Name="VCPreBuildEventTool" + /> + <Tool + Name="VCCustomBuildTool" + /> + <Tool + Name="VCXMLDataGeneratorTool" + /> + <Tool + Name="VCWebServiceProxyGeneratorTool" + /> + <Tool + Name="VCMIDLTool" + PreprocessorDefinitions="NDEBUG" + MkTypLibCompatible="false" + TargetEnvironment="1" + GenerateStublessProxies="true" + TypeLibraryName="$(IntDir)/MeasurePageLoadTime.tlb" + HeaderFileName="MeasurePageLoadTime.h" + DLLDataFileName="" + InterfaceIdentifierFileName="MeasurePageLoadTime_i.c" + ProxyFileName="MeasurePageLoadTime_p.c" + ValidateParameters="false" + /> + <Tool + Name="VCCLCompilerTool" + Optimization="2" + PreprocessorDefinitions="WIN32;_WINDOWS;NDEBUG;_USRDLL" + RuntimeLibrary="0" + UsePrecompiledHeader="2" + WarningLevel="3" + Detect64BitPortabilityProblems="true" + DebugInformationFormat="3" + /> + <Tool + Name="VCManagedResourceCompilerTool" + /> + <Tool + Name="VCResourceCompilerTool" + PreprocessorDefinitions="NDEBUG" + Culture="1033" + AdditionalIncludeDirectories="$(IntDir)" + /> + <Tool + Name="VCPreLinkEventTool" + /> + <Tool + Name="VCLinkerTool" + RegisterOutput="true" + IgnoreImportLibrary="true" + AdditionalDependencies="ws2_32.lib" + LinkIncremental="1" + ModuleDefinitionFile=".\MeasurePageLoadTime.def" + GenerateDebugInformation="true" + SubSystem="2" + OptimizeReferences="2" + EnableCOMDATFolding="2" + TargetMachine="1" + /> + <Tool + Name="VCALinkTool" + /> + <Tool + Name="VCManifestTool" + /> + <Tool + Name="VCXDCMakeTool" + /> + <Tool + Name="VCBscMakeTool" + /> + <Tool + Name="VCFxCopTool" + /> + <Tool + Name="VCAppVerifierTool" + /> + <Tool + Name="VCWebDeploymentTool" + /> + <Tool + Name="VCPostBuildEventTool" + /> + </Configuration> + </Configurations> + <References> + </References> + <Files> + <Filter + Name="Source Files" + Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx" + UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" + > + <File + RelativePath=".\MeasurePageLoadTime.cpp" + > + </File> + <File + RelativePath=".\MeasurePageLoadTime.def" + > + </File> + <File + RelativePath=".\MeasurePageLoadTime.idl" + > + </File> + <File + RelativePath=".\MeasurePageLoadTimeBHO.cpp" + > + </File> + <File + RelativePath=".\stdafx.cpp" + > + <FileConfiguration + Name="Debug|Win32" + > + <Tool + Name="VCCLCompilerTool" + UsePrecompiledHeader="1" + /> + </FileConfiguration> + <FileConfiguration + Name="Release|Win32" + > + <Tool + Name="VCCLCompilerTool" + UsePrecompiledHeader="1" + /> + </FileConfiguration> + </File> + </Filter> + <Filter + Name="Header Files" + Filter="h;hpp;hxx;hm;inl;inc;xsd" + UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}" + > + <File + RelativePath=".\MeasurePageLoadTimeBHO.h" + > + </File> + <File + RelativePath=".\stdafx.h" + > + </File> + </Filter> + <Filter + Name="Resource Files" + Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav" + UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}" + > + <File + RelativePath=".\MeasurePageLoadTime.rc" + > + </File> + <File + RelativePath=".\MeasurePageLoadTime.rgs" + > + </File> + <File + RelativePath=".\MeasurePageLoadTimeBHO.rgs" + > + </File> + </Filter> + <Filter + Name="Generated Files" + SourceControlFiles="false" + > + <File + RelativePath=".\MeasurePageLoadTime.h" + > + </File> + <File + RelativePath=".\MeasurePageLoadTime_i.c" + > + <FileConfiguration + Name="Debug|Win32" + > + <Tool + Name="VCCLCompilerTool" + UsePrecompiledHeader="0" + /> + </FileConfiguration> + <FileConfiguration + Name="Release|Win32" + > + <Tool + Name="VCCLCompilerTool" + UsePrecompiledHeader="0" + /> + </FileConfiguration> + </File> + </Filter> + <File + RelativePath=".\ReadMe.txt" + > + </File> + </Files> + <Globals> + </Globals> +</VisualStudioProject> diff --git a/tools/measure_page_load_time/ie_bho/MeasurePageLoadTimeBHO.cpp b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTimeBHO.cpp new file mode 100644 index 0000000..7e9d5c0 --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTimeBHO.cpp @@ -0,0 +1,317 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Implements a Browser Helper Object (BHO) which opens a socket +// and waits to receive URLs over it. Visits those URLs, measuring +// how long it takes between the start of navigation and the +// DocumentComplete event, and returns the time in milliseconds as +// a string to the caller. + +#include "stdafx.h" +#include "MeasurePageLoadTimeBHO.h" + +#define MAX_URL 1024 // size of URL buffer +#define MAX_PAGELOADTIME (4*60*1000) // assume all pages take < 4 minutes +#define PORT 42492 // port to listen on. Also jhaas's + // old MSFT employee number + + +// Static function to serve as thread entry point, takes a "this" +// pointer as pParam and calls the method in the object +static DWORD WINAPI ProcessPageTimeRequests(LPVOID pThis) { + reinterpret_cast<CMeasurePageLoadTimeBHO*>(pThis)->ProcessPageTimeRequests(); + + return 0; +} + + +STDMETHODIMP CMeasurePageLoadTimeBHO::SetSite(IUnknown* pUnkSite) +{ + if (pUnkSite != NULL) + { + // Cache the pointer to IWebBrowser2. + HRESULT hr = pUnkSite->QueryInterface(IID_IWebBrowser2, (void **)&m_spWebBrowser); + if (SUCCEEDED(hr)) + { + // Register to sink events from DWebBrowserEvents2. + hr = DispEventAdvise(m_spWebBrowser); + if (SUCCEEDED(hr)) + { + m_fAdvised = TRUE; + } + + // Stash the interface in the global interface table + CComGITPtr<IWebBrowser2> git(m_spWebBrowser); + m_dwCookie = git.Detach(); + + // Create the event to be signaled when navigation completes. + // Start it in nonsignaled state, and allow it to be triggered + // when the initial page load is done. + m_hEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + + // Create a thread to wait on the socket + HANDLE hThread = CreateThread(NULL, 0, ::ProcessPageTimeRequests, this, 0, NULL); + } + } + else + { + // Unregister event sink. + if (m_fAdvised) + { + DispEventUnadvise(m_spWebBrowser); + m_fAdvised = FALSE; + } + + // Release cached pointers and other resources here. + m_spWebBrowser.Release(); + } + + // Call base class implementation. + return IObjectWithSiteImpl<CMeasurePageLoadTimeBHO>::SetSite(pUnkSite); +} + + +void STDMETHODCALLTYPE CMeasurePageLoadTimeBHO::OnDocumentComplete(IDispatch *pDisp, VARIANT *pvarURL) +{ + if (pDisp == m_spWebBrowser) + { + // Fire the event when the page is done loading + // to unblock the other thread. + SetEvent(m_hEvent); + } +} + + +void CMeasurePageLoadTimeBHO::ProcessPageTimeRequests() +{ + CoInitialize(NULL); + + // The event will start in nonsignaled state, meaning that + // the initial page load isn't done yet. Wait for that to + // finish before doing anything. + // + // It seems to be the case that the BHO will get loaded + // and SetSite called always before the initial page load + // even begins, but just to be on the safe side, we won't + // wait indefinitely. + WaitForSingleObject(m_hEvent, MAX_PAGELOADTIME); + + // Retrieve the web browser interface from the global table + CComGITPtr<IWebBrowser2> git(m_dwCookie); + IWebBrowser2* browser; + git.CopyTo(&browser); + + // Create a listening socket + m_sockListen = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (m_sockListen == SOCKET_ERROR) + ErrorExit(); + + BOOL on = TRUE; + if (setsockopt(m_sockListen, SOL_SOCKET, SO_REUSEADDR, + (const char*)&on, sizeof(on))) + ErrorExit(); + + // Bind the listening socket + SOCKADDR_IN addrBind; + + addrBind.sin_family = AF_INET; + addrBind.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addrBind.sin_port = htons(PORT); + + if (bind(m_sockListen, (sockaddr*)&addrBind, sizeof(addrBind))) + ErrorExit(); + + // Listen for incoming connections + if (listen(m_sockListen, 1)) + ErrorExit(); + + // Ensure the socket is blocking... it should be by default, but + // it can't hurt to make sure + unsigned long nNonblocking = 0; + if (ioctlsocket(m_sockListen, FIONBIO, &nNonblocking)) + ErrorExit(); + + m_sockTransport = 0; + + // Loop indefinitely waiting for connections + while(1) + { + SOCKADDR_IN addrConnected; + int sConnected = sizeof(addrConnected); + + // Wait for a client to connect and send a URL + m_sockTransport = accept( + m_sockListen, (sockaddr*)&addrConnected, &sConnected); + + if (m_sockTransport == SOCKET_ERROR) + ErrorExit(); + + char pbBuffer[MAX_URL], strURL[MAX_URL]; + DWORD cbRead, cbWritten; + + bool fDone = false; + + // Loop until we're done with this client + while (!fDone) + { + *strURL = '\0'; + bool fReceivedCR = false; + + do + { + // Only receive up to the first carriage return + cbRead = recv(m_sockTransport, pbBuffer, MAX_URL-1, MSG_PEEK); + + // An error on read most likely means that the remote peer + // closed the connection. Go back to waiting + if (cbRead == 0) + { + fDone = true; + break; + } + + // Null terminate the received characters so strchr() is safe + pbBuffer[cbRead] = '\0'; + + if(char* pchFirstCR = strchr(pbBuffer, '\n')) + { + cbRead = (DWORD)(pchFirstCR - pbBuffer + 1); + fReceivedCR = true; + } + + // The below call will not block, since we determined with + // MSG_PEEK that at least cbRead bytes are in the TCP receive buffer + recv(m_sockTransport, pbBuffer, cbRead, 0); + pbBuffer[cbRead] = '\0'; + + strcat_s(strURL, sizeof(strURL), pbBuffer); + } while (!fReceivedCR); + + // If an error occurred while reading, exit this loop + if (fDone) + break; + + // Strip the trailing CR and/or LF + int i; + for (i = (int)strlen(strURL)-1; i >= 0 && isspace(strURL[i]); i--) + { + strURL[i] = '\0'; + } + + if (i < 0) + { + // Sending a carriage return on a line by itself means that + // the client is done making requests + fDone = true; + } + else + { + // Send the browser to the requested URL + CComVariant vNavFlags( navNoReadFromCache ); + CComVariant vTargetFrame("_self"); + CComVariant vPostData(""); + CComVariant vHTTPHeaders(""); + + ResetEvent(m_hEvent); + DWORD dwStartTime = GetTickCount(); + + HRESULT hr = browser->Navigate( + CComBSTR(strURL), + &vNavFlags, + &vTargetFrame, // TargetFrameName + &vPostData, // PostData + &vHTTPHeaders // Headers + ); + + // The main browser thread will call OnDocumentComplete() when + // the page is done loading, which will in turn trigger + // m_hEvent. Wait here until then; the event will reset itself + // once this thread is released + if (WaitForSingleObject(m_hEvent, MAX_PAGELOADTIME) == WAIT_TIMEOUT) + { + sprintf_s(pbBuffer, sizeof(pbBuffer), "%s,timeout\n", strURL); + + browser->Stop(); + } + else + { + // Format the elapsed time as a string + DWORD dwLoadTime = GetTickCount() - dwStartTime; + sprintf_s( + pbBuffer, sizeof(pbBuffer), "%s,%d\n", strURL, dwLoadTime); + } + + // Send the result. Just in case the TCP buffer can't handle + // the whole thing, send in parts if necessary + char *chSend = pbBuffer; + + while (*chSend) + { + cbWritten = send( + m_sockTransport, chSend, (int)strlen(chSend), 0); + + // Error on send probably means connection reset by peer + if (cbWritten == 0) + { + fDone = true; + break; + } + + chSend += cbWritten; + } + } + } + + // Close the transport socket and wait for another connection + closesocket(m_sockTransport); + m_sockTransport = 0; + } +} + + +void CMeasurePageLoadTimeBHO::ErrorExit() +{ + // Unlink from IE, close the sockets, then terminate this + // thread + SetSite(NULL); + + if (m_sockTransport && m_sockTransport != SOCKET_ERROR) + { + closesocket(m_sockTransport); + m_sockTransport = 0; + } + + if (m_sockListen && m_sockListen != SOCKET_ERROR) + { + closesocket(m_sockListen); + m_sockListen = 0; + } + + TerminateThread(GetCurrentThread(), -1); +}
\ No newline at end of file diff --git a/tools/measure_page_load_time/ie_bho/MeasurePageLoadTimeBHO.h b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTimeBHO.h new file mode 100644 index 0000000..fcfe04d --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTimeBHO.h @@ -0,0 +1,113 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// MeasurePageLoadTimeBHO.h : Declaration of the CMeasurePageLoadTimeBHO + +#pragma once +#include "resource.h" // main symbols + +#include <shlguid.h> // IID_IWebBrowser2, DIID_DWebBrowserEvents2, et +#include <exdispid.h> // DISPID_DOCUMENTCOMPLETE, etc. + +#include <string> + +#include "MeasurePageLoadTime.h" + + +#if defined(_WIN32_WCE) && !defined(_CE_DCOM) && !defined(_CE_ALLOW_SINGLE_THREADED_OBJECTS_IN_MTA) +#error "Single-threaded COM objects are not properly supported on Windows CE platform, such as the Windows Mobile platforms that do not include full DCOM support. Define _CE_ALLOW_SINGLE_THREADED_OBJECTS_IN_MTA to force ATL to support creating single-thread COM object's and allow use of it's single-threaded COM object implementations. The threading model in your rgs file was set to 'Free' as that is the only threading model supported in non DCOM Windows CE platforms." +#endif + + + +// CMeasurePageLoadTimeBHO + +class ATL_NO_VTABLE CMeasurePageLoadTimeBHO : + public CComObjectRootEx<CComSingleThreadModel>, + public CComCoClass<CMeasurePageLoadTimeBHO, &CLSID_MeasurePageLoadTimeBHO>, + public IObjectWithSiteImpl<CMeasurePageLoadTimeBHO>, + public IDispatchImpl<IMeasurePageLoadTimeBHO, &IID_IMeasurePageLoadTimeBHO, &LIBID_MeasurePageLoadTimeLib, /*wMajor =*/ 1, /*wMinor =*/ 0>, + public IDispEventImpl<1, CMeasurePageLoadTimeBHO, &DIID_DWebBrowserEvents2, &LIBID_SHDocVw, 1, 1> +{ +public: + CMeasurePageLoadTimeBHO() + { + } + +DECLARE_REGISTRY_RESOURCEID(IDR_MEASUREPAGELOADTIMEBHO) + +DECLARE_NOT_AGGREGATABLE(CMeasurePageLoadTimeBHO) + +BEGIN_COM_MAP(CMeasurePageLoadTimeBHO) + COM_INTERFACE_ENTRY(IMeasurePageLoadTimeBHO) + COM_INTERFACE_ENTRY(IDispatch) + COM_INTERFACE_ENTRY(IObjectWithSite) +END_COM_MAP() + +BEGIN_SINK_MAP(CMeasurePageLoadTimeBHO) + SINK_ENTRY_EX(1, DIID_DWebBrowserEvents2, DISPID_DOCUMENTCOMPLETE, OnDocumentComplete) +END_SINK_MAP() + + // DWebBrowserEvents2 + void STDMETHODCALLTYPE OnDocumentComplete(IDispatch *pDisp, VARIANT *pvarURL); + STDMETHOD(SetSite)(IUnknown *pUnkSite); + + DECLARE_PROTECT_FINAL_CONSTRUCT() + + HRESULT FinalConstruct() + { + return S_OK; + } + + void FinalRelease() + { + } + + void ProcessPageTimeRequests(void); + void VisitNextURL(void); + void ErrorExit(void); + +private: + CComPtr<IWebBrowser2> m_spWebBrowser; + BOOL m_fAdvised; + + // Handle to global interface table + DWORD m_dwCookie; + + // Handle to event to signal when navigation completes + HANDLE m_hEvent; + + // Socket for accepting incoming connections + SOCKET m_sockListen; + + // Socket for communicating with remote peers + SOCKET m_sockTransport; +}; + +OBJECT_ENTRY_AUTO(__uuidof(MeasurePageLoadTimeBHO), CMeasurePageLoadTimeBHO) diff --git a/tools/measure_page_load_time/ie_bho/MeasurePageLoadTimeBHO.rgs b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTimeBHO.rgs new file mode 100644 index 0000000..907015f --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/MeasurePageLoadTimeBHO.rgs @@ -0,0 +1,27 @@ +HKCR
+{
+ MeasurePageLoadTime.MeasurePageLoadTi.1 = s 'MeasurePageLoadTimeBHO Class'
+ {
+ CLSID = s '{807E68BC-238F-4163-AE4B-0A3604F3E145}'
+ }
+ MeasurePageLoadTime.MeasurePageLoadTime = s 'MeasurePageLoadTimeBHO Class'
+ {
+ CLSID = s '{807E68BC-238F-4163-AE4B-0A3604F3E145}'
+ CurVer = s 'MeasurePageLoadTime.MeasurePageLoadTi.1'
+ }
+ NoRemove CLSID
+ {
+ ForceRemove {807E68BC-238F-4163-AE4B-0A3604F3E145} = s 'MeasurePageLoadTimeBHO Class'
+ {
+ ProgID = s 'MeasurePageLoadTime.MeasurePageLoadTi.1'
+ VersionIndependentProgID = s 'MeasurePageLoadTime.MeasurePageLoadTime'
+ ForceRemove 'Programmable'
+ InprocServer32 = s '%MODULE%'
+ {
+ val ThreadingModel = s 'Apartment'
+ }
+ val AppID = s '%APPID%'
+ 'TypeLib' = s '{61AC7AC4-B715-4955-A238-5F9AEA80DF4B}'
+ }
+ }
+}
diff --git a/tools/measure_page_load_time/ie_bho/resource.h b/tools/measure_page_load_time/ie_bho/resource.h new file mode 100644 index 0000000..a8004dc --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/resource.h @@ -0,0 +1,18 @@ +//{{NO_DEPENDENCIES}} +// Microsoft Visual C++ generated include file. +// Used by MeasurePageLoadTime.rc +// +#define IDS_PROJNAME 100 +#define IDR_MEASUREPAGELOADTIME 101 +#define IDR_MEASUREPAGELOADTIMEBHO 102 + +// Next default values for new objects +// +#ifdef APSTUDIO_INVOKED +#ifndef APSTUDIO_READONLY_SYMBOLS +#define _APS_NEXT_RESOURCE_VALUE 201 +#define _APS_NEXT_COMMAND_VALUE 32768 +#define _APS_NEXT_CONTROL_VALUE 201 +#define _APS_NEXT_SYMED_VALUE 103 +#endif +#endif diff --git a/tools/measure_page_load_time/ie_bho/stdafx.cpp b/tools/measure_page_load_time/ie_bho/stdafx.cpp new file mode 100644 index 0000000..c67b429 --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/stdafx.cpp @@ -0,0 +1,5 @@ +// stdafx.cpp : source file that includes just the standard includes +// MeasurePageLoadTime.pch will be the pre-compiled header +// stdafx.obj will contain the pre-compiled type information + +#include "stdafx.h" diff --git a/tools/measure_page_load_time/ie_bho/stdafx.h b/tools/measure_page_load_time/ie_bho/stdafx.h new file mode 100644 index 0000000..3f26087 --- /dev/null +++ b/tools/measure_page_load_time/ie_bho/stdafx.h @@ -0,0 +1,39 @@ +// stdafx.h : include file for standard system include files, +// or project specific include files that are used frequently, +// but are changed infrequently + +#pragma once + +#ifndef STRICT +#define STRICT +#endif + +// Modify the following defines if you have to target a platform prior to the ones specified below. +// Refer to MSDN for the latest info on corresponding values for different platforms. +#ifndef WINVER // Allow use of features specific to Windows XP or later. +#define WINVER 0x0501 // Change this to the appropriate value to target other versions of Windows. +#endif + +#ifndef _WIN32_WINNT // Allow use of features specific to Windows XP or later. +#define _WIN32_WINNT 0x0501 // Change this to the appropriate value to target other versions of Windows. +#endif + +#ifndef _WIN32_WINDOWS // Allow use of features specific to Windows 98 or later. +#define _WIN32_WINDOWS 0x0410 // Change this to the appropriate value to target Windows Me or later. +#endif + +#ifndef _WIN32_IE // Allow use of features specific to IE 6.0 or later. +#define _WIN32_IE 0x0600 // Change this to the appropriate value to target other versions of IE. +#endif + +#define _ATL_APARTMENT_THREADED +#define _ATL_NO_AUTOMATIC_NAMESPACE + +#define _ATL_CSTRING_EXPLICIT_CONSTRUCTORS // some CString constructors will be explicit + + +#include "resource.h" +#include <atlbase.h> +#include <atlcom.h> + +using namespace ATL;
\ No newline at end of file diff --git a/tools/memory_watcher/call_stack.cc b/tools/memory_watcher/call_stack.cc new file mode 100644 index 0000000..538f4da --- /dev/null +++ b/tools/memory_watcher/call_stack.cc @@ -0,0 +1,400 @@ +// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "call_stack.h" +#include <shlwapi.h> +#include <tlhelp32.h> + +#include "memory_hook.h" +#include "base/string_util.h" + +// Typedefs for explicit dynamic linking with functions exported from +// dbghelp.dll. +typedef BOOL (__stdcall *t_StackWalk64)(DWORD, HANDLE, HANDLE, + LPSTACKFRAME64, PVOID, + PREAD_PROCESS_MEMORY_ROUTINE64, + PFUNCTION_TABLE_ACCESS_ROUTINE64, + PGET_MODULE_BASE_ROUTINE64, + PTRANSLATE_ADDRESS_ROUTINE64); +typedef PVOID (__stdcall *t_SymFunctionTableAccess64)(HANDLE, DWORD64); +typedef DWORD64 (__stdcall *t_SymGetModuleBase64)(HANDLE, DWORD64); +typedef BOOL (__stdcall *t_SymCleanup)(HANDLE); +typedef BOOL (__stdcall *t_SymGetSymFromAddr64)(HANDLE, DWORD64, + PDWORD64, PIMAGEHLP_SYMBOL64); +typedef BOOL (__stdcall *t_SymGetLineFromAddr64)(HANDLE, DWORD64, PDWORD, + PIMAGEHLP_LINE64); +typedef BOOL (__stdcall *t_SymInitialize)(HANDLE, PCTSTR, BOOL); +typedef DWORD (__stdcall *t_SymGetOptions)(void); +typedef DWORD (__stdcall *t_SymSetOptions)(DWORD); +typedef BOOL (__stdcall *t_SymGetSearchPath)(HANDLE, PTSTR, DWORD); +typedef DWORD64 (__stdcall *t_SymLoadModule64)(HANDLE, HANDLE, PCSTR, + PCSTR, DWORD64, DWORD); +typedef BOOL (__stdcall *t_SymGetModuleInfo64)(HANDLE, DWORD64, + PIMAGEHLP_MODULE64); + +// According to http://msdn2.microsoft.com/en-us/library/ms680650(VS.85).aspx +// "All DbgHelp functions, such as this one, are single threaded. Therefore, +// calls from more than one thread to this function will likely result in +// unexpected behavior or memory corruption. To avoid this, you must +// synchromize all concurrent calls from one thread to this function." +// +// dbghelp_lock_ is used to serialize access across all calls to the DbgHelp +// library. This may be overly conservative (serializing them all together), +// but does guarantee correctness. +static Lock dbghelp_lock_; + +static t_StackWalk64 pStackWalk64 = NULL; +static t_SymCleanup pSymCleanup = NULL; +static t_SymGetSymFromAddr64 pSymGetSymFromAddr64 = NULL; +static t_SymFunctionTableAccess64 pSymFunctionTableAccess64 = NULL; +static t_SymGetModuleBase64 pSymGetModuleBase64 = NULL; +static t_SymGetLineFromAddr64 pSymGetLineFromAddr64 = NULL; +static t_SymInitialize pSymInitialize = NULL; +static t_SymGetOptions pSymGetOptions = NULL; +static t_SymSetOptions pSymSetOptions = NULL; +static t_SymGetModuleInfo64 pSymGetModuleInfo64 = NULL; +static t_SymGetSearchPath pSymGetSearchPath = NULL; +static t_SymLoadModule64 pSymLoadModule64 = NULL; + +#define LOADPROC(module, name) do { \ + p##name = reinterpret_cast<t_##name>(GetProcAddress(module, #name)); \ + if (p##name == NULL) return false; \ +} while (0) + +// Dynamically load the DbgHelp library and supporting routines that we +// will use. +static bool LoadDbgHelp() { + static bool loaded = false; + if (!loaded) { + AutoLock Lock(dbghelp_lock_); + + // Re-check if we've loaded successfully now that we have the lock. + if (loaded) + return true; + + // Load dbghelp.dll, and obtain pointers to the exported functions that we + // will be using. + HMODULE dbghelp_module = LoadLibrary(L"dbghelp.dll"); + if (dbghelp_module) { + LOADPROC(dbghelp_module, StackWalk64); + LOADPROC(dbghelp_module, SymFunctionTableAccess64); + LOADPROC(dbghelp_module, SymGetModuleBase64); + LOADPROC(dbghelp_module, SymCleanup); + LOADPROC(dbghelp_module, SymGetSymFromAddr64); + LOADPROC(dbghelp_module, SymGetLineFromAddr64); + LOADPROC(dbghelp_module, SymInitialize); + LOADPROC(dbghelp_module, SymGetOptions); + LOADPROC(dbghelp_module, SymSetOptions); + LOADPROC(dbghelp_module, SymGetModuleInfo64); + LOADPROC(dbghelp_module, SymGetSearchPath); + LOADPROC(dbghelp_module, SymLoadModule64); + loaded = true; + } else { + return false; + } + } + return loaded; +} + +// Load the symbols for generating stack traces. +static bool LoadSymbols(HANDLE process_handle) { + static bool symbols_loaded = false; + if (symbols_loaded) return true; + + BOOL ok; + + // Initialize the symbol engine. + ok = pSymInitialize(process_handle, /* hProcess */ + NULL, /* UserSearchPath */ + FALSE); /* fInvadeProcess */ + if (!ok) return false; + + DWORD options = pSymGetOptions(); + options |= SYMOPT_LOAD_LINES; + options |= SYMOPT_FAIL_CRITICAL_ERRORS; + options |= SYMOPT_UNDNAME; + options = pSymSetOptions(options); + + const DWORD kMaxSearchPath = 1024; + TCHAR buf[kMaxSearchPath] = {0}; + ok = pSymGetSearchPath(process_handle, buf, kMaxSearchPath); + if (!ok) + return false; + + HANDLE snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, + GetCurrentProcessId()); + if (snapshot == INVALID_HANDLE_VALUE) + return false; + + MODULEENTRY32W module; + module.dwSize = sizeof(module); // Set the size of the structure. + BOOL cont = Module32FirstW(snapshot, &module); + while (cont) { + DWORD64 base; + // NOTE the SymLoadModule64 function has the peculiarity of accepting a + // both unicode and ASCII strings even though the parameter is PSTR. + base = pSymLoadModule64(process_handle, + 0, + reinterpret_cast<PSTR>(module.szExePath), + reinterpret_cast<PSTR>(module.szModule), + reinterpret_cast<DWORD64>(module.modBaseAddr), + module.modBaseSize); + if (base == 0) { + int err = GetLastError(); + if (err != ERROR_MOD_NOT_FOUND && err != ERROR_INVALID_HANDLE) + return false; + } + cont = Module32NextW(snapshot, &module); + } + CloseHandle(snapshot); + + symbols_loaded = true; + return true; +} + + +CallStack::SymbolCache* CallStack::symbol_cache_; + +bool CallStack::Initialize() { + // We need to delay load the symbol cache until after + // the MemoryHook heap is alive. + symbol_cache_ = new SymbolCache(); + return LoadDbgHelp(); +} + +CallStack::CallStack() { + static LONG callstack_id = 0; + frame_count_ = 0; + hash_ = 0; + id_ = InterlockedIncrement(&callstack_id); + + LoadDbgHelp(); + CHECK(GetStackTrace()); +} + +bool CallStack::IsEqual(const CallStack &target) { + if (frame_count_ != target.frame_count_) + return false; // They can't be equal if the sizes are different. + + // Walk the frames array until we + // either find a mismatch, or until we reach the end of the call stacks. + for (int index = 0; index < frame_count_; index++) { + if (frames_[index] != target.frames_[index]) + return false; // Found a mismatch. They are not equal. + } + + // Reached the end of the call stacks. They are equal. + return true; +} + +void CallStack::AddFrame(DWORD_PTR pc) { + DCHECK(frame_count_ < kMaxTraceFrames); + frames_[frame_count_++] = pc; + + // Create a unique id for this CallStack. + pc = pc + (frame_count_ * 13); // Alter the PC based on position in stack. + hash_ = ~hash_ + (pc << 15);
+ hash_ = hash_ ^ (pc >> 12);
+ hash_ = hash_ + (pc << 2);
+ hash_ = hash_ ^ (pc >> 4);
+ hash_ = hash_ * 2057;
+ hash_ = hash_ ^ (pc >> 16);
+} + +bool CallStack::GetStackTrace() { + // Initialize the context record. + CONTEXT context; + memset(&context, 0, sizeof(context)); + context.ContextFlags = CONTEXT_FULL; + __asm call x + __asm x: pop eax + __asm mov context.Eip, eax + __asm mov context.Ebp, ebp + __asm mov context.Esp, esp + + STACKFRAME64 frame; + memset(&frame, 0, sizeof(frame)); + +#ifdef _M_IX86 + DWORD image_type = IMAGE_FILE_MACHINE_I386; + frame.AddrPC.Offset = context.Eip; + frame.AddrPC.Mode = AddrModeFlat; + frame.AddrFrame.Offset = context.Ebp; + frame.AddrFrame.Mode = AddrModeFlat; + frame.AddrStack.Offset = context.Esp; + frame.AddrStack.Mode = AddrModeFlat; +#elif + NOT IMPLEMENTED! +#endif + + HANDLE current_process = GetCurrentProcess(); + HANDLE current_thread = GetCurrentThread(); + + // Walk the stack. + unsigned int count = 0; + { + AutoLock lock(dbghelp_lock_); + + while (count < kMaxTraceFrames) { + count++; + if (!pStackWalk64(image_type, + current_process, + current_thread, + &frame, + &context, + 0, + pSymFunctionTableAccess64, + pSymGetModuleBase64, + NULL)) + break; // Couldn't trace back through any more frames. + + if (frame.AddrFrame.Offset == 0) + continue; // End of stack. + + // Push this frame's program counter onto the provided CallStack. + AddFrame((DWORD_PTR)frame.AddrPC.Offset); + } + } + return true; +} + +void CallStack::ToString(std::string* output) { + static const int kStackWalkMaxNameLen = MAX_SYM_NAME; + HANDLE current_process = GetCurrentProcess(); + + if (!LoadSymbols(current_process)) { + *output = "Error"; + return; + } + + AutoLock lock(dbghelp_lock_); + + // Iterate through each frame in the call stack. + for (int32 index = 0; index < frame_count_; index++) { + std::string line; + + DWORD_PTR intruction_pointer = frame(index); + + SymbolCache::iterator it; + it = symbol_cache_->find( intruction_pointer ); + if (it != symbol_cache_->end()) { + line = it->second; + } else { + // Try to locate a symbol for this frame. + DWORD64 symbol_displacement = 0; + ULONG64 buffer[(sizeof(IMAGEHLP_SYMBOL64) + + sizeof(TCHAR)*kStackWalkMaxNameLen + + sizeof(ULONG64) - 1) / sizeof(ULONG64)]; + IMAGEHLP_SYMBOL64* symbol = reinterpret_cast<IMAGEHLP_SYMBOL64*>(buffer); + memset(buffer, 0, sizeof(buffer)); + symbol->SizeOfStruct = sizeof(IMAGEHLP_SYMBOL64); + symbol->MaxNameLength = kStackWalkMaxNameLen; + BOOL ok = pSymGetSymFromAddr64(current_process, // hProcess + intruction_pointer, // Address + &symbol_displacement, // Displacement + symbol); // Symbol + if (ok) { + // Try to locate more source information for the symbol. + IMAGEHLP_LINE64 Line; + memset(&Line, 0, sizeof(Line)); + Line.SizeOfStruct = sizeof(Line); + DWORD line_displacement; + ok = pSymGetLineFromAddr64(current_process, + intruction_pointer, + &line_displacement, + &Line); + if (ok) { + // Skip junk symbols from our internal stuff. + if (strstr(symbol->Name, "CallStack::") || + strstr(symbol->Name, "MemoryWatcher::") || + strstr(symbol->Name, "Perftools_") || + strstr(symbol->Name, "MemoryHook::") ) { + // Just record a blank string. + (*symbol_cache_)[intruction_pointer] = std::string(""); + continue; + } + + line += " "; + line += static_cast<char*>(Line.FileName); + line += " ("; + line += IntToString(Line.LineNumber); + line += "): "; + line += symbol->Name; + line += "\n"; + } else { + line += " unknown (0):"; + line += symbol->Name; + line += "\n"; + } + } else { + // OK - couldn't get any info. Try for the module. + IMAGEHLP_MODULE64 module_info; + module_info.SizeOfStruct = sizeof(module_info); + if (pSymGetModuleInfo64(current_process, intruction_pointer, + &module_info)) { + line += " ("; + line += static_cast<char*>(module_info.ModuleName); + line += ")\n"; + } else { + line += " ???\n"; + } + } + } + + (*symbol_cache_)[intruction_pointer] = line; + *output += line; + } + *output += "==================\n"; +} + + +Lock AllocationStack::freelist_lock_; +AllocationStack* AllocationStack::freelist_ = NULL; + +void* AllocationStack::operator new(size_t size) { + DCHECK(size == sizeof(AllocationStack)); + { + AutoLock lock(freelist_lock_); + if (freelist_ != NULL) { + AllocationStack* stack = freelist_; + freelist_ = freelist_->next_; + stack->next_ = NULL; + return stack; + } + } + return MemoryHook::Alloc(size); +} + +void AllocationStack::operator delete(void* ptr) { + AllocationStack *stack = reinterpret_cast<AllocationStack*>(ptr); + AutoLock lock(freelist_lock_); + DCHECK(stack->next_ == NULL); + stack->next_ = freelist_; + freelist_ = stack; +} diff --git a/tools/memory_watcher/call_stack.h b/tools/memory_watcher/call_stack.h new file mode 100644 index 0000000..f4dd7bd --- /dev/null +++ b/tools/memory_watcher/call_stack.h @@ -0,0 +1,138 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this module come from: +// http://www.codeproject.com/KB/applications/visualleakdetector.aspx +// by Dan Moulding. +// http://www.codeproject.com/KB/threads/StackWalker.aspx +// by Jochen Kalmbach + +#ifndef MEMORY_WATCHER_CALL_STACK_H_ +#define MEMORY_WATCHER_CALL_STACK_H_ + +#include <windows.h> +#include <dbghelp.h> +#include <functional> +#include <map> + +#include "memory_watcher.h" +#include "base/lock.h" +#include "base/logging.h" + +// The CallStack Class +// A stack where memory has been allocated. +class CallStack { + public: + // Initialize for tracing CallStacks. + static bool Initialize(); + + CallStack(); + virtual ~CallStack() {} + + // Get a hash for this CallStack. + // Identical stack traces will have matching hashes. + int32 hash() { return hash_; } + + // Get a unique ID for this CallStack. + // No two CallStacks will ever have the same ID. The ID is a monotonically + // increasing number. Newer CallStacks always have larger IDs. + int32 id() { return id_; } + + // Retrieves the frame at the specified index. + DWORD_PTR frame(int32 index) { + DCHECK(index < frame_count_ && index >= 0); + return frames_[index]; + } + + // Compares the CallStack to another CallStack + // for equality. Two CallStacks are equal if they are the same size and if + // every frame in each is identical to the corresponding frame in the other. + bool IsEqual(const CallStack &target); + + // Convert the callstack to a string stored in output. + void CallStack::ToString(std::string* output); + + private: + // The maximum number of frames to trace. + static const int kMaxTraceFrames = 32; + + // Pushes a frame's program counter onto the CallStack. + void AddFrame(DWORD_PTR programcounter); + + // Traces the stack, starting from this function, up to kMaxTraceFrames + // frames. + bool GetStackTrace(); + + // Functions for manipulating the frame list. + void ClearFrames(); + + int frame_count_; // Current size (in frames) + DWORD_PTR frames_[kMaxTraceFrames]; + int32 hash_; + int32 id_; + + // Cache ProgramCounter -> Symbol lookups. + // This cache is not thread safe. + typedef std::map<int32, std::string, std::less<int32>, + PrivateHookAllocator<int32> > SymbolCache; + static SymbolCache* symbol_cache_; + + DISALLOW_EVIL_CONSTRUCTORS(CallStack); +}; + +// An AllocationStack is a type of CallStack which represents +// a CallStack where memory has been allocated. As such, in +// addition to the CallStack information, it also tracks the +// amount of memory allocated. +class AllocationStack : public CallStack { + public: + explicit AllocationStack(int32 alloc_size) + : allocation_size_(alloc_size), + next_(NULL), + CallStack() { + } + + // The size of the allocation. + int32 allocation_size() { return allocation_size_; } + + // We maintain a freelist of the AllocationStacks. + void* operator new(size_t s); + void operator delete(void*p); + + private: + int32 allocation_size_; // The size of the allocation + + AllocationStack* next_; // Pointer used when on the freelist. + static AllocationStack* freelist_; + static Lock freelist_lock_; + + DISALLOW_EVIL_CONSTRUCTORS(AllocationStack); +}; + +#endif // MEMORY_WATCHER_CALL_STACK_H_ diff --git a/tools/memory_watcher/dllmain.cc b/tools/memory_watcher/dllmain.cc new file mode 100644 index 0000000..6a4bbbd --- /dev/null +++ b/tools/memory_watcher/dllmain.cc @@ -0,0 +1,170 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// The memory_watcher.dll is hooked by simply linking it. When we get the +// windows notification that this DLL is loaded, we do a few things: +// 1) Register a Hot Key. +// Only one process can hook the Hot Key, so one will get it, and the +// others will silently fail. +// 2) Create a thread to wait on an event. +// Since only one process will get the HotKey, it will be responsible for +// notifying all process when it's time to do something. Each process +// will have a thread waiting for communication from the master to dump +// the callstacks. + +#include <windows.h> +#include "tools/memory_watcher/memory_watcher.h" +#include "tools/memory_watcher/hotkey.h" + +static wchar_t* kDumpEvent = L"MemWatcher.DumpEvent"; +static MemoryWatcher* g_memory_watcher = NULL; +static HANDLE g_dump_event = INVALID_HANDLE_VALUE; +static HANDLE g_quit_event = INVALID_HANDLE_VALUE; +static HANDLE g_watcher_thread = INVALID_HANDLE_VALUE; + +// A HotKey to dump the memory statistics. +class MemoryWatcherDumpKey : public HotKeyHandler { + public: + MemoryWatcherDumpKey(UINT modifiers, UINT vkey) + : HotKeyHandler(modifiers, vkey) {} + + virtual LRESULT OnHotKey(UINT, WPARAM, LPARAM, BOOL& bHandled) { + SetEvent(g_dump_event); + return 1; + } +}; + +// Register ALT-CONTROL-D to Dump Memory stats. +MemoryWatcherDumpKey hHotKeyHandler(MOD_ALT|MOD_CONTROL, 0x44); + +// Creates the global memory watcher. +void CreateMemoryWatcher() { + g_memory_watcher = new MemoryWatcher(); +} + +// Deletes the global memory watcher. +void DeleteMemoryWatcher() { + if (g_memory_watcher) + delete g_memory_watcher; + g_memory_watcher = NULL; +} + +// Thread for watching for key events. +DWORD WINAPI ThreadMain(LPVOID) { + bool stopping = false; + HANDLE events[2] = { g_dump_event, g_quit_event }; + while (!stopping) { + DWORD rv = WaitForMultipleObjects(2, events, FALSE, INFINITE); + switch (rv) { + case WAIT_OBJECT_0: + if (g_memory_watcher) { + g_memory_watcher->DumpLeaks(); + // After dumping, we teardown. + ExitProcess(0); + } + break; + case WAIT_OBJECT_0 + 1: + stopping = true; + break; + default: + NOTREACHED(); + break; + } + } + return 0; +} + +// Creates the background thread +void CreateBackgroundThread() { + // Create a named event which can be used to notify + // all watched processes. + g_dump_event = CreateEvent(0, TRUE, FALSE, kDumpEvent); + DCHECK(g_dump_event != NULL); + + // Create a local event which can be used to kill our + // background thread. + g_quit_event = CreateEvent(0, TRUE, FALSE, NULL); + DCHECK(g_quit_event != NULL); + + // Create the background thread. + g_watcher_thread = CreateThread(0, + 0, + ThreadMain, + 0, + 0, + 0); + DCHECK(g_watcher_thread != NULL); +} + +// Tell the background thread to stop. +void StopBackgroundThread() { + // Send notification to our background thread. + SetEvent(g_quit_event); + + // Wait for our background thread to die. + DWORD rv = WaitForSingleObject(g_watcher_thread, INFINITE); + DCHECK(rv == WAIT_OBJECT_0); + + // Cleanup our global handles. + CloseHandle(g_quit_event); + CloseHandle(g_dump_event); + CloseHandle(g_watcher_thread); +} + +bool IsChromeExe() { + return GetModuleHandleA("chrome.exe") != NULL; +} + +extern "C" { +// DllMain is the windows entry point to this DLL. +// We use the entry point as the mechanism for starting and stopping +// the MemoryWatcher. +BOOL WINAPI DllMain(HINSTANCE dll_instance, DWORD reason, + LPVOID reserved) { + if (!IsChromeExe()) + return FALSE; + + switch (reason) { + case DLL_PROCESS_ATTACH: + CreateMemoryWatcher(); + CreateBackgroundThread(); + break; + case DLL_PROCESS_DETACH: + DeleteMemoryWatcher(); + StopBackgroundThread(); + break; + } + return TRUE; +} + +__declspec(dllexport) void __cdecl SetLogName(char* name) { + g_memory_watcher->SetLogName(name); +} + +} // extern "C" diff --git a/tools/memory_watcher/hotkey.h b/tools/memory_watcher/hotkey.h new file mode 100644 index 0000000..d6746c2 --- /dev/null +++ b/tools/memory_watcher/hotkey.h @@ -0,0 +1,74 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef TOOLS_MEMORY_WATCHER_HOTKEY_H_ +#define TOOLS_MEMORY_WATCHER_HOTKEY_H_ + +#include <atlbase.h> +#include <atlwin.h> +#include "base/logging.h" + +// HotKey handler. +// Programs wishing to register a hotkey can use this. +class HotKeyHandler : public CWindowImpl<HotKeyHandler> { + public: + HotKeyHandler(UINT modifiers, UINT vk) + : m_modifiers(modifiers), + m_vkey(vk) { + Start(); + } + ~HotKeyHandler() { Stop(); } + +BEGIN_MSG_MAP(HotKeyHandler) + MESSAGE_HANDLER(WM_HOTKEY, OnHotKey) +END_MSG_MAP() + + private: + static const int hotkey_id = 0x0000baba; + + bool Start() { + if (NULL == Create(NULL, NULL, NULL, WS_POPUP)) + return false; + return RegisterHotKey(m_hWnd, hotkey_id, m_modifiers, m_vkey) == TRUE; + } + + void Stop() { + UnregisterHotKey(m_hWnd, hotkey_id); + DestroyWindow(); + } + + // Handle the registered Hotkey being pressed. + virtual LRESULT OnHotKey(UINT /*uMsg*/, WPARAM /*wParam*/, + LPARAM /*lParam*/, BOOL& bHandled) = 0; + + UINT m_modifiers; + UINT m_vkey; +}; + +#endif // TOOLS_MEMORY_WATCHER_HOTKEY_H_ diff --git a/tools/memory_watcher/ia32_modrm_map.cc b/tools/memory_watcher/ia32_modrm_map.cc new file mode 100644 index 0000000..eaae07c --- /dev/null +++ b/tools/memory_watcher/ia32_modrm_map.cc @@ -0,0 +1,120 @@ +/* Copyright (c) 2007, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * + * Table of relevant information about how to decode the ModR/M byte. + * Based on information in the IA 32 Intel Architecture + * Software Developer's Manual Volume 2: Instruction Set Reference. + */ + +#include "mini_disassembler.h" +#include "mini_disassembler_types.h" + +namespace sidestep { + +const ModrmEntry MiniDisassembler::s_ia16_modrm_map_[] = { +// mod == 00 + /* r/m == 000 */ { false, false, OS_ZERO }, + /* r/m == 001 */ { false, false, OS_ZERO }, + /* r/m == 010 */ { false, false, OS_ZERO }, + /* r/m == 011 */ { false, false, OS_ZERO }, + /* r/m == 100 */ { false, false, OS_ZERO }, + /* r/m == 101 */ { false, false, OS_ZERO }, + /* r/m == 110 */ { true, false, OS_WORD }, + /* r/m == 111 */ { false, false, OS_ZERO }, +// mod == 01 + /* r/m == 000 */ { true, false, OS_BYTE }, + /* r/m == 001 */ { true, false, OS_BYTE }, + /* r/m == 010 */ { true, false, OS_BYTE }, + /* r/m == 011 */ { true, false, OS_BYTE }, + /* r/m == 100 */ { true, false, OS_BYTE }, + /* r/m == 101 */ { true, false, OS_BYTE }, + /* r/m == 110 */ { true, false, OS_BYTE }, + /* r/m == 111 */ { true, false, OS_BYTE }, +// mod == 10 + /* r/m == 000 */ { true, false, OS_WORD }, + /* r/m == 001 */ { true, false, OS_WORD }, + /* r/m == 010 */ { true, false, OS_WORD }, + /* r/m == 011 */ { true, false, OS_WORD }, + /* r/m == 100 */ { true, false, OS_WORD }, + /* r/m == 101 */ { true, false, OS_WORD }, + /* r/m == 110 */ { true, false, OS_WORD }, + /* r/m == 111 */ { true, false, OS_WORD }, +// mod == 11 + /* r/m == 000 */ { false, false, OS_ZERO }, + /* r/m == 001 */ { false, false, OS_ZERO }, + /* r/m == 010 */ { false, false, OS_ZERO }, + /* r/m == 011 */ { false, false, OS_ZERO }, + /* r/m == 100 */ { false, false, OS_ZERO }, + /* r/m == 101 */ { false, false, OS_ZERO }, + /* r/m == 110 */ { false, false, OS_ZERO }, + /* r/m == 111 */ { false, false, OS_ZERO } +}; + +const ModrmEntry MiniDisassembler::s_ia32_modrm_map_[] = { +// mod == 00 + /* r/m == 000 */ { false, false, OS_ZERO }, + /* r/m == 001 */ { false, false, OS_ZERO }, + /* r/m == 010 */ { false, false, OS_ZERO }, + /* r/m == 011 */ { false, false, OS_ZERO }, + /* r/m == 100 */ { false, true, OS_ZERO }, + /* r/m == 101 */ { true, false, OS_DOUBLE_WORD }, + /* r/m == 110 */ { false, false, OS_ZERO }, + /* r/m == 111 */ { false, false, OS_ZERO }, +// mod == 01 + /* r/m == 000 */ { true, false, OS_BYTE }, + /* r/m == 001 */ { true, false, OS_BYTE }, + /* r/m == 010 */ { true, false, OS_BYTE }, + /* r/m == 011 */ { true, false, OS_BYTE }, + /* r/m == 100 */ { true, true, OS_BYTE }, + /* r/m == 101 */ { true, false, OS_BYTE }, + /* r/m == 110 */ { true, false, OS_BYTE }, + /* r/m == 111 */ { true, false, OS_BYTE }, +// mod == 10 + /* r/m == 000 */ { true, false, OS_DOUBLE_WORD }, + /* r/m == 001 */ { true, false, OS_DOUBLE_WORD }, + /* r/m == 010 */ { true, false, OS_DOUBLE_WORD }, + /* r/m == 011 */ { true, false, OS_DOUBLE_WORD }, + /* r/m == 100 */ { true, true, OS_DOUBLE_WORD }, + /* r/m == 101 */ { true, false, OS_DOUBLE_WORD }, + /* r/m == 110 */ { true, false, OS_DOUBLE_WORD }, + /* r/m == 111 */ { true, false, OS_DOUBLE_WORD }, +// mod == 11 + /* r/m == 000 */ { false, false, OS_ZERO }, + /* r/m == 001 */ { false, false, OS_ZERO }, + /* r/m == 010 */ { false, false, OS_ZERO }, + /* r/m == 011 */ { false, false, OS_ZERO }, + /* r/m == 100 */ { false, false, OS_ZERO }, + /* r/m == 101 */ { false, false, OS_ZERO }, + /* r/m == 110 */ { false, false, OS_ZERO }, + /* r/m == 111 */ { false, false, OS_ZERO }, +}; + +}; // namespace sidestep diff --git a/tools/memory_watcher/ia32_opcode_map.cc b/tools/memory_watcher/ia32_opcode_map.cc new file mode 100644 index 0000000..df57b2a --- /dev/null +++ b/tools/memory_watcher/ia32_opcode_map.cc @@ -0,0 +1,1187 @@ +/* Copyright (c) 2007, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * + * Opcode decoding maps. Based on the IA-32 Intel Architecture + * Software Developer's Manual Volume 2: Instruction Set Reference. Idea + * for how to lay out the tables in memory taken from the implementation + * in the Bastard disassembly environment. + */ + +#include "mini_disassembler.h" + +namespace sidestep { + +/* +* This is the first table to be searched; the first field of each +* Opcode in the table is either 0 to indicate you're in the +* right table, or an index to the correct table, in the global +* map g_pentiumOpcodeMap +*/ +const Opcode s_first_opcode_byte[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "add", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "add", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_G | OT_B, AM_E | OT_B, AM_NOT_USED, "add", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "add", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "add", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "add", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "or", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "or", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA */ { 0, IT_GENERIC, AM_G | OT_B, AM_E | OT_B, AM_NOT_USED, "or", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "or", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "or", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "or", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF */ { 1, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x10 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "adc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x11 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "adc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x12 */ { 0, IT_GENERIC, AM_G | OT_B, AM_E | OT_B, AM_NOT_USED, "adc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x13 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "adc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x14 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "adc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x15 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "adc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x16 */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x17 */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x18 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "sbb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x19 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "sbb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1A */ { 0, IT_GENERIC, AM_G | OT_B, AM_E | OT_B, AM_NOT_USED, "sbb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1B */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "sbb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1C */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "sbb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1D */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "sbb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1E */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1F */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x20 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "and", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x21 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "and", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x22 */ { 0, IT_GENERIC, AM_G | OT_B, AM_E | OT_B, AM_NOT_USED, "and", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x23 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "and", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x24 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "and", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x25 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "and", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x26 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x27 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "daa", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x28 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "sub", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x29 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "sub", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2A */ { 0, IT_GENERIC, AM_G | OT_B, AM_E | OT_B, AM_NOT_USED, "sub", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2B */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "sub", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2C */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "sub", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2D */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "sub", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2E */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2F */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "das", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x30 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "xor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x31 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "xor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x32 */ { 0, IT_GENERIC, AM_G | OT_B, AM_E | OT_B, AM_NOT_USED, "xor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x33 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "xor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x34 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "xor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x35 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "xor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x36 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x37 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "aaa", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x38 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x39 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3A */ { 0, IT_GENERIC, AM_G | OT_B, AM_E | OT_B, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3B */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3C */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3D */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3E */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3F */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "aas", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x40 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x41 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x42 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x43 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x44 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x45 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x46 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x47 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x48 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x49 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4A */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4B */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4C */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4D */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4E */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4F */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x50 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x51 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x52 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x53 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x54 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x55 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x56 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x57 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x58 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x59 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5A */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5B */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5C */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5D */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5E */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5F */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x60 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "pushad", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x61 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "popad", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x62 */ { 0, IT_GENERIC, AM_G | OT_V, AM_M | OT_A, AM_NOT_USED, "bound", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x63 */ { 0, IT_GENERIC, AM_E | OT_W, AM_G | OT_W, AM_NOT_USED, "arpl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x64 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x65 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x66 */ { 0, IT_PREFIX_OPERAND, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x67 */ { 0, IT_PREFIX_ADDRESS, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x68 */ { 0, IT_GENERIC, AM_I | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x69 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_I | OT_V, "imul", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6A */ { 0, IT_GENERIC, AM_I | OT_B, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6B */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_I | OT_B, "imul", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6C */ { 0, IT_GENERIC, AM_Y | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "insb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6D */ { 0, IT_GENERIC, AM_Y | OT_V, AM_REGISTER | OT_V, AM_NOT_USED, "insd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6E */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_X | OT_B, AM_NOT_USED, "outsb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6F */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_X | OT_V, AM_NOT_USED, "outsb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x70 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jo", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x71 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jno", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x72 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x73 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jnc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x74 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x75 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jnz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x76 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jbe", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x77 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "ja", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x78 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "js", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x79 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jns", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7A */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jpe", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7B */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jpo", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7C */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7D */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jge", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7E */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jle", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7F */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x80 */ { 2, IT_REFERENCE, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x81 */ { 3, IT_REFERENCE, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x82 */ { 4, IT_REFERENCE, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x83 */ { 5, IT_REFERENCE, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x84 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "test", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x85 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "test", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x86 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "xchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x87 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "xchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x88 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x89 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8A */ { 0, IT_GENERIC, AM_G | OT_B, AM_E | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8B */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8C */ { 0, IT_GENERIC, AM_E | OT_W, AM_S | OT_W, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8D */ { 0, IT_GENERIC, AM_G | OT_V, AM_M | OT_ADDRESS_MODE_M, AM_NOT_USED, "lea", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8E */ { 0, IT_GENERIC, AM_S | OT_W, AM_E | OT_W, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8F */ { 0, IT_GENERIC, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x90 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "nop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x91 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_REGISTER | OT_V, AM_NOT_USED, "xchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x92 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_REGISTER | OT_V, AM_NOT_USED, "xchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x93 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_REGISTER | OT_V, AM_NOT_USED, "xchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x94 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_REGISTER | OT_V, AM_NOT_USED, "xchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x95 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_REGISTER | OT_V, AM_NOT_USED, "xchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x96 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_REGISTER | OT_V, AM_NOT_USED, "xchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x97 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_REGISTER | OT_V, AM_NOT_USED, "xchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x98 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "cwde", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x99 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "cdq", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9A */ { 0, IT_JUMP, AM_A | OT_P, AM_NOT_USED, AM_NOT_USED, "callf", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9B */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "wait", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9C */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "pushfd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9D */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "popfd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9E */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "sahf", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9F */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "lahf", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA0 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_O | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA1 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_O | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA2 */ { 0, IT_GENERIC, AM_O | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA3 */ { 0, IT_GENERIC, AM_O | OT_V, AM_REGISTER | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA4 */ { 0, IT_GENERIC, AM_X | OT_B, AM_Y | OT_B, AM_NOT_USED, "movsb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA5 */ { 0, IT_GENERIC, AM_X | OT_V, AM_Y | OT_V, AM_NOT_USED, "movsd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA6 */ { 0, IT_GENERIC, AM_X | OT_B, AM_Y | OT_B, AM_NOT_USED, "cmpsb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA7 */ { 0, IT_GENERIC, AM_X | OT_V, AM_Y | OT_V, AM_NOT_USED, "cmpsd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA8 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "test", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA9 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "test", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAA */ { 0, IT_GENERIC, AM_Y | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "stosb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAB */ { 0, IT_GENERIC, AM_Y | OT_V, AM_REGISTER | OT_V, AM_NOT_USED, "stosd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAC */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_X| OT_B, AM_NOT_USED, "lodsb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAD */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_X| OT_V, AM_NOT_USED, "lodsd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAE */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_Y | OT_B, AM_NOT_USED, "scasb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAF */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_Y | OT_V, AM_NOT_USED, "scasd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB0 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB1 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB2 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB3 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB4 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB5 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB6 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB7 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB8 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB9 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBA */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBB */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBC */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBD */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBE */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBF */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC0 */ { 6, IT_REFERENCE, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC1 */ { 7, IT_REFERENCE, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC2 */ { 0, IT_RETURN, AM_I | OT_W, AM_NOT_USED, AM_NOT_USED, "ret", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC3 */ { 0, IT_RETURN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "ret", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC4 */ { 0, IT_GENERIC, AM_G | OT_V, AM_M | OT_P, AM_NOT_USED, "les", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC5 */ { 0, IT_GENERIC, AM_G | OT_V, AM_M | OT_P, AM_NOT_USED, "lds", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC6 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC7 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC8 */ { 0, IT_GENERIC, AM_I | OT_W, AM_I | OT_B, AM_NOT_USED, "enter", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC9 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "leave", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCA */ { 0, IT_RETURN, AM_I | OT_W, AM_NOT_USED, AM_NOT_USED, "retf", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCB */ { 0, IT_RETURN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "retf", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCC */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "int3", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCD */ { 0, IT_GENERIC, AM_I | OT_B, AM_NOT_USED, AM_NOT_USED, "int", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCE */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "into", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCF */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "iret", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD0 */ { 8, IT_REFERENCE, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD1 */ { 9, IT_REFERENCE, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD2 */ { 10, IT_REFERENCE, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD3 */ { 11, IT_REFERENCE, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD4 */ { 0, IT_GENERIC, AM_I | OT_B, AM_NOT_USED, AM_NOT_USED, "aam", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD5 */ { 0, IT_GENERIC, AM_I | OT_B, AM_NOT_USED, AM_NOT_USED, "aad", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD6 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD7 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "xlat", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + + // The following 8 lines would be references to the FPU tables, but we currently + // do not support the FPU instructions in this disassembler. + + /* 0xD8 */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD9 */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xDA */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xDB */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xDC */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xDD */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xDE */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xDF */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + + + /* 0xE0 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "loopnz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE1 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "loopz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE2 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "loop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE3 */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jcxz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE4 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "in", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE5 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "in", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE6 */ { 0, IT_GENERIC, AM_I | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "out", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE7 */ { 0, IT_GENERIC, AM_I | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "out", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE8 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "call", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE9 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xEA */ { 0, IT_JUMP, AM_A | OT_P, AM_NOT_USED, AM_NOT_USED, "jmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xEB */ { 0, IT_JUMP, AM_J | OT_B, AM_NOT_USED, AM_NOT_USED, "jmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xEC */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_REGISTER | OT_W, AM_NOT_USED, "in", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xED */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_REGISTER | OT_W, AM_NOT_USED, "in", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xEE */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_REGISTER | OT_B, AM_NOT_USED, "out", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xEF */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_REGISTER | OT_V, AM_NOT_USED, "out", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF0 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "lock:", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF1 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF2 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "repne:", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF3 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "rep:", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF4 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "hlt", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF5 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "cmc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF6 */ { 12, IT_REFERENCE, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF7 */ { 13, IT_REFERENCE, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF8 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "clc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF9 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "stc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xFA */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "cli", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xFB */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "sti", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xFC */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "cld", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xFD */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "std", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xFE */ { 14, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xFF */ { 15, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_0f[] = { + /* 0x0 */ { 16, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 17, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_W, AM_NOT_USED, "lar", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_W, AM_NOT_USED, "lsl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "clts", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "invd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "wbinvd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "ud2", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xE */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x10 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "movups", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "movsd" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "movss" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "movupd" } }, + /* 0x11 */ { 0, IT_GENERIC, AM_W | OT_PS, AM_V | OT_PS, AM_NOT_USED, "movups", true, + /* F2h */ { 0, IT_GENERIC, AM_W | OT_SD, AM_V | OT_SD, AM_NOT_USED, "movsd" }, + /* F3h */ { 0, IT_GENERIC, AM_W | OT_SS, AM_V | OT_SS, AM_NOT_USED, "movss" }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_PD, AM_V | OT_PD, AM_NOT_USED, "movupd" } }, + /* 0x12 */ { 0, IT_GENERIC, AM_W | OT_Q, AM_V | OT_Q, AM_NOT_USED, "movlps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_Q, AM_V | OT_Q, AM_NOT_USED, "movhlps" }, // only one of ... + /* F3h */ { 0, IT_GENERIC, AM_V | OT_Q, AM_V | OT_Q, AM_NOT_USED, "movhlps" }, // ...these two is correct, Intel doesn't specify which + /* 66h */ { 0, IT_GENERIC, AM_V | OT_Q, AM_W | OT_S, AM_NOT_USED, "movlpd" } }, + /* 0x13 */ { 0, IT_GENERIC, AM_V | OT_Q, AM_W | OT_Q, AM_NOT_USED, "movlps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_Q, AM_W | OT_Q, AM_NOT_USED, "movlpd" } }, + /* 0x14 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_Q, AM_NOT_USED, "unpcklps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_Q, AM_NOT_USED, "unpcklpd" } }, + /* 0x15 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_Q, AM_NOT_USED, "unpckhps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_Q, AM_NOT_USED, "unpckhpd" } }, + /* 0x16 */ { 0, IT_GENERIC, AM_V | OT_Q, AM_W | OT_Q, AM_NOT_USED, "movhps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_Q, AM_V | OT_Q, AM_NOT_USED, "movlhps" }, // only one of... + /* F3h */ { 0, IT_GENERIC, AM_V | OT_Q, AM_V | OT_Q, AM_NOT_USED, "movlhps" }, // ...these two is correct, Intel doesn't specify which + /* 66h */ { 0, IT_GENERIC, AM_V | OT_Q, AM_W | OT_Q, AM_NOT_USED, "movhpd" } }, + /* 0x17 */ { 0, IT_GENERIC, AM_W | OT_Q, AM_V | OT_Q, AM_NOT_USED, "movhps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_Q, AM_V | OT_Q, AM_NOT_USED, "movhpd" } }, + /* 0x18 */ { 18, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x19 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1A */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1B */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1C */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1D */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1E */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1F */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x20 */ { 0, IT_GENERIC, AM_R | OT_D, AM_C | OT_D, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x21 */ { 0, IT_GENERIC, AM_R | OT_D, AM_D | OT_D, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x22 */ { 0, IT_GENERIC, AM_C | OT_D, AM_R | OT_D, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x23 */ { 0, IT_GENERIC, AM_D | OT_D, AM_R | OT_D, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x24 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x25 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x26 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x27 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x28 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "movaps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "movapd" } }, + /* 0x29 */ { 0, IT_GENERIC, AM_W | OT_PS, AM_V | OT_PS, AM_NOT_USED, "movaps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_PD, AM_V | OT_PD, AM_NOT_USED, "movapd" } }, + /* 0x2A */ { 0, IT_GENERIC, AM_V | OT_PS, AM_Q | OT_Q, AM_NOT_USED, "cvtpi2ps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_E | OT_D, AM_NOT_USED, "cvtsi2sd" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_E | OT_D, AM_NOT_USED, "cvtsi2ss" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_Q | OT_DQ, AM_NOT_USED, "cvtpi2pd" } }, + /* 0x2B */ { 0, IT_GENERIC, AM_W | OT_PS, AM_V | OT_PS, AM_NOT_USED, "movntps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_PD, AM_V | OT_PD, AM_NOT_USED, "movntpd" } }, + /* 0x2C */ { 0, IT_GENERIC, AM_Q | OT_Q, AM_W | OT_PS, AM_NOT_USED, "cvttps2pi", true, + /* F2h */ { 0, IT_GENERIC, AM_G | OT_D, AM_W | OT_SD, AM_NOT_USED, "cvttsd2si" }, + /* F3h */ { 0, IT_GENERIC, AM_G | OT_D, AM_W | OT_SS, AM_NOT_USED, "cvttss2si" }, + /* 66h */ { 0, IT_GENERIC, AM_Q | OT_DQ, AM_W | OT_PD, AM_NOT_USED, "cvttpd2pi" } }, + /* 0x2D */ { 0, IT_GENERIC, AM_Q | OT_Q, AM_W | OT_PS, AM_NOT_USED, "cvtps2pi", true, + /* F2h */ { 0, IT_GENERIC, AM_G | OT_D, AM_W | OT_SD, AM_NOT_USED, "cvtsd2si" }, + /* F3h */ { 0, IT_GENERIC, AM_G | OT_D, AM_W | OT_SS, AM_NOT_USED, "cvtss2si" }, + /* 66h */ { 0, IT_GENERIC, AM_Q | OT_DQ, AM_W | OT_PD, AM_NOT_USED, "cvtpd2pi" } }, + /* 0x2E */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "ucomiss", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "ucomisd" } }, + /* 0x2F */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_SS, AM_NOT_USED, "comiss", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "comisd" } }, + /* 0x30 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "wrmsr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x31 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "rdtsc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x32 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "rdmsr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x33 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "rdpmc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x34 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "sysenter", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x35 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "sysexit", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x36 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x37 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x38 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x39 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3A */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3B */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3C */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "movnti", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3D */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3E */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3F */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x40 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovo", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x41 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovno", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x42 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x43 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovnc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x44 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x45 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovnz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x46 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovbe", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x47 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmova", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x48 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovs", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x49 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovns", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4A */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovpe", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4B */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovpo", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4C */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4D */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovge", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4E */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovle", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4F */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "cmovg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x50 */ { 0, IT_GENERIC, AM_E | OT_D, AM_V | OT_PS, AM_NOT_USED, "movmskps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_E | OT_D, AM_V | OT_PD, AM_NOT_USED, "movmskpd" } }, + /* 0x51 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "sqrtps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "sqrtsd" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "sqrtss" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "sqrtpd" } }, + /* 0x52 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "rsqrtps", true, + /* F2h */ { 0 }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "rsqrtss" }, + /* 66h */ { 0 } }, + /* 0x53 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "rcpps", true, + /* F2h */ { 0 }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "rcpss" }, + /* 66h */ { 0 } }, + /* 0x54 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "andps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "andpd" } }, + /* 0x55 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "andnps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "andnpd" } }, + /* 0x56 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "orps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "orpd" } }, + /* 0x57 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "xorps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "xorpd" } }, + /* 0x58 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "addps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "addsd" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "addss" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "addpd" } }, + /* 0x59 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "mulps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "mulsd" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "mulss" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "mulpd" } }, + /* 0x5A */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PS, AM_NOT_USED, "cvtps2pd", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "cvtsd2ss" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "cvtss2sd" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PD, AM_NOT_USED, "cvtpd2ps" } }, + /* 0x5B */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_DQ, AM_NOT_USED, "cvtdq2ps", true, + /* F2h */ { 0 }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_PS, AM_NOT_USED, "cvttps2dq" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_PS, AM_NOT_USED, "cvtps2dq" } }, + /* 0x5C */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "subps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "subsd" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "subss" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "subpd" } }, + /* 0x5D */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "minps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "minsd" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "minss" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "minpd" } }, + /* 0x5E */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "divps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "divsd" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "divss" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "divpd" } }, + /* 0x5F */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_NOT_USED, "maxps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_NOT_USED, "maxsd" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_NOT_USED, "maxss" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_NOT_USED, "maxpd" } }, + /* 0x60 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "punpcklbw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "punpcklbw" } }, + /* 0x61 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "punpcklwd", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "punpcklwd" } }, + /* 0x62 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "punpckldq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "punpckldq" } }, + /* 0x63 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "packsswb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "packsswb" } }, + /* 0x64 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "pcmpgtb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pcmpgtb" } }, + /* 0x65 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "pcmpgtw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pcmpgtw" } }, + /* 0x66 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "pcmpgtd", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pcmpgtd" } }, + /* 0x67 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "packuswb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "packuswb" } }, + /* 0x68 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "punpckhbw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_P | OT_DQ, AM_Q | OT_DQ, AM_NOT_USED, "punpckhbw" } }, + /* 0x69 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "punpckhwd", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_P | OT_DQ, AM_Q | OT_DQ, AM_NOT_USED, "punpckhwd" } }, + /* 0x6A */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "punpckhdq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_P | OT_DQ, AM_Q | OT_DQ, AM_NOT_USED, "punpckhdq" } }, + /* 0x6B */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "packssdw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_P | OT_DQ, AM_Q | OT_DQ, AM_NOT_USED, "packssdw" } }, + /* 0x6C */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "not used without prefix", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "punpcklqdq" } }, + /* 0x6D */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "not used without prefix", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "punpcklqdq" } }, + /* 0x6E */ { 0, IT_GENERIC, AM_P | OT_D, AM_E | OT_D, AM_NOT_USED, "movd", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_E | OT_D, AM_NOT_USED, "movd" } }, + /* 0x6F */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_D, AM_NOT_USED, "movq", true, + /* F2h */ { 0 }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "movdqu" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "movdqa" } }, + /* 0x70 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_I | OT_B, "pshuf", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_I | OT_B, "pshuflw" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_I | OT_B, "pshufhw" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_I | OT_B, "pshufd" } }, + /* 0x71 */ { 19, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x72 */ { 20, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x73 */ { 21, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x74 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pcmpeqb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pcmpeqb" } }, + /* 0x75 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pcmpeqw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pcmpeqw" } }, + /* 0x76 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pcmpeqd", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pcmpeqd" } }, + /* 0x77 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "emms", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + + // The following six opcodes are escapes into the MMX stuff, which this disassembler does not support. + /* 0x78 */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x79 */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7A */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7B */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7C */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7D */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + + /* 0x7E */ { 0, IT_GENERIC, AM_E | OT_D, AM_P | OT_D, AM_NOT_USED, "movd", true, + /* F2h */ { 0 }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_Q, AM_W | OT_Q, AM_NOT_USED, "movq" }, + /* 66h */ { 0, IT_GENERIC, AM_E | OT_D, AM_V | OT_DQ, AM_NOT_USED, "movd" } }, + /* 0x7F */ { 0, IT_GENERIC, AM_Q | OT_Q, AM_P | OT_Q, AM_NOT_USED, "movq", true, + /* F2h */ { 0 }, + /* F3h */ { 0, IT_GENERIC, AM_W | OT_DQ, AM_V | OT_DQ, AM_NOT_USED, "movdqu" }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_DQ, AM_V | OT_DQ, AM_NOT_USED, "movdqa" } }, + /* 0x80 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jo", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x81 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jno", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x82 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x83 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jnc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x84 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x85 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jnz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x86 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jbe", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x87 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "ja", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x88 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "js", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x89 */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jns", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8A */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jpe", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8B */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jpo", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8C */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8D */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jge", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8E */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jle", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x8F */ { 0, IT_JUMP, AM_J | OT_V, AM_NOT_USED, AM_NOT_USED, "jg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x90 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "seto", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x91 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setno", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x92 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x93 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setnc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x94 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x95 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setnz", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x96 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setbe", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x97 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "seta", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x98 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "sets", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x99 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setns", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9A */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setpe", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9B */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setpo", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9C */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9D */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setge", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9E */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setle", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x9F */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "setg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA0 */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA1 */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA2 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "cpuid", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA3 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "bt", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA4 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_I | OT_B, "shld", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA5 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_I | OT_B | AM_REGISTER, "shld", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA6 */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA7 */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA8 */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xA9 */ { 0, IT_GENERIC, AM_REGISTER | OT_W, AM_NOT_USED, AM_NOT_USED, "pop", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAA */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "rsm", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAB */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "bts", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAC */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_I | OT_B, "shrd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAD */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_I | OT_B | AM_REGISTER, "shrd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAE */ { 22, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xAF */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "imul", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB0 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "cmpxchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "cmpxchg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB2 */ { 0, IT_GENERIC, AM_M | OT_P, AM_NOT_USED, AM_NOT_USED, "lss", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB3 */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "btr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB4 */ { 0, IT_GENERIC, AM_M | OT_P, AM_NOT_USED, AM_NOT_USED, "lfs", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB5 */ { 0, IT_GENERIC, AM_M | OT_P, AM_NOT_USED, AM_NOT_USED, "lgs", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB6 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_B, AM_NOT_USED, "movzx", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB7 */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_W, AM_NOT_USED, "movzx", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB8 */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB9 */ { 0, IT_UNKNOWN, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "ud1", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBA */ { 23, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBB */ { 0, IT_GENERIC, AM_E | OT_V, AM_G | OT_V, AM_NOT_USED, "btc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBC */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "bsf", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBD */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_V, AM_NOT_USED, "bsr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBE */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_B, AM_NOT_USED, "movsx", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBF */ { 0, IT_GENERIC, AM_G | OT_V, AM_E | OT_W, AM_NOT_USED, "movsx", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC0 */ { 0, IT_GENERIC, AM_E | OT_B, AM_G | OT_B, AM_NOT_USED, "xadd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, "xadd", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC2 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_I | OT_B, "cmpps", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_SD, AM_W | OT_SD, AM_I | OT_B, "cmpsd" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_SS, AM_W | OT_SS, AM_I | OT_B, "cmpss" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_I | OT_B, "cmppd" } }, + /* 0xC3 */ { 0, IT_GENERIC, AM_E | OT_D, AM_G | OT_D, AM_NOT_USED, "movnti", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC4 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_E | OT_D, AM_I | OT_B, "pinsrw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_E | OT_D, AM_I | OT_B, "pinsrw" } }, + /* 0xC5 */ { 0, IT_GENERIC, AM_G | OT_D, AM_P | OT_Q, AM_I | OT_B, "pextrw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_G | OT_D, AM_V | OT_DQ, AM_I | OT_B, "pextrw" } }, + /* 0xC6 */ { 0, IT_GENERIC, AM_V | OT_PS, AM_W | OT_PS, AM_I | OT_B, "shufps", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_PD, AM_I | OT_B, "shufpd" } }, + /* 0xC7 */ { 24, IT_REFERENCE, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC8 */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "bswap", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xC9 */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "bswap", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCA */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "bswap", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCB */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "bswap", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCC */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "bswap", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCD */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "bswap", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCE */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "bswap", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xCF */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "bswap", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD0 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xD1 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psrlw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psrlw" } }, + /* 0xD2 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psrld", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psrld" } }, + /* 0xD3 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psrlq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psrlq" } }, + /* 0xD4 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "paddq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "paddq" } }, + /* 0xD5 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pmullw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pmullw" } }, + /* 0xD6 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "unused without prefix", true, + /* F2h */ { 0, IT_GENERIC, AM_P | OT_Q, AM_W | OT_Q, AM_NOT_USED, "movdq2q" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_Q | OT_Q, AM_NOT_USED, "movq2dq" }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_Q, AM_V | OT_Q, AM_NOT_USED, "movq" } }, + /* 0xD7 */ { 0, IT_GENERIC, AM_G | OT_D, AM_P | OT_Q, AM_NOT_USED, "pmovmskb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_G | OT_D, AM_V | OT_DQ, AM_NOT_USED, "pmovmskb" } }, + /* 0xD8 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psubusb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psubusb" } }, + /* 0xD9 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psubusw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psubusw" } }, + /* 0xDA */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pminub", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pminub" } }, + /* 0xDB */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pand", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pand" } }, + /* 0xDC */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "paddusb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "paddusb" } }, + /* 0xDD */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "paddusw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "paddusw" } }, + /* 0xDE */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pmaxub", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pmaxub" } }, + /* 0xDF */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pandn", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pandn" } }, + /* 0xE0 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pavgb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pavgb" } }, + /* 0xE1 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psraw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psrqw" } }, + /* 0xE2 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psrad", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psrad" } }, + /* 0xE3 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pavgw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pavgw" } }, + /* 0xE4 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pmulhuw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pmulhuw" } }, + /* 0xE5 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pmulhuw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pmulhw" } }, + /* 0xE6 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "not used without prefix", true, + /* F2h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_PD, AM_NOT_USED, "cvtpd2dq" }, + /* F3h */ { 0, IT_GENERIC, AM_V | OT_PD, AM_W | OT_DQ, AM_NOT_USED, "cvtdq2pd" }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_PD, AM_NOT_USED, "cvttpd2dq" } }, + /* 0xE7 */ { 0, IT_GENERIC, AM_W | OT_Q, AM_V | OT_Q, AM_NOT_USED, "movntq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_DQ, AM_V | OT_DQ, AM_NOT_USED, "movntdq" } }, + /* 0xE8 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psubsb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psubsb" } }, + /* 0xE9 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psubsw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psubsw" } }, + /* 0xEA */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pminsw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pminsw" } }, + /* 0xEB */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "por", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "por" } }, + /* 0xEC */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "paddsb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "paddsb" } }, + /* 0xED */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "paddsw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "paddsw" } }, + /* 0xEE */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pmaxsw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pmaxsw" } }, + /* 0xEF */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pxor", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pxor" } }, + /* 0xF0 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xF1 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psllw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psllw" } }, + /* 0xF2 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pslld", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pslld" } }, + /* 0xF3 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psllq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psllq" } }, + /* 0xF4 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pmuludq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pmuludq" } }, + /* 0xF5 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "pmaddwd", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "pmaddwd" } }, + /* 0xF6 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psadbw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psadbw" } }, + /* 0xF7 */ { 0, IT_GENERIC, AM_P | OT_PI, AM_Q | OT_PI, AM_NOT_USED, "maskmovq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "maskmovdqu" } }, + /* 0xF8 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psubb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psubb" } }, + /* 0xF9 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psubw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psubw" } }, + /* 0xFA */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psubd", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psubd" } }, + /* 0xFB */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "psubq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "psubq" } }, + /* 0xFC */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "paddb", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "paddb" } }, + /* 0xFD */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "paddw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "paddw" } }, + /* 0xFE */ { 0, IT_GENERIC, AM_P | OT_Q, AM_Q | OT_Q, AM_NOT_USED, "paddd", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_V | OT_DQ, AM_W | OT_DQ, AM_NOT_USED, "paddd" } }, + /* 0xFF */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_0f00[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_W, AM_NOT_USED, AM_NOT_USED, "sldt", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_W, AM_NOT_USED, AM_NOT_USED, "str", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_W, AM_NOT_USED, AM_NOT_USED, "lldt", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_W, AM_NOT_USED, AM_NOT_USED, "ltr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_W, AM_NOT_USED, AM_NOT_USED, "verr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_W, AM_NOT_USED, AM_NOT_USED, "verw", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_0f01[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_M | OT_S, AM_NOT_USED, AM_NOT_USED, "sgdt", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_M | OT_S, AM_NOT_USED, AM_NOT_USED, "sidt", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_M | OT_S, AM_NOT_USED, AM_NOT_USED, "lgdt", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_M | OT_S, AM_NOT_USED, AM_NOT_USED, "lidt", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_W, AM_NOT_USED, AM_NOT_USED, "smsw", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_W, AM_NOT_USED, AM_NOT_USED, "lmsw", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_M | OT_B, AM_NOT_USED, AM_NOT_USED, "invlpg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_0f18[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_M | OT_ADDRESS_MODE_M, AM_NOT_USED, AM_NOT_USED, "prefetch", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "prefetch", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "prefetch", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_REGISTER | OT_D, AM_NOT_USED, AM_NOT_USED, "prefetch", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_0f71[] = { + /* 0x0 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_I | OT_B, AM_NOT_USED, "psrlw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_P | OT_DQ, AM_I | OT_B, AM_NOT_USED, "psrlw" } }, + /* 0x3 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_I | OT_B, AM_NOT_USED, "psraw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_P | OT_DQ, AM_I | OT_B, AM_NOT_USED, "psraw" } }, + /* 0x5 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_I | OT_B, AM_NOT_USED, "psllw", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_P | OT_DQ, AM_I | OT_B, AM_NOT_USED, "psllw" } }, + /* 0x7 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_0f72[] = { + /* 0x0 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_I | OT_B, AM_NOT_USED, "psrld", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_DQ, AM_I | OT_B, AM_NOT_USED, "psrld" } }, + /* 0x3 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_I | OT_B, AM_NOT_USED, "psrad", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_DQ, AM_I | OT_B, AM_NOT_USED, "psrad" } }, + /* 0x5 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_I | OT_B, AM_NOT_USED, "pslld", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_DQ, AM_I | OT_B, AM_NOT_USED, "pslld" } }, + /* 0x7 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_0f73[] = { + /* 0x0 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_I | OT_B, AM_NOT_USED, "psrlq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_DQ, AM_I | OT_B, AM_NOT_USED, "psrlq" } }, + /* 0x3 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_P | OT_Q, AM_I | OT_B, AM_NOT_USED, "psllq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_DQ, AM_I | OT_B, AM_NOT_USED, "psllq" } }, + /* 0x7 */ { 0, IT_GENERIC, AM_W | OT_DQ, AM_I | OT_B, AM_NOT_USED, "pslldq", true, + /* F2h */ { 0 }, + /* F3h */ { 0 }, + /* 66h */ { 0, IT_GENERIC, AM_W | OT_DQ, AM_I | OT_B, AM_NOT_USED, "pslldq" } }, +}; + +const Opcode s_opcode_byte_after_0fae[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "fxsave", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "fxrstor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "ldmxcsr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "stmxcsr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "lfence", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "mfence", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "clflush/sfence", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, +}; + +const Opcode s_opcode_byte_after_0fba[] = { + /* 0x0 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "bt", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "bts", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "btr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "btc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_0fc7[] = { + /* 0x0 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_M | OT_Q, AM_NOT_USED, AM_NOT_USED, "cmpxch8b", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_80[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "add", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "or", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "adc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "sbb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "and", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "sub", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "xor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_81[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "add", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "or", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "adc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "sbb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "and", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "sub", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "xor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_82[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "add", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "or", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "adc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "sbb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "and", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "sub", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "xor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_83[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "add", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "or", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "adc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "sbb", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "and", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "sub", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "xor", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_c0[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "rol", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "ror", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "rcl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "rcr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "shl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "shr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "sal", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "sar", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_c1[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "rol", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "ror", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "rcl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "rcr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "shl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "shr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "sal", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, "sar", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_d0[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_B, AM_IMPLICIT, AM_NOT_USED, "rol", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_B, AM_IMPLICIT, AM_NOT_USED, "ror", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_B, AM_IMPLICIT, AM_NOT_USED, "rcl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_B, AM_IMPLICIT, AM_NOT_USED, "rcr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_B, AM_IMPLICIT, AM_NOT_USED, "shl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_B, AM_IMPLICIT, AM_NOT_USED, "shr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_B, AM_IMPLICIT, AM_NOT_USED, "sal", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_B, AM_IMPLICIT, AM_NOT_USED, "sar", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_d1[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_V, AM_IMPLICIT, AM_NOT_USED, "rol", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_IMPLICIT, AM_NOT_USED, "ror", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_V, AM_IMPLICIT, AM_NOT_USED, "rcl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_V, AM_IMPLICIT, AM_NOT_USED, "rcr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_V, AM_IMPLICIT, AM_NOT_USED, "shl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_V, AM_IMPLICIT, AM_NOT_USED, "shr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_V, AM_IMPLICIT, AM_NOT_USED, "sal", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_V, AM_IMPLICIT, AM_NOT_USED, "sar", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_d2[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "rol", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "ror", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "rcl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "rcr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "shl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "shr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "sal", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_B, AM_REGISTER | OT_B, AM_NOT_USED, "sar", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_d3[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_V, AM_REGISTER | OT_B, AM_NOT_USED, "rol", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_REGISTER | OT_B, AM_NOT_USED, "ror", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_V, AM_REGISTER | OT_B, AM_NOT_USED, "rcl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_V, AM_REGISTER | OT_B, AM_NOT_USED, "rcr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_E | OT_V, AM_REGISTER | OT_B, AM_NOT_USED, "shl", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_E | OT_V, AM_REGISTER | OT_B, AM_NOT_USED, "shr", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_V, AM_REGISTER | OT_B, AM_NOT_USED, "sal", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_E | OT_V, AM_REGISTER | OT_B, AM_NOT_USED, "sar", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_f6[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "test", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, "test", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "not", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "neg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, OT_B | AM_REGISTER, AM_E | OT_B, AM_NOT_USED, "mul", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, OT_B | AM_REGISTER, AM_E | OT_B, AM_NOT_USED, "imul", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_E | OT_B, AM_NOT_USED, "div", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_E | OT_B, AM_NOT_USED, "idiv", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_f7[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "test", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_I | OT_V, AM_NOT_USED, "test", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_GENERIC, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, "not", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_GENERIC, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, "neg", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_E | OT_V, AM_NOT_USED, "mul", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_E | OT_V, AM_NOT_USED, "imul", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_E | OT_V, AM_NOT_USED, "div", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_E | OT_V, AM_NOT_USED, "idiv", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_fe[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_B, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +const Opcode s_opcode_byte_after_ff[] = { + /* 0x0 */ { 0, IT_GENERIC, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x1 */ { 0, IT_GENERIC, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x2 */ { 0, IT_JUMP, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, "call", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x3 */ { 0, IT_JUMP, AM_E | OT_P, AM_NOT_USED, AM_NOT_USED, "call", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4 */ { 0, IT_JUMP, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, "jmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x5 */ { 0, IT_JUMP, AM_E | OT_P, AM_NOT_USED, AM_NOT_USED, "jmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x6 */ { 0, IT_GENERIC, AM_E | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x7 */ { 0, IT_UNUSED, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } } +}; + +/* +* A table of all the other tables, containing some extra information, e.g. +* how to mask out the byte we're looking at. +*/ +const OpcodeTable MiniDisassembler::s_ia32_opcode_map_[]={ + // One-byte opcodes and jumps to larger + /* 0 */ {s_first_opcode_byte, 0, 0xff, 0, 0xff}, + // Two-byte opcodes (second byte) + /* 1 */ {s_opcode_byte_after_0f, 0, 0xff, 0, 0xff}, + // Start of tables for opcodes using ModR/M bits as extension + /* 2 */ {s_opcode_byte_after_80, 3, 0x07, 0, 0x07}, + /* 3 */ {s_opcode_byte_after_81, 3, 0x07, 0, 0x07}, + /* 4 */ {s_opcode_byte_after_82, 3, 0x07, 0, 0x07}, + /* 5 */ {s_opcode_byte_after_83, 3, 0x07, 0, 0x07}, + /* 6 */ {s_opcode_byte_after_c0, 3, 0x07, 0, 0x07}, + /* 7 */ {s_opcode_byte_after_c1, 3, 0x07, 0, 0x07}, + /* 8 */ {s_opcode_byte_after_d0, 3, 0x07, 0, 0x07}, + /* 9 */ {s_opcode_byte_after_d1, 3, 0x07, 0, 0x07}, + /* 10 */ {s_opcode_byte_after_d2, 3, 0x07, 0, 0x07}, + /* 11 */ {s_opcode_byte_after_d3, 3, 0x07, 0, 0x07}, + /* 12 */ {s_opcode_byte_after_f6, 3, 0x07, 0, 0x07}, + /* 13 */ {s_opcode_byte_after_f7, 3, 0x07, 0, 0x07}, + /* 14 */ {s_opcode_byte_after_fe, 3, 0x07, 0, 0x01}, + /* 15 */ {s_opcode_byte_after_ff, 3, 0x07, 0, 0x07}, + /* 16 */ {s_opcode_byte_after_0f00, 3, 0x07, 0, 0x07}, + /* 17 */ {s_opcode_byte_after_0f01, 3, 0x07, 0, 0x07}, + /* 18 */ {s_opcode_byte_after_0f18, 3, 0x07, 0, 0x07}, + /* 19 */ {s_opcode_byte_after_0f71, 3, 0x07, 0, 0x07}, + /* 20 */ {s_opcode_byte_after_0f72, 3, 0x07, 0, 0x07}, + /* 21 */ {s_opcode_byte_after_0f73, 3, 0x07, 0, 0x07}, + /* 22 */ {s_opcode_byte_after_0fae, 3, 0x07, 0, 0x07}, + /* 23 */ {s_opcode_byte_after_0fba, 3, 0x07, 0, 0x07}, + /* 24 */ {s_opcode_byte_after_0fc7, 3, 0x07, 0, 0x01} +}; + +}; // namespace sidestep diff --git a/tools/memory_watcher/memory_hook.cc b/tools/memory_watcher/memory_hook.cc new file mode 100644 index 0000000..2997fc0 --- /dev/null +++ b/tools/memory_watcher/memory_hook.cc @@ -0,0 +1,580 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Static class for hooking Win32 API routines. + +// Some notes about how to hook Memory Allocation Routines in Windows. +// +// For our purposes we do not hook the libc routines. There are two +// reasons for this. First, the libc routines all go through HeapAlloc +// anyway. So, it's redundant to log both HeapAlloc and malloc. +// Second, it can be tricky to hook in both static and dynamic linkages +// of libc. + +#include <windows.h> + +#include "memory_hook.h" +#include "memory_watcher.h" +#include "preamble_patcher.h" + +// Calls GetProcAddress, but casts to the correct type. +#define GET_PROC_ADDRESS(hmodule, name) \ + ( (Type_##name)(::GetProcAddress(hmodule, #name)) ) + +// Macro to declare Patch functions. +#define DECLARE_PATCH(name) Patch<Type_##name> patch_##name + +// Macro to install Patch functions. +#define INSTALL_PATCH(name) do { \ + patch_##name.set_original(GET_PROC_ADDRESS(hkernel32, ##name)); \ + patch_##name.Install(&Perftools_##name); \ +} while (0) + +// Macro to install Patch functions. +#define INSTALL_NTDLLPATCH(name) do { \ + patch_##name.set_original(GET_PROC_ADDRESS(hntdll, ##name)); \ + patch_##name.Install(&Perftools_##name); \ +} while (0) + +// Macro to uninstall Patch functions. +#define UNINSTALL_PATCH(name) patch_##name.Uninstall(); + + + +// Windows APIs to be hooked + +// HeapAlloc routines +typedef HANDLE (WINAPI *Type_HeapCreate)(DWORD flOptions, + SIZE_T dwInitialSize, + SIZE_T dwMaximumSize); +typedef BOOL (WINAPI *Type_HeapDestroy)(HANDLE hHeap); +typedef LPVOID (WINAPI *Type_HeapAlloc)(HANDLE hHeap, DWORD dwFlags, + DWORD_PTR dwBytes); +typedef LPVOID (WINAPI *Type_HeapReAlloc)(HANDLE hHeap, DWORD dwFlags, + LPVOID lpMem, SIZE_T dwBytes); +typedef BOOL (WINAPI *Type_HeapFree)(HANDLE hHeap, DWORD dwFlags, + LPVOID lpMem); + +// GlobalAlloc routines +typedef HGLOBAL (WINAPI *Type_GlobalAlloc)(UINT uFlags, SIZE_T dwBytes); +typedef HGLOBAL (WINAPI *Type_GlobalReAlloc)(HGLOBAL hMem, SIZE_T dwBytes, + UINT uFlags); +typedef HGLOBAL (WINAPI *Type_GlobalFree)(HGLOBAL hMem); + +// LocalAlloc routines +typedef HLOCAL (WINAPI *Type_LocalAlloc)(UINT uFlags, SIZE_T uBytes); +typedef HLOCAL (WINAPI *Type_LocalReAlloc)(HLOCAL hMem, SIZE_T uBytes, + UINT uFlags); +typedef HLOCAL (WINAPI *Type_LocalFree)(HLOCAL hMem); + +// A Windows-API equivalent of mmap and munmap, for "anonymous regions" +typedef LPVOID (WINAPI *Type_VirtualAllocEx)(HANDLE process, LPVOID address, + SIZE_T size, DWORD type, + DWORD protect); +typedef BOOL (WINAPI *Type_VirtualFreeEx)(HANDLE process, LPVOID address, + SIZE_T size, DWORD type); + +// A Windows-API equivalent of mmap and munmap, for actual files +typedef LPVOID (WINAPI *Type_MapViewOfFile)(HANDLE hFileMappingObject, + DWORD dwDesiredAccess, + DWORD dwFileOffsetHigh, + DWORD dwFileOffsetLow, + SIZE_T dwNumberOfBytesToMap); +typedef LPVOID (WINAPI *Type_MapViewOfFileEx)(HANDLE hFileMappingObject, + DWORD dwDesiredAccess, + DWORD dwFileOffsetHigh, + DWORD dwFileOffsetLow, + SIZE_T dwNumberOfBytesToMap, + LPVOID lpBaseAddress); +typedef BOOL (WINAPI *Type_UnmapViewOfFile)(LPVOID lpBaseAddress); + +typedef DWORD (WINAPI *Type_NtUnmapViewOfSection)(HANDLE process, + LPVOID lpBaseAddress); + + +// Patch is a template for keeping the pointer to the original +// hooked routine, the function to call when hooked, and the +// stub routine which is patched. +template<class T> +class Patch { + public: + // Constructor. Does not hook the function yet. + Patch<T>() + : original_function_(NULL), + patch_function_(NULL), + stub_function_(NULL) { + } + + // Destructor. Unhooks the function if it has been hooked. + ~Patch<T>() { + Uninstall(); + } + + // Patches original function with func. + // Must have called set_original to set the original function. + void Install(T func) { + patch_function_ = func; + CHECK(patch_function_ != NULL); + CHECK(original_function_ != NULL); + CHECK(stub_function_ == NULL); + CHECK(sidestep::SIDESTEP_SUCCESS == + sidestep::PreamblePatcher::Patch(original_function_, + patch_function_, &stub_function_)); + } + + // Un-patches the function. + void Uninstall() { + if (stub_function_) + sidestep::PreamblePatcher::Unpatch(original_function_, + patch_function_, stub_function_); + stub_function_ = NULL; + } + + // Set the function to be patched. + void set_original(T original) { original_function_ = original; } + + // Get the original function being patched. + T original() { return original_function_; } + + // Get the patched function. (e.g. the replacement function) + T patched() { return patch_function_; } + + // Access to the stub for calling the original function + // while it is patched. + T operator()() { + DCHECK(stub_function_); + return stub_function_; + } + + private: + // The function that we plan to patch. + T original_function_; + // The function to replace the original with. + T patch_function_; + // To unpatch, we also need to keep around a "stub" that points to the + // pre-patched Windows function. + T stub_function_; +}; + + +// All Windows memory-allocation routines call through to one of these. +DECLARE_PATCH(HeapCreate); +DECLARE_PATCH(HeapDestroy); +DECLARE_PATCH(HeapAlloc); +DECLARE_PATCH(HeapReAlloc); +DECLARE_PATCH(HeapFree); +DECLARE_PATCH(VirtualAllocEx); +DECLARE_PATCH(VirtualFreeEx); +DECLARE_PATCH(MapViewOfFile); +DECLARE_PATCH(MapViewOfFileEx); +DECLARE_PATCH(UnmapViewOfFile); +DECLARE_PATCH(GlobalAlloc); +DECLARE_PATCH(GlobalReAlloc); +DECLARE_PATCH(GlobalFree); +DECLARE_PATCH(LocalAlloc); +DECLARE_PATCH(LocalReAlloc); +DECLARE_PATCH(LocalFree); +DECLARE_PATCH(NtUnmapViewOfSection); + +// Our replacement functions. + +static HANDLE WINAPI Perftools_HeapCreate(DWORD flOptions, + SIZE_T dwInitialSize, + SIZE_T dwMaximumSize) { + if (dwInitialSize > 4096) + dwInitialSize = 4096; + return patch_HeapCreate()(flOptions, dwInitialSize, dwMaximumSize); +} + +static BOOL WINAPI Perftools_HeapDestroy(HANDLE hHeap) { + return patch_HeapDestroy()(hHeap); +} + +static LPVOID WINAPI Perftools_HeapAlloc(HANDLE hHeap, DWORD dwFlags, + DWORD_PTR dwBytes) { + LPVOID rv = patch_HeapAlloc()(hHeap, dwFlags, dwBytes); + MemoryHook::hook()->OnTrack(hHeap, reinterpret_cast<int32>(rv), dwBytes); + return rv; +} + +static BOOL WINAPI Perftools_HeapFree(HANDLE hHeap, DWORD dwFlags, + LPVOID lpMem) { + size_t size = 0; + if (lpMem != 0) { + size = HeapSize(hHeap, 0, lpMem); // Will crash if lpMem is 0. + // Note: size could be 0; HeapAlloc does allocate 0 length buffers. + } + MemoryHook::hook()->OnUntrack(hHeap, reinterpret_cast<int32>(lpMem), size); + return patch_HeapFree()(hHeap, dwFlags, lpMem); +} + +static LPVOID WINAPI Perftools_HeapReAlloc(HANDLE hHeap, DWORD dwFlags, + LPVOID lpMem, SIZE_T dwBytes) { + // Don't call realloc, but instead do a free/malloc. The problem is that + // the builtin realloc may either expand a buffer, or it may simply + // just call free/malloc. If so, we will already have tracked the new + // block via Perftools_HeapAlloc. + + LPVOID rv = Perftools_HeapAlloc(hHeap, dwFlags, dwBytes); + + // If there was an old buffer, now copy the data to the new buffer. + if (lpMem != 0) { + size_t size = HeapSize(hHeap, 0, lpMem); + if (size > dwBytes) + size = dwBytes; + // Note: size could be 0; HeapAlloc does allocate 0 length buffers. + memcpy(rv, lpMem, size); + Perftools_HeapFree(hHeap, dwFlags, lpMem); + } + return rv; +} + +static LPVOID WINAPI Perftools_VirtualAllocEx(HANDLE process, LPVOID address, + SIZE_T size, DWORD type, + DWORD protect) { + bool already_committed = false; + if (address != NULL) { + MEMORY_BASIC_INFORMATION info; + CHECK(VirtualQuery(address, &info, sizeof(info))); + if (info.State & MEM_COMMIT) + already_committed = true; + } + bool reserving = (address == NULL) || (type & MEM_RESERVE); + bool committing = !already_committed && (type & MEM_COMMIT); + + + LPVOID result = patch_VirtualAllocEx()(process, address, size, type, + protect); + MEMORY_BASIC_INFORMATION info; + CHECK(VirtualQuery(result, &info, sizeof(info))); + size = info.RegionSize; + + if (committing) + MemoryHook::hook()->OnTrack(0, reinterpret_cast<int32>(result), size); + + return result; +} + +static BOOL WINAPI Perftools_VirtualFreeEx(HANDLE process, LPVOID address, + SIZE_T size, DWORD type) { + int chunk_size = size; + MEMORY_BASIC_INFORMATION info; + CHECK(VirtualQuery(address, &info, sizeof(info))); + if (chunk_size == 0) + chunk_size = info.RegionSize; + bool decommit = (info.State & MEM_COMMIT); + + if (decommit) + MemoryHook::hook()->OnUntrack(0, reinterpret_cast<int32>(address), + chunk_size); + + return patch_VirtualFreeEx()(process, address, size, type); +} + +static Lock known_maps_lock; +static std::map<void*, int> known_maps; + +static LPVOID WINAPI Perftools_MapViewOfFileEx(HANDLE hFileMappingObject, + DWORD dwDesiredAccess, + DWORD dwFileOffsetHigh, + DWORD dwFileOffsetLow, + SIZE_T dwNumberOfBytesToMap, + LPVOID lpBaseAddress) { + // For this function pair, you always deallocate the full block of + // data that you allocate, so NewHook/DeleteHook is the right API. + LPVOID result = patch_MapViewOfFileEx()(hFileMappingObject, dwDesiredAccess, + dwFileOffsetHigh, dwFileOffsetLow, + dwNumberOfBytesToMap, lpBaseAddress); + { + AutoLock lock(known_maps_lock); + MEMORY_BASIC_INFORMATION info; + if (known_maps.find(result) == known_maps.end()) { + CHECK(VirtualQuery(result, &info, sizeof(info))); + // TODO(mbelshe): THIS map uses the standard heap!!!! + known_maps[result] = 1; + MemoryHook::hook()->OnTrack(0, reinterpret_cast<int32>(result), + info.RegionSize); + } else { + known_maps[result] = known_maps[result] + 1; + } + } + return result; +} + +static LPVOID WINAPI Perftools_MapViewOfFile(HANDLE hFileMappingObject, + DWORD dwDesiredAccess, + DWORD dwFileOffsetHigh, + DWORD dwFileOffsetLow, + SIZE_T dwNumberOfBytesToMap) { + return Perftools_MapViewOfFileEx(hFileMappingObject, dwDesiredAccess, + dwFileOffsetHigh, dwFileOffsetLow, + dwNumberOfBytesToMap, 0); +} + +static BOOL WINAPI Perftools_UnmapViewOfFile(LPVOID lpBaseAddress) { + // This will call into NtUnmapViewOfSection(). + return patch_UnmapViewOfFile()(lpBaseAddress); +} + +static DWORD WINAPI Perftools_NtUnmapViewOfSection(HANDLE process, + LPVOID lpBaseAddress) { + // Some windows APIs call directly into this routine rather + // than calling UnmapViewOfFile. If we didn't trap this function, + // then we appear to have bogus leaks. + { + AutoLock lock(known_maps_lock); + MEMORY_BASIC_INFORMATION info; + CHECK(VirtualQuery(lpBaseAddress, &info, sizeof(info))); + if (known_maps.find(lpBaseAddress) != known_maps.end()) { + if (known_maps[lpBaseAddress] == 1) { + MemoryHook::hook()->OnUntrack(0, reinterpret_cast<int32>(lpBaseAddress), + info.RegionSize); + known_maps.erase(lpBaseAddress); + } else { + known_maps[lpBaseAddress] = known_maps[lpBaseAddress] - 1; + } + } + } + return patch_NtUnmapViewOfSection()(process, lpBaseAddress); +} + +static HGLOBAL WINAPI Perftools_GlobalAlloc(UINT uFlags, SIZE_T dwBytes) { + // GlobalAlloc is built atop HeapAlloc anyway. So we don't track these. + // GlobalAlloc will internally call into HeapAlloc and we track there. + + // Force all memory to be fixed. + uFlags &= ~GMEM_MOVEABLE; + HGLOBAL rv = patch_GlobalAlloc()(uFlags, dwBytes); + return rv; +} + +static HGLOBAL WINAPI Perftools_GlobalFree(HGLOBAL hMem) { + return patch_GlobalFree()(hMem); +} + +static HGLOBAL WINAPI Perftools_GlobalReAlloc(HGLOBAL hMem, SIZE_T dwBytes, + UINT uFlags) { + // GlobalDiscard is a macro which calls LocalReAlloc with size 0. + if (dwBytes == 0) { + return patch_GlobalReAlloc()(hMem, dwBytes, uFlags); + } + + HGLOBAL rv = Perftools_GlobalAlloc(uFlags, dwBytes); + if (hMem != 0) { + size_t size = GlobalSize(hMem); + if (size > dwBytes) + size = dwBytes; + // Note: size could be 0; HeapAlloc does allocate 0 length buffers. + memcpy(rv, hMem, size); + Perftools_GlobalFree(hMem); + } + + return rv; +} + +static HLOCAL WINAPI Perftools_LocalAlloc(UINT uFlags, SIZE_T dwBytes) { + // LocalAlloc is built atop HeapAlloc anyway. So we don't track these. + // LocalAlloc will internally call into HeapAlloc and we track there. + + // Force all memory to be fixed. + uFlags &= ~LMEM_MOVEABLE; + HLOCAL rv = patch_LocalAlloc()(uFlags, dwBytes); + return rv; +} + +static HLOCAL WINAPI Perftools_LocalFree(HLOCAL hMem) { + return patch_LocalFree()(hMem); +} + +static HLOCAL WINAPI Perftools_LocalReAlloc(HLOCAL hMem, SIZE_T dwBytes, + UINT uFlags) { + // LocalDiscard is a macro which calls LocalReAlloc with size 0. + if (dwBytes == 0) { + return patch_LocalReAlloc()(hMem, dwBytes, uFlags); + } + + HGLOBAL rv = Perftools_LocalAlloc(uFlags, dwBytes); + if (hMem != 0) { + size_t size = LocalSize(hMem); + if (size > dwBytes) + size = dwBytes; + // Note: size could be 0; HeapAlloc does allocate 0 length buffers. + memcpy(rv, hMem, size); + Perftools_LocalFree(hMem); + } + + return rv; +} + +bool MemoryHook::hooked_ = false; +MemoryHook* MemoryHook::global_hook_ = NULL; + +MemoryHook::MemoryHook() + : watcher_(NULL), + heap_(NULL) { + CreateHeap(); +} + +MemoryHook::~MemoryHook() { + // It's a bit dangerous to ever close this heap; MemoryWatchers may have + // used this heap for their tracking data. Closing the heap while any + // MemoryWatchers still exist is pretty dangerous. + CloseHeap(); +} + +bool MemoryHook::Initialize() { + if (global_hook_ == NULL) + global_hook_ = new MemoryHook(); + return true; +} + +bool MemoryHook::Hook() { + DCHECK(!hooked_); + if (!hooked_) { + DCHECK(global_hook_); + + // Luckily, Patch() doesn't call malloc or windows alloc routines + // itself -- though it does call new (we can use PatchWithStub to + // get around that, and will need to if we need to patch new). + + HMODULE hkernel32 = ::GetModuleHandle(L"kernel32"); + CHECK(hkernel32 != NULL); + + HMODULE hntdll = ::GetModuleHandle(L"ntdll"); + CHECK(hntdll != NULL); + + // Now that we've found all the functions, patch them + INSTALL_PATCH(HeapCreate); + INSTALL_PATCH(HeapDestroy); + INSTALL_PATCH(HeapAlloc); + INSTALL_PATCH(HeapReAlloc); + INSTALL_PATCH(HeapFree); + INSTALL_PATCH(VirtualAllocEx); + INSTALL_PATCH(VirtualFreeEx); + INSTALL_PATCH(MapViewOfFileEx); + INSTALL_PATCH(MapViewOfFile); + INSTALL_PATCH(UnmapViewOfFile); + INSTALL_NTDLLPATCH(NtUnmapViewOfSection); + INSTALL_PATCH(GlobalAlloc); + INSTALL_PATCH(GlobalReAlloc); + INSTALL_PATCH(GlobalFree); + INSTALL_PATCH(LocalAlloc); + INSTALL_PATCH(LocalReAlloc); + INSTALL_PATCH(LocalFree); + + // We are finally completely hooked. + hooked_ = true; + } + return true; +} + +bool MemoryHook::Unhook() { + if (hooked_) { + // We need to go back to the system malloc/etc at global destruct time, + // so objects that were constructed before tcmalloc, using the system + // malloc, can destroy themselves using the system free. This depends + // on DLLs unloading in the reverse order in which they load! + // + // We also go back to the default HeapAlloc/etc, just for consistency. + // Who knows, it may help avoid weird bugs in some situations. + UNINSTALL_PATCH(HeapCreate); + UNINSTALL_PATCH(HeapDestroy); + UNINSTALL_PATCH(HeapAlloc); + UNINSTALL_PATCH(HeapReAlloc); + UNINSTALL_PATCH(HeapFree); + UNINSTALL_PATCH(VirtualAllocEx); + UNINSTALL_PATCH(VirtualFreeEx); + UNINSTALL_PATCH(MapViewOfFile); + UNINSTALL_PATCH(MapViewOfFileEx); + UNINSTALL_PATCH(UnmapViewOfFile); + UNINSTALL_PATCH(NtUnmapViewOfSection); + UNINSTALL_PATCH(GlobalAlloc); + UNINSTALL_PATCH(GlobalReAlloc); + UNINSTALL_PATCH(GlobalFree); + UNINSTALL_PATCH(LocalAlloc); + UNINSTALL_PATCH(LocalReAlloc); + UNINSTALL_PATCH(LocalFree); + + hooked_ = false; + } + return true; +} + +bool MemoryHook::RegisterWatcher(MemoryObserver* watcher) { + DCHECK(global_hook_->watcher_ == NULL); + + if (!hooked_) + Hook(); + + DCHECK(global_hook_); + global_hook_->watcher_ = watcher; + return true; +} + +bool MemoryHook::UnregisterWatcher(MemoryObserver* watcher) { + DCHECK(hooked_); + DCHECK(global_hook_->watcher_ == watcher); + global_hook_->watcher_ = NULL; + + // For now, since there are no more watchers, unhook memory. + return Unhook(); +} + +bool MemoryHook::CreateHeap() { + // Create a heap for our own memory. + DCHECK(heap_ == NULL); + heap_ = HeapCreate(0, 0, 0); + DCHECK(heap_ != NULL); + return heap_ != NULL; +} + +bool MemoryHook::CloseHeap() { + DCHECK(heap_ != NULL); + HeapDestroy(heap_); + heap_ = NULL; + return true; +} + +void MemoryHook::OnTrack(HANDLE heap, int32 id, int32 size) { + // Don't notify about allocations to our internal heap. + if (heap == heap_) + return; + + if (watcher_) + watcher_->OnTrack(heap, id, size); +} + +void MemoryHook::OnUntrack(HANDLE heap, int32 id, int32 size) { + // Don't notify about allocations to our internal heap. + if (heap == heap_) + return; + + if (watcher_) + watcher_->OnUntrack(heap, id, size); +} diff --git a/tools/memory_watcher/memory_hook.h b/tools/memory_watcher/memory_hook.h new file mode 100644 index 0000000..22c3d89 --- /dev/null +++ b/tools/memory_watcher/memory_hook.h @@ -0,0 +1,163 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Static class for hooking Win32 API routines. For now, +// we only add one watcher at a time. +// +// TODO(mbelshe): Support multiple watchers. + +#ifndef MEMORY_WATCHER_MEMORY_HOOK_ +#define MEMORY_WATCHER_MEMORY_HOOK_ + +#include "base/logging.h" + +// When allocating memory for internal use with the MemoryHook, +// we must always use the MemoryHook's heap; otherwise, the memory +// gets tracked, and it becomes an infinite loop (allocation() calls +// MemoryHook() which calls allocation(), etc). +// +// PrivateHookAllocator is an STL-friendly Allocator so that STL lists, +// maps, etc can be used on the global MemoryHook's heap. +template <class T> +class PrivateHookAllocator { + public: + // These type definitions are needed for stl allocators. + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef T* pointer; + typedef const T* const_pointer; + typedef T& reference; + typedef const T& const_reference; + typedef T value_type; + + PrivateHookAllocator() {} + + // Allocate memory for STL. + pointer allocate(size_type n, const void * = 0) { + return reinterpret_cast<T*>(MemoryHook::Alloc(n * sizeof(T))); + } + + // Deallocate memory for STL. + void deallocate(void* p, size_type) { + if (p) + MemoryHook::Free(p); + } + + // Construct the object + void construct(pointer p, const T& val) { + new (reinterpret_cast<T*>(p))T(val); + } + + // Destruct an object + void destroy(pointer p) { p->~T(); } + + size_type max_size() const { return size_t(-1); } + + template <class U> + struct rebind { typedef PrivateHookAllocator<U> other; }; + + template <class U> + explicit PrivateHookAllocator(const PrivateHookAllocator<U>&) {} +}; + +// Classes which monitor memory from these hooks implement +// the MemoryObserver interface. +class MemoryObserver { + public: + // Track a pointer. Will capture the current StackTrace. + virtual void OnTrack(HANDLE heap, int32 id, int32 size) = 0; + + // Untrack a pointer, removing it from our list. + virtual void OnUntrack(HANDLE heap, int32 id, int32 size) = 0; +}; + +class MemoryHook : MemoryObserver { + public: + // Initialize the MemoryHook. Must be called before + // registering watchers. This can be called repeatedly, + // but is not thread safe. + static bool Initialize(); + + // Returns true is memory allocations and deallocations + // are being traced. + static bool hooked() { return hooked_ != NULL; } + + // Register a class to receive memory allocation & deallocation + // callbacks. If we haven't hooked memory yet, this call will + // force memory hooking to start. + static bool RegisterWatcher(MemoryObserver* watcher); + + // Register a class to stop receiving callbacks. If there are + // no more watchers, this call will unhook memory. + static bool UnregisterWatcher(MemoryObserver* watcher); + + // MemoryHook provides a private heap for allocating + // unwatched memory. + static void* Alloc(size_t size) { + DCHECK(global_hook_ && global_hook_->heap_); + return HeapAlloc(global_hook_->heap_, 0, size); + } + static void Free(void* ptr) { + DCHECK(global_hook_ && global_hook_->heap_); + HeapFree(global_hook_->heap_, 0, ptr); + } + + // Access the global hook. For internal use only from static "C" + // hooks. + static MemoryHook* hook() { return global_hook_; } + + // MemoryObserver interface. + virtual void OnTrack(HANDLE hHeap, int32 id, int32 size); + virtual void OnUntrack(HANDLE hHeap, int32 id, int32 size); + + private: + MemoryHook(); + ~MemoryHook(); + + // Enable memory tracing. When memory is 'hooked', + // MemoryWatchers which have registered will be called + // as memory is allocated and deallocated. + static bool Hook(); + + // Disables memory tracing. + static bool Unhook(); + + // Create our private heap + bool CreateHeap(); + + // Close our private heap. + bool CloseHeap(); + + MemoryObserver* watcher_; + HANDLE heap_; // An internal accounting heap. + static bool hooked_; + static MemoryHook* global_hook_; +}; + +#endif // MEMORY_WATCHER_MEMORY_HOOK_ diff --git a/tools/memory_watcher/memory_watcher.cc b/tools/memory_watcher/memory_watcher.cc new file mode 100644 index 0000000..cd1279d --- /dev/null +++ b/tools/memory_watcher/memory_watcher.cc @@ -0,0 +1,232 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <windows.h> +#include <tlhelp32.h> // for CreateToolhelp32Snapshot() +#include <map> + +#include "tools/memory_watcher/memory_watcher.h" +#include "base/file_util.h" +#include "base/lock.h" +#include "base/logging.h" +#include "base/stats_counters.h" +#include "base/string_util.h" +#include "tools/memory_watcher/call_stack.h" +#include "tools/memory_watcher/preamble_patcher.h" + +static StatsCounter mem_in_use(L"MemoryInUse.Bytes"); +static StatsCounter mem_in_use_blocks(L"MemoryInUse.Blocks"); +static StatsCounter mem_in_use_allocs(L"MemoryInUse.Allocs"); +static StatsCounter mem_in_use_frees(L"MemoryInUse.Frees"); + +// --------------------------------------------------------------------- + +MemoryWatcher::MemoryWatcher() + : file_(NULL), + hooked_(false), + in_track_(false), + block_map_size_(0) { + MemoryHook::Initialize(); + CallStack::Initialize(); + + block_map_ = new CallStackMap(); + stack_map_ = new CallStackIdMap(); + + // Register last - only after we're ready for notifications! + Hook(); +} + +MemoryWatcher::~MemoryWatcher() { + Unhook(); + + CloseLogFile(); + + // Pointers in the block_map are part of the MemoryHook heap. Be sure + // to delete the map before closing the heap. + delete block_map_; +} + +void MemoryWatcher::Hook() { + DCHECK(!hooked_); + MemoryHook::RegisterWatcher(this); + hooked_ = true; +} + +void MemoryWatcher::Unhook() { + if (hooked_) { + MemoryHook::UnregisterWatcher(this); + hooked_ = false; + } +} + +void MemoryWatcher::OpenLogFile() { + DCHECK(file_ == NULL); + file_name_ = "memwatcher"; + if (!log_name_.empty()) { + file_name_ += "."; + file_name_ += log_name_; + } + file_name_ += ".log"; + char buf[16]; + file_name_ += _itoa(GetCurrentProcessId(), buf, 10); + + std::string tmp_name(file_name_); + tmp_name += ".tmp"; + file_ = fopen(tmp_name.c_str(), "w+"); +} + +void MemoryWatcher::CloseLogFile() { + if (file_ != NULL) { + fclose(file_); + file_ = NULL; + std::wstring tmp_name = ASCIIToWide(file_name_); + tmp_name += L".tmp"; + file_util::Move(tmp_name, ASCIIToWide(file_name_)); + } +} + +void MemoryWatcher::OnTrack(HANDLE heap, int32 id, int32 size) { + // AllocationStack overrides new/delete to not allocate + // from the main heap. + AllocationStack* stack = new AllocationStack(size); + { + // Don't track zeroes. It's a waste of time. + if (size == 0) { + delete stack; + return; + } + + AutoLock lock(block_map_lock_); + + // Ideally, we'd like to verify that the block being added + // here is not already in our list of tracked blocks. However, + // the lookup in our hash table is expensive and slows us too + // much. Uncomment this line if you think you need it. + //DCHECK(block_map_->find(id) == block_map_->end()); + + (*block_map_)[id] = stack; + + CallStackIdMap::iterator it = stack_map_->find(stack->hash()); + if (it != stack_map_->end()) { + it->second.size += size; + it->second.count++; + } else { + StackTrack tracker; + tracker.count = 1; + tracker.size = size; + tracker.stack = stack; + (*stack_map_)[stack->hash()] = tracker; + } + + block_map_size_ += size; + } + + mem_in_use.Set(block_map_size_); + mem_in_use_blocks.Increment(); + mem_in_use_allocs.Increment(); +} + +void MemoryWatcher::OnUntrack(HANDLE heap, int32 id, int32 size) { + DCHECK(size >= 0); + + // Don't bother with these. + if (size == 0) + return; + + { + AutoLock lock(block_map_lock_); + + // First, find the block in our block_map. + CallStackMap::iterator it = block_map_->find(id); + if (it != block_map_->end()) { + AllocationStack* stack = it->second; + CallStackIdMap::iterator id_it = stack_map_->find(stack->hash()); + DCHECK(id_it != stack_map_->end()); + id_it->second.size -= size; + id_it->second.count--; + DCHECK(id_it->second.count >= 0); + + // If there are no more callstacks with this stack, then we + // have cleaned up all instances, and can safely delete the + // stack pointer in the stack_map. + bool safe_to_delete = true; + if (id_it->second.count == 0) + stack_map_->erase(id_it); + else if (id_it->second.stack == stack) + safe_to_delete = false; // we're still using the stack + + block_map_size_ -= size; + block_map_->erase(id); + if (safe_to_delete) + delete stack; + } else { + // Untracked item. This happens a fair amount, and it is + // normal. A lot of time elapses during process startup + // before the allocation routines are hooked. + } + } + + mem_in_use.Set(block_map_size_); + mem_in_use_blocks.Decrement(); + mem_in_use_frees.Increment(); +} + +void MemoryWatcher::SetLogName(char* log_name) { + if (!log_name) + return; + + log_name_ = log_name; +} + +void MemoryWatcher::DumpLeaks() { + // We can only dump the leaks once. We'll cleanup the hooks here. + DCHECK(hooked_); + Unhook(); + + AutoLock lock(block_map_lock_); + + OpenLogFile(); + + // Dump the stack map. + CallStackIdMap::iterator it = stack_map_->begin(); + while (it != stack_map_->end()) { + fwprintf(file_, L"%d bytes, %d items (0x%x)\n", + it->second.size, it->second.count, it->first); + CallStack* stack = it->second.stack; + std::string output; + stack->ToString(&output); + fprintf(file_, "%s", output.c_str()); + it++; + } + fprintf(file_, "Total Leaks: %d\n", block_map_->size()); + fprintf(file_, "Total Stacks: %d\n", stack_map_->size()); + fprintf(file_, "Total Bytes: %d\n", block_map_size_); + CloseLogFile(); +} + diff --git a/tools/memory_watcher/memory_watcher.h b/tools/memory_watcher/memory_watcher.h new file mode 100644 index 0000000..f06664e --- /dev/null +++ b/tools/memory_watcher/memory_watcher.h @@ -0,0 +1,115 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// MemoryWatcher. +// The MemoryWatcher is a library that can be linked into any +// win32 application. It will override the default memory allocators +// and track call stacks for any allocations that are made. It can +// then be used to see what memory is in use. + +#ifndef MEMORY_WATCHER_MEMORY_WATCHER_ +#define MEMORY_WATCHER_MEMORY_WATCHER_ + +#include <map> +#include <functional> +#include "base/lock.h" +#include "base/logging.h" +#include "tools/memory_watcher/memory_hook.h" + +class CallStack; +class AllocationStack; + +// The MemoryWatcher installs allocation hooks and monitors +// allocations and frees. +class MemoryWatcher : MemoryObserver { + public: + MemoryWatcher(); + virtual ~MemoryWatcher(); + + // Dump all tracked pointers still in use. + void DumpLeaks(); + + // MemoryObserver interface. + virtual void OnTrack(HANDLE heap, int32 id, int32 size); + virtual void OnUntrack(HANDLE heap, int32 id, int32 size); + + // Sets a name that appears in the generated file name. + void SetLogName(char* log_name); + + private: + // Opens the logfile which we create. + void OpenLogFile(); + + // Close the logfile. + void CloseLogFile(); + + // Hook the memory hooks. + void Hook(); + + // Unhooks our memory hooks. + void Unhook(); + + // This is for logging. + FILE* file_; + + struct StackTrack { + CallStack* stack; + int count; + int size; + }; + + bool hooked_; // True when this class has the memory_hooks hooked. + + bool in_track_; + Lock block_map_lock_; + typedef std::map<int32, AllocationStack*, std::less<int32>, + PrivateHookAllocator<int32>> CallStackMap; + typedef std::map<int32, StackTrack, std::less<int32>, + PrivateHookAllocator<int32>> CallStackIdMap; + // The block_map provides quick lookups based on the allocation + // pointer. This is important for having fast round trips through + // malloc/free. + CallStackMap *block_map_; + // The stack_map keeps track of the known CallStacks based on the + // hash of the CallStack. This is so that we can quickly aggregate + // like-CallStacks together. + CallStackIdMap *stack_map_; + int32 block_map_size_; + + // The file name for that log. + std::string file_name_; + + // An optional name that appears in the log file name (used to differentiate + // logs). + std::string log_name_; +}; + + + +#endif // MEMORY_WATCHER_ diff --git a/tools/memory_watcher/memory_watcher.vcproj b/tools/memory_watcher/memory_watcher.vcproj new file mode 100644 index 0000000..dd740c9 --- /dev/null +++ b/tools/memory_watcher/memory_watcher.vcproj @@ -0,0 +1,203 @@ +<?xml version="1.0" encoding="Windows-1252"?> +<VisualStudioProject + ProjectType="Visual C++" + Version="8.00" + Name="memory_watcher" + ProjectGUID="{3BD81303-4E14-4559-AA69-B30C3BAB08DD}" + RootNamespace="memory_watcher" + Keyword="Win32Proj" + > + <Platforms> + <Platform + Name="Win32" + /> + </Platforms> + <ToolFiles> + </ToolFiles> + <Configurations> + <Configuration + Name="Debug|Win32" + ConfigurationType="2" + InheritedPropertySheets="$(SolutionDir)..\build\common.vsprops;$(SolutionDir)..\build\debug.vsprops;$(SolutionDir)..\build\external_code.vsprops" + > + <Tool + Name="VCPreBuildEventTool" + /> + <Tool + Name="VCCustomBuildTool" + /> + <Tool + Name="VCXMLDataGeneratorTool" + /> + <Tool + Name="VCWebServiceProxyGeneratorTool" + /> + <Tool + Name="VCMIDLTool" + /> + <Tool + Name="VCCLCompilerTool" + PreprocessorDefinitions="BUILD_MEMORY_WATCHER" + /> + <Tool + Name="VCManagedResourceCompilerTool" + /> + <Tool + Name="VCResourceCompilerTool" + /> + <Tool + Name="VCPreLinkEventTool" + /> + <Tool + Name="VCLinkerTool" + /> + <Tool + Name="VCALinkTool" + /> + <Tool + Name="VCManifestTool" + /> + <Tool + Name="VCXDCMakeTool" + /> + <Tool + Name="VCBscMakeTool" + /> + <Tool + Name="VCFxCopTool" + /> + <Tool + Name="VCAppVerifierTool" + /> + <Tool + Name="VCWebDeploymentTool" + /> + <Tool + Name="VCPostBuildEventTool" + /> + </Configuration> + <Configuration + Name="Release|Win32" + ConfigurationType="2" + InheritedPropertySheets="$(SolutionDir)..\build\common.vsprops;$(SolutionDir)..\build\release.vsprops;$(SolutionDir)..\build\external_code.vsprops" + > + <Tool + Name="VCPreBuildEventTool" + /> + <Tool + Name="VCCustomBuildTool" + /> + <Tool + Name="VCXMLDataGeneratorTool" + /> + <Tool + Name="VCWebServiceProxyGeneratorTool" + /> + <Tool + Name="VCMIDLTool" + /> + <Tool + Name="VCCLCompilerTool" + PreprocessorDefinitions="BUILD_MEMORY_WATCHER" + /> + <Tool + Name="VCManagedResourceCompilerTool" + /> + <Tool + Name="VCResourceCompilerTool" + /> + <Tool + Name="VCPreLinkEventTool" + /> + <Tool + Name="VCLinkerTool" + /> + <Tool + Name="VCALinkTool" + /> + <Tool + Name="VCManifestTool" + /> + <Tool + Name="VCXDCMakeTool" + /> + <Tool + Name="VCBscMakeTool" + /> + <Tool + Name="VCFxCopTool" + /> + <Tool + Name="VCAppVerifierTool" + /> + <Tool + Name="VCWebDeploymentTool" + /> + <Tool + Name="VCPostBuildEventTool" + /> + </Configuration> + </Configurations> + <References> + </References> + <Files> + <File + RelativePath=".\call_stack.cc" + > + </File> + <File + RelativePath=".\call_stack.h" + > + </File> + <File + RelativePath=".\dllmain.cc" + > + </File> + <File + RelativePath=".\hotkey.h" + > + </File> + <File + RelativePath=".\ia32_modrm_map.cc" + > + </File> + <File + RelativePath=".\ia32_opcode_map.cc" + > + </File> + <File + RelativePath=".\memory_hook.cc" + > + </File> + <File + RelativePath=".\memory_hook.h" + > + </File> + <File + RelativePath=".\memory_watcher.cc" + > + </File> + <File + RelativePath=".\memory_watcher.h" + > + </File> + <File + RelativePath=".\mini_disassembler.cc" + > + </File> + <File + RelativePath=".\preamble_patcher.cc" + > + </File> + <File + RelativePath=".\preamble_patcher.h" + > + </File> + <File + RelativePath=".\preamble_patcher_with_stub.cc" + > + </File> + </Files> + <Globals> + </Globals> +</VisualStudioProject> diff --git a/tools/memory_watcher/mini_disassembler.cc b/tools/memory_watcher/mini_disassembler.cc new file mode 100644 index 0000000..6b1dec8 --- /dev/null +++ b/tools/memory_watcher/mini_disassembler.cc @@ -0,0 +1,418 @@ +/* Copyright (c) 2007, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * + * Implementation of MiniDisassembler. + */ + +#include "mini_disassembler.h" + +namespace sidestep { + +MiniDisassembler::MiniDisassembler(bool operand_default_is_32_bits, + bool address_default_is_32_bits) + : operand_default_is_32_bits_(operand_default_is_32_bits), + address_default_is_32_bits_(address_default_is_32_bits) { + Initialize(); +} + +MiniDisassembler::MiniDisassembler() + : operand_default_is_32_bits_(true), + address_default_is_32_bits_(true) { + Initialize(); +} + +InstructionType MiniDisassembler::Disassemble( + unsigned char* start_byte, + unsigned int& instruction_bytes) { + // Clean up any state from previous invocations. + Initialize(); + + // Start by processing any prefixes. + unsigned char* current_byte = start_byte; + unsigned int size = 0; + InstructionType instruction_type = ProcessPrefixes(current_byte, size); + + if (IT_UNKNOWN == instruction_type) + return instruction_type; + + current_byte += size; + size = 0; + + // Invariant: We have stripped all prefixes, and the operand_is_32_bits_ + // and address_is_32_bits_ flags are correctly set. + + instruction_type = ProcessOpcode(current_byte, 0, size); + + // Check for error processing instruction + if ((IT_UNKNOWN == instruction_type_) || (IT_UNUSED == instruction_type_)) { + return IT_UNKNOWN; + } + + current_byte += size; + + // Invariant: operand_bytes_ indicates the total size of operands + // specified by the opcode and/or ModR/M byte and/or SIB byte. + // pCurrentByte points to the first byte after the ModR/M byte, or after + // the SIB byte if it is present (i.e. the first byte of any operands + // encoded in the instruction). + + // We get the total length of any prefixes, the opcode, and the ModR/M and + // SIB bytes if present, by taking the difference of the original starting + // address and the current byte (which points to the first byte of the + // operands if present, or to the first byte of the next instruction if + // they are not). Adding the count of bytes in the operands encoded in + // the instruction gives us the full length of the instruction in bytes. + instruction_bytes += operand_bytes_ + (current_byte - start_byte); + + // Return the instruction type, which was set by ProcessOpcode(). + return instruction_type_; +} + +void MiniDisassembler::Initialize() { + operand_is_32_bits_ = operand_default_is_32_bits_; + address_is_32_bits_ = address_default_is_32_bits_; + operand_bytes_ = 0; + have_modrm_ = false; + should_decode_modrm_ = false; + instruction_type_ = IT_UNKNOWN; + got_f2_prefix_ = false; + got_f3_prefix_ = false; + got_66_prefix_ = false; +} + +InstructionType MiniDisassembler::ProcessPrefixes(unsigned char* start_byte, + unsigned int& size) { + InstructionType instruction_type = IT_GENERIC; + const Opcode& opcode = s_ia32_opcode_map_[0].table_[*start_byte]; + + switch (opcode.type_) { + case IT_PREFIX_ADDRESS: + address_is_32_bits_ = !address_default_is_32_bits_; + goto nochangeoperand; + case IT_PREFIX_OPERAND: + operand_is_32_bits_ = !operand_default_is_32_bits_; + nochangeoperand: + case IT_PREFIX: + + if (0xF2 == (*start_byte)) + got_f2_prefix_ = true; + else if (0xF3 == (*start_byte)) + got_f3_prefix_ = true; + else if (0x66 == (*start_byte)) + got_66_prefix_ = true; + + instruction_type = opcode.type_; + size ++; + // we got a prefix, so add one and check next byte + ProcessPrefixes(start_byte + 1, size); + default: + break; // not a prefix byte + } + + return instruction_type; +} + +InstructionType MiniDisassembler::ProcessOpcode(unsigned char* start_byte, + unsigned int table_index, + unsigned int& size) { + const OpcodeTable& table = s_ia32_opcode_map_[table_index]; // Get our table + unsigned char current_byte = (*start_byte) >> table.shift_; + current_byte = current_byte & table.mask_; // Mask out the bits we will use + + // Check whether the byte we have is inside the table we have. + if (current_byte < table.min_lim_ || current_byte > table.max_lim_) { + instruction_type_ = IT_UNKNOWN; + return instruction_type_; + } + + const Opcode& opcode = table.table_[current_byte]; + if (IT_UNUSED == opcode.type_) { + // This instruction is not used by the IA-32 ISA, so we indicate + // this to the user. Probably means that we were pointed to + // a byte in memory that was not the start of an instruction. + instruction_type_ = IT_UNUSED; + return instruction_type_; + } else if (IT_REFERENCE == opcode.type_) { + // We are looking at an opcode that has more bytes (or is continued + // in the ModR/M byte). Recursively find the opcode definition in + // the table for the opcode's next byte. + size++; + ProcessOpcode(start_byte + 1, opcode.table_index_, size); + return instruction_type_; + } + + const SpecificOpcode* specific_opcode = (SpecificOpcode*)&opcode; + if (opcode.is_prefix_dependent_) { + if (got_f2_prefix_ && opcode.opcode_if_f2_prefix_.mnemonic_ != 0) { + specific_opcode = &opcode.opcode_if_f2_prefix_; + } else if (got_f3_prefix_ && opcode.opcode_if_f3_prefix_.mnemonic_ != 0) { + specific_opcode = &opcode.opcode_if_f3_prefix_; + } else if (got_66_prefix_ && opcode.opcode_if_66_prefix_.mnemonic_ != 0) { + specific_opcode = &opcode.opcode_if_66_prefix_; + } + } + + // Inv: The opcode type is known. + instruction_type_ = specific_opcode->type_; + + // Let's process the operand types to see if we have any immediate + // operands, and/or a ModR/M byte. + + ProcessOperand(specific_opcode->flag_dest_); + ProcessOperand(specific_opcode->flag_source_); + ProcessOperand(specific_opcode->flag_aux_); + + // Inv: We have processed the opcode and incremented operand_bytes_ + // by the number of bytes of any operands specified by the opcode + // that are stored in the instruction (not registers etc.). Now + // we need to return the total number of bytes for the opcode and + // for the ModR/M or SIB bytes if they are present. + + if (table.mask_ != 0xff) { + if (have_modrm_) { + // we're looking at a ModR/M byte so we're not going to + // count that into the opcode size + ProcessModrm(start_byte, size); + return IT_GENERIC; + } else { + // need to count the ModR/M byte even if it's just being + // used for opcode extension + size++; + return IT_GENERIC; + } + } else { + if (have_modrm_) { + // The ModR/M byte is the next byte. + size++; + ProcessModrm(start_byte + 1, size); + return IT_GENERIC; + } else { + size++; + return IT_GENERIC; + } + } +} + +bool MiniDisassembler::ProcessOperand(int flag_operand) { + bool succeeded = true; + if (AM_NOT_USED == flag_operand) + return succeeded; + + // Decide what to do based on the addressing mode. + switch (flag_operand & AM_MASK) { + // No ModR/M byte indicated by these addressing modes, and no + // additional (e.g. immediate) parameters. + case AM_A: // Direct address + case AM_F: // EFLAGS register + case AM_X: // Memory addressed by the DS:SI register pair + case AM_Y: // Memory addressed by the ES:DI register pair + case AM_IMPLICIT: // Parameter is implicit, occupies no space in + // instruction + break; + + // There is a ModR/M byte but it does not necessarily need + // to be decoded. + case AM_C: // reg field of ModR/M selects a control register + case AM_D: // reg field of ModR/M selects a debug register + case AM_G: // reg field of ModR/M selects a general register + case AM_P: // reg field of ModR/M selects an MMX register + case AM_R: // mod field of ModR/M may refer only to a general register + case AM_S: // reg field of ModR/M selects a segment register + case AM_T: // reg field of ModR/M selects a test register + case AM_V: // reg field of ModR/M selects a 128-bit XMM register + have_modrm_ = true; + break; + + // In these addressing modes, there is a ModR/M byte and it needs to be + // decoded. No other (e.g. immediate) params than indicated in ModR/M. + case AM_E: // Operand is either a general-purpose register or memory, + // specified by ModR/M byte + case AM_M: // ModR/M byte will refer only to memory + case AM_Q: // Operand is either an MMX register or memory (complex + // evaluation), specified by ModR/M byte + case AM_W: // Operand is either a 128-bit XMM register or memory (complex + // eval), specified by ModR/M byte + have_modrm_ = true; + should_decode_modrm_ = true; + break; + + // These addressing modes specify an immediate or an offset value + // directly, so we need to look at the operand type to see how many + // bytes. + case AM_I: // Immediate data. + case AM_J: // Jump to offset. + case AM_O: // Operand is at offset. + switch (flag_operand & OT_MASK) { + case OT_B: // Byte regardless of operand-size attribute. + operand_bytes_ += OS_BYTE; + break; + case OT_C: // Byte or word, depending on operand-size attribute. + if (operand_is_32_bits_) + operand_bytes_ += OS_WORD; + else + operand_bytes_ += OS_BYTE; + break; + case OT_D: // Doubleword, regardless of operand-size attribute. + operand_bytes_ += OS_DOUBLE_WORD; + break; + case OT_DQ: // Double-quadword, regardless of operand-size attribute. + operand_bytes_ += OS_DOUBLE_QUAD_WORD; + break; + case OT_P: // 32-bit or 48-bit pointer, depending on operand-size + // attribute. + if (operand_is_32_bits_) + operand_bytes_ += OS_48_BIT_POINTER; + else + operand_bytes_ += OS_32_BIT_POINTER; + break; + case OT_PS: // 128-bit packed single-precision floating-point data. + operand_bytes_ += OS_128_BIT_PACKED_SINGLE_PRECISION_FLOATING; + break; + case OT_Q: // Quadword, regardless of operand-size attribute. + operand_bytes_ += OS_QUAD_WORD; + break; + case OT_S: // 6-byte pseudo-descriptor. + operand_bytes_ += OS_PSEUDO_DESCRIPTOR; + break; + case OT_SD: // Scalar Double-Precision Floating-Point Value + case OT_PD: // Unaligned packed double-precision floating point value + operand_bytes_ += OS_DOUBLE_PRECISION_FLOATING; + break; + case OT_SS: + // Scalar element of a 128-bit packed single-precision + // floating data. + // We simply return enItUnknown since we don't have to support + // floating point + succeeded = false; + break; + case OT_V: // Word or doubleword, depending on operand-size attribute. + if (operand_is_32_bits_) + operand_bytes_ += OS_DOUBLE_WORD; + else + operand_bytes_ += OS_WORD; + break; + case OT_W: // Word, regardless of operand-size attribute. + operand_bytes_ += OS_WORD; + break; + + // Can safely ignore these. + case OT_A: // Two one-word operands in memory or two double-word + // operands in memory + case OT_PI: // Quadword MMX technology register (e.g. mm0) + case OT_SI: // Doubleword integer register (e.g., eax) + break; + + default: + break; + } + break; + + default: + break; + } + + return succeeded; +} + +bool MiniDisassembler::ProcessModrm(unsigned char* start_byte, + unsigned int& size) { + // If we don't need to decode, we just return the size of the ModR/M + // byte (there is never a SIB byte in this case). + if (!should_decode_modrm_) { + size++; + return true; + } + + // We never care about the reg field, only the combination of the mod + // and r/m fields, so let's start by packing those fields together into + // 5 bits. + unsigned char modrm = (*start_byte); + unsigned char mod = modrm & 0xC0; // mask out top two bits to get mod field + modrm = modrm & 0x07; // mask out bottom 3 bits to get r/m field + mod = mod >> 3; // shift the mod field to the right place + modrm = mod | modrm; // combine the r/m and mod fields as discussed + mod = mod >> 3; // shift the mod field to bits 2..0 + + // Invariant: modrm contains the mod field in bits 4..3 and the r/m field + // in bits 2..0, and mod contains the mod field in bits 2..0 + + const ModrmEntry* modrm_entry = 0; + if (address_is_32_bits_) + modrm_entry = &s_ia32_modrm_map_[modrm]; + else + modrm_entry = &s_ia16_modrm_map_[modrm]; + + // Invariant: modrm_entry points to information that we need to decode + // the ModR/M byte. + + // Add to the count of operand bytes, if the ModR/M byte indicates + // that some operands are encoded in the instruction. + if (modrm_entry->is_encoded_in_instruction_) + operand_bytes_ += modrm_entry->operand_size_; + + // Process the SIB byte if necessary, and return the count + // of ModR/M and SIB bytes. + if (modrm_entry->use_sib_byte_) { + size++; + return ProcessSib(start_byte + 1, mod, size); + } else { + size++; + return true; + } +} + +bool MiniDisassembler::ProcessSib(unsigned char* start_byte, + unsigned char mod, + unsigned int& size) { + // get the mod field from the 2..0 bits of the SIB byte + unsigned char sib_base = (*start_byte) & 0x07; + if (0x05 == sib_base) { + switch (mod) { + case 0x00: // mod == 00 + case 0x02: // mod == 10 + operand_bytes_ += OS_DOUBLE_WORD; + break; + case 0x01: // mod == 01 + operand_bytes_ += OS_BYTE; + break; + case 0x03: // mod == 11 + // According to the IA-32 docs, there does not seem to be a disp + // value for this value of mod + default: + break; + } + } + + size++; + return true; +} + +}; // namespace sidestep diff --git a/tools/memory_watcher/mini_disassembler.h b/tools/memory_watcher/mini_disassembler.h new file mode 100644 index 0000000..8af2c42 --- /dev/null +++ b/tools/memory_watcher/mini_disassembler.h @@ -0,0 +1,189 @@ +/* Copyright (c) 2007, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * + * Definition of MiniDisassembler. + */ + +#ifndef GOOGLE_PERFTOOLS_MINI_DISASSEMBLER_H__ +#define GOOGLE_PERFTOOLS_MINI_DISASSEMBLER_H__ + +#include <windows.h> +#include "mini_disassembler_types.h" + +// compatibility shim +#include "base/logging.h" +#define ASSERT(cond, msg) DCHECK(cond) +#define ASSERT1(cond) DCHECK(cond) + +namespace sidestep { + +// This small disassembler is very limited +// in its functionality, and in fact does only the bare minimum required by the +// preamble patching utility. It may be useful for other purposes, however. +// +// The limitations include at least the following: +// -# No support for coprocessor opcodes, MMX, etc. +// -# No machine-readable identification of opcodes or decoding of +// assembly parameters. The name of the opcode (as a string) is given, +// however, to aid debugging. +// +// You may ask what this little disassembler actually does, then? The answer is +// that it does the following, which is exactly what the patching utility needs: +// -# Indicates if opcode is a jump (any kind) or a return (any kind) +// because this is important for the patching utility to determine if +// a function is too short or there are jumps too early in it for it +// to be preamble patched. +// -# The opcode length is always calculated, so that the patching utility +// can figure out where the next instruction starts, and whether it +// already has enough instructions to replace with the absolute jump +// to the patching code. +// +// The usage is quite simple; just create a MiniDisassembler and use its +// Disassemble() method. +// +// If you would like to extend this disassembler, please refer to the +// IA-32 Intel Architecture Software Developer’s Manual Volume 2: +// Instruction Set Reference for information about operand decoding +// etc. +class MiniDisassembler { + public: + + // Creates a new instance and sets defaults. + // + // @param operand_default_32_bits If true, the default operand size is + // set to 32 bits, which is the default under Win32. Otherwise it is 16 bits. + // @param address_default_32_bits If true, the default address size is + // set to 32 bits, which is the default under Win32. Otherwise it is 16 bits. + MiniDisassembler(bool operand_default_32_bits, + bool address_default_32_bits); + + // Equivalent to MiniDisassembler(true, true); + MiniDisassembler(); + + // Attempts to disassemble a single instruction starting from the + // address in memory it is pointed to. + // + // @param start Address where disassembly should start. + // @param instruction_bytes Variable that will be <b>incremented</b> by + // the length in bytes of the instruction. + // @return enItJump, enItReturn or enItGeneric on success. enItUnknown + // if unable to disassemble, enItUnused if this seems to be an unused + // opcode. In the last two (error) cases, cbInstruction will be set + // to 0xffffffff. + // + // @post This instance of the disassembler is ready to be used again, + // with unchanged defaults from creation time. + InstructionType Disassemble(unsigned char* start, unsigned int& instruction_bytes); + + private: + + // Makes the disassembler ready for reuse. + void Initialize(); + + // Sets the flags for address and operand sizes. + // @return Number of prefix bytes. + InstructionType ProcessPrefixes(unsigned char* start, unsigned int& size); + + // Sets the flag for whether we have ModR/M, and increments + // operand_bytes_ if any are specifies by the opcode directly. + // @return Number of opcode bytes. + InstructionType ProcessOpcode(unsigned char * start, + unsigned int table, + unsigned int& size); + + // Checks the type of the supplied operand. Increments + // operand_bytes_ if it directly indicates an immediate etc. + // operand. Asserts have_modrm_ if the operand specifies + // a ModR/M byte. + bool ProcessOperand(int flag_operand); + + // Increments operand_bytes_ by size specified by ModR/M and + // by SIB if present. + // @return 0 in case of error, 1 if there is just a ModR/M byte, + // 2 if there is a ModR/M byte and a SIB byte. + bool ProcessModrm(unsigned char* start, unsigned int& size); + + // Processes the SIB byte that it is pointed to. + // @param start Pointer to the SIB byte. + // @param mod The mod field from the ModR/M byte. + // @return 1 to indicate success (indicates 1 SIB byte) + bool ProcessSib(unsigned char* start, unsigned char mod, unsigned int& size); + + // The instruction type we have decoded from the opcode. + InstructionType instruction_type_; + + // Counts the number of bytes that is occupied by operands in + // the current instruction (note: we don't care about how large + // operands stored in registers etc. are). + unsigned int operand_bytes_; + + // True iff there is a ModR/M byte in this instruction. + bool have_modrm_; + + // True iff we need to decode the ModR/M byte (sometimes it just + // points to a register, we can tell by the addressing mode). + bool should_decode_modrm_; + + // Current operand size is 32 bits if true, 16 bits if false. + bool operand_is_32_bits_; + + // Default operand size is 32 bits if true, 16 bits if false. + bool operand_default_is_32_bits_; + + // Current address size is 32 bits if true, 16 bits if false. + bool address_is_32_bits_; + + // Default address size is 32 bits if true, 16 bits if false. + bool address_default_is_32_bits_; + + // Huge big opcode table based on the IA-32 manual, defined + // in Ia32OpcodeMap.cc + static const OpcodeTable s_ia32_opcode_map_[]; + + // Somewhat smaller table to help with decoding ModR/M bytes + // when 16-bit addressing mode is being used. Defined in + // Ia32ModrmMap.cc + static const ModrmEntry s_ia16_modrm_map_[]; + + // Somewhat smaller table to help with decoding ModR/M bytes + // when 32-bit addressing mode is being used. Defined in + // Ia32ModrmMap.cc + static const ModrmEntry s_ia32_modrm_map_[]; + + // Indicators of whether we got certain prefixes that certain + // silly Intel instructions depend on in nonstandard ways for + // their behaviors. + bool got_f2_prefix_, got_f3_prefix_, got_66_prefix_; +}; + +}; // namespace sidestep + +#endif // GOOGLE_PERFTOOLS_MINI_DISASSEMBLER_H__ diff --git a/tools/memory_watcher/mini_disassembler_types.h b/tools/memory_watcher/mini_disassembler_types.h new file mode 100644 index 0000000..3abc85d --- /dev/null +++ b/tools/memory_watcher/mini_disassembler_types.h @@ -0,0 +1,225 @@ +/* Copyright (c) 2007, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * + * Several simple types used by the disassembler and some of the patching + * mechanisms. + */ + +#ifndef GOOGLE_PERFTOOLS_MINI_DISASSEMBLER_TYPES_H__ +#define GOOGLE_PERFTOOLS_MINI_DISASSEMBLER_TYPES_H__ + +namespace sidestep { + +// Categories of instructions that we care about +enum InstructionType { + // This opcode is not used + IT_UNUSED, + // This disassembler does not recognize this opcode (error) + IT_UNKNOWN, + // This is not an instruction but a reference to another table + IT_REFERENCE, + // This byte is a prefix byte that we can ignore + IT_PREFIX, + // This is a prefix byte that switches to the nondefault address size + IT_PREFIX_ADDRESS, + // This is a prefix byte that switches to the nondefault operand size + IT_PREFIX_OPERAND, + // A jump or call instruction + IT_JUMP, + // A return instruction + IT_RETURN, + // Any other type of instruction (in this case we don't care what it is) + IT_GENERIC, +}; + +// Lists IA-32 operand sizes in multiples of 8 bits +enum OperandSize { + OS_ZERO = 0, + OS_BYTE = 1, + OS_WORD = 2, + OS_DOUBLE_WORD = 4, + OS_QUAD_WORD = 8, + OS_DOUBLE_QUAD_WORD = 16, + OS_32_BIT_POINTER = 32/8, + OS_48_BIT_POINTER = 48/8, + OS_SINGLE_PRECISION_FLOATING = 32/8, + OS_DOUBLE_PRECISION_FLOATING = 64/8, + OS_DOUBLE_EXTENDED_PRECISION_FLOATING = 80/8, + OS_128_BIT_PACKED_SINGLE_PRECISION_FLOATING = 128/8, + OS_PSEUDO_DESCRIPTOR = 6 +}; + +// Operand addressing methods from the IA-32 manual. The enAmMask value +// is a mask for the rest. The other enumeration values are named for the +// names given to the addressing methods in the manual, e.g. enAm_D is for +// the D addressing method. +// +// The reason we use a full 4 bytes and a mask, is that we need to combine +// these flags with the enOperandType to store the details +// on the operand in a single integer. +enum AddressingMethod { + AM_NOT_USED = 0, // This operand is not used for this instruction + AM_MASK = 0x00FF0000, // Mask for the rest of the values in this enumeration + AM_A = 0x00010000, // A addressing type + AM_C = 0x00020000, // C addressing type + AM_D = 0x00030000, // D addressing type + AM_E = 0x00040000, // E addressing type + AM_F = 0x00050000, // F addressing type + AM_G = 0x00060000, // G addressing type + AM_I = 0x00070000, // I addressing type + AM_J = 0x00080000, // J addressing type + AM_M = 0x00090000, // M addressing type + AM_O = 0x000A0000, // O addressing type + AM_P = 0x000B0000, // P addressing type + AM_Q = 0x000C0000, // Q addressing type + AM_R = 0x000D0000, // R addressing type + AM_S = 0x000E0000, // S addressing type + AM_T = 0x000F0000, // T addressing type + AM_V = 0x00100000, // V addressing type + AM_W = 0x00110000, // W addressing type + AM_X = 0x00120000, // X addressing type + AM_Y = 0x00130000, // Y addressing type + AM_REGISTER = 0x00140000, // Specific register is always used as this op + AM_IMPLICIT = 0x00150000, // An implicit, fixed value is used +}; + +// Operand types from the IA-32 manual. The enOtMask value is +// a mask for the rest. The rest of the values are named for the +// names given to these operand types in the manual, e.g. enOt_ps +// is for the ps operand type in the manual. +// +// The reason we use a full 4 bytes and a mask, is that we need +// to combine these flags with the enAddressingMethod to store the details +// on the operand in a single integer. +enum OperandType { + OT_MASK = 0xFF000000, + OT_A = 0x01000000, + OT_B = 0x02000000, + OT_C = 0x03000000, + OT_D = 0x04000000, + OT_DQ = 0x05000000, + OT_P = 0x06000000, + OT_PI = 0x07000000, + OT_PS = 0x08000000, // actually unsupported for (we don't know its size) + OT_Q = 0x09000000, + OT_S = 0x0A000000, + OT_SS = 0x0B000000, + OT_SI = 0x0C000000, + OT_V = 0x0D000000, + OT_W = 0x0E000000, + OT_SD = 0x0F000000, // scalar double-precision floating-point value + OT_PD = 0x10000000, // double-precision floating point + // dummy "operand type" for address mode M - which doesn't specify + // operand type + OT_ADDRESS_MODE_M = 0x80000000 +}; + +// Everything that's in an Opcode (see below) except the three +// alternative opcode structs for different prefixes. +struct SpecificOpcode { + // Index to continuation table, or 0 if this is the last + // byte in the opcode. + int table_index_; + + // The opcode type + InstructionType type_; + + // Description of the type of the dest, src and aux operands, + // put together from an enOperandType flag and an enAddressingMethod + // flag. + int flag_dest_; + int flag_source_; + int flag_aux_; + + // We indicate the mnemonic for debugging purposes + const char* mnemonic_; +}; + +// The information we keep in our tables about each of the different +// valid instructions recognized by the IA-32 architecture. +struct Opcode { + // Index to continuation table, or 0 if this is the last + // byte in the opcode. + int table_index_; + + // The opcode type + InstructionType type_; + + // Description of the type of the dest, src and aux operands, + // put together from an enOperandType flag and an enAddressingMethod + // flag. + int flag_dest_; + int flag_source_; + int flag_aux_; + + // We indicate the mnemonic for debugging purposes + const char* mnemonic_; + + // Alternative opcode info if certain prefixes are specified. + // In most cases, all of these are zeroed-out. Only used if + // bPrefixDependent is true. + bool is_prefix_dependent_; + SpecificOpcode opcode_if_f2_prefix_; + SpecificOpcode opcode_if_f3_prefix_; + SpecificOpcode opcode_if_66_prefix_; +}; + +// Information about each table entry. +struct OpcodeTable { + // Table of instruction entries + const Opcode* table_; + // How many bytes left to shift ModR/M byte <b>before</b> applying mask + unsigned char shift_; + // Mask to apply to byte being looked at before comparing to table + unsigned char mask_; + // Minimum/maximum indexes in table. + unsigned char min_lim_; + unsigned char max_lim_; +}; + +// Information about each entry in table used to decode ModR/M byte. +struct ModrmEntry { + // Is the operand encoded as bytes in the instruction (rather than + // if it's e.g. a register in which case it's just encoded in the + // ModR/M byte) + bool is_encoded_in_instruction_; + + // Is there a SIB byte? In this case we always need to decode it. + bool use_sib_byte_; + + // What is the size of the operand (only important if it's encoded + // in the instruction)? + OperandSize operand_size_; +}; + +}; // namespace sidestep + +#endif // GOOGLE_PERFTOOLS_MINI_DISASSEMBLER_TYPES_H__ diff --git a/tools/memory_watcher/preamble_patcher.cc b/tools/memory_watcher/preamble_patcher.cc new file mode 100644 index 0000000..2beb555 --- /dev/null +++ b/tools/memory_watcher/preamble_patcher.cc @@ -0,0 +1,259 @@ +/* Copyright (c) 2007, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * + * Implementation of PreamblePatcher + */ + +#include "preamble_patcher.h" +#include "memory_hook.h" +#include "mini_disassembler.h" + +// compatibility shims +#include "base/logging.h" + +// Definitions of assembly statements we need +#define ASM_JMP32REL 0xE9 +#define ASM_INT3 0xCC + +namespace sidestep { + +SideStepError PreamblePatcher::RawPatchWithStubAndProtections( + void* target_function, void *replacement_function, + unsigned char* preamble_stub, unsigned long stub_size, + unsigned long* bytes_needed) { + // We need to be able to write to a process-local copy of the first + // MAX_PREAMBLE_STUB_SIZE bytes of target_function. We may be giving execute + // privilege to something that doesn't have it, but that's the price to pay + // for tools. + DWORD old_target_function_protect = 0; + BOOL succeeded = ::VirtualProtect(reinterpret_cast<void*>(target_function), + MAX_PREAMBLE_STUB_SIZE, + PAGE_EXECUTE_READWRITE, + &old_target_function_protect); + if (!succeeded) { + ASSERT(false, "Failed to make page containing target function " + "copy-on-write."); + return SIDESTEP_ACCESS_DENIED; + } + + SideStepError error_code = RawPatchWithStub(target_function, + replacement_function, + preamble_stub, + stub_size, + bytes_needed); + if (SIDESTEP_SUCCESS != error_code) { + ASSERT1(false); + return error_code; + } + + // Restore the protection of the first MAX_PREAMBLE_STUB_SIZE bytes of + // pTargetFunction to what they were before we started goofing around. + succeeded = ::VirtualProtect(reinterpret_cast<void*>(target_function), + MAX_PREAMBLE_STUB_SIZE, + old_target_function_protect, + &old_target_function_protect); + if (!succeeded) { + ASSERT(false, "Failed to restore protection to target function."); + // We must not return an error here because the function has actually + // been patched, and returning an error would likely cause our client + // code not to unpatch it. So we just keep going. + } + + // Flush the instruction cache to make sure the processor doesn't execute the + // old version of the instructions (before our patch). + // + // FlushInstructionCache is actually a no-op at least on single-processor + // XP machines. I'm not sure why this is so, but it is, yet I want to keep the + // call to the API here for correctness in case there is a difference in + // some variants of Windows/hardware. + succeeded = ::FlushInstructionCache(::GetCurrentProcess(), + target_function, + MAX_PREAMBLE_STUB_SIZE); + if (!succeeded) { + ASSERT(false, "Failed to flush instruction cache."); + // We must not return an error here because the function has actually + // been patched, and returning an error would likely cause our client + // code not to unpatch it. So we just keep going. + } + + return SIDESTEP_SUCCESS; +} + +SideStepError PreamblePatcher::RawPatch(void* target_function, + void* replacement_function, + void** original_function_stub) { + if (!target_function || !replacement_function || !original_function_stub || + (*original_function_stub) || target_function == replacement_function) { + ASSERT(false, "Preconditions not met"); + return SIDESTEP_INVALID_PARAMETER; + } + + // @see MAX_PREAMBLE_STUB_SIZE for an explanation of how we arrives at + // this size + unsigned char* preamble_stub = + reinterpret_cast<unsigned char*>( + MemoryHook::Alloc(sizeof(unsigned char) * MAX_PREAMBLE_STUB_SIZE)); + if (!preamble_stub) { + ASSERT(false, "Unable to allocate preamble-stub."); + return SIDESTEP_INSUFFICIENT_BUFFER; + } + + // Change the protection of the newly allocated preamble stub to + // PAGE_EXECUTE_READWRITE. This is required to work with DEP (Data + // Execution Prevention) which will cause an exception if code is executed + // from a page on which you do not have read access. + DWORD old_stub_protect = 0; + BOOL succeeded = VirtualProtect(preamble_stub, MAX_PREAMBLE_STUB_SIZE, + PAGE_EXECUTE_READWRITE, &old_stub_protect); + if (!succeeded) { + ASSERT(false, "Failed to make page preamble stub read-write-execute."); + delete[] preamble_stub; + return SIDESTEP_ACCESS_DENIED; + } + + SideStepError error_code = RawPatchWithStubAndProtections(target_function, + replacement_function, + preamble_stub, + MAX_PREAMBLE_STUB_SIZE, + NULL); + if (SIDESTEP_SUCCESS != error_code) { + ASSERT1(false); + delete[] preamble_stub; + return error_code; + } + + *original_function_stub = reinterpret_cast<void*>(preamble_stub); + + // NOTE: For hooking malloc/free, we don't want to use streams which + // allocate. Basically, we've hooked malloc, but not necessarily + // hooked free yet. To do anything which uses the heap could crash + // with a mismatched malloc/free! + //LOG(INFO) << "PreamblePatcher::RawPatch successfully patched 0x" << + // target_function; + + return SIDESTEP_SUCCESS; +} + +SideStepError PreamblePatcher::Unpatch(void* target_function, + void* replacement_function, + void* original_function_stub) { + ASSERT1(target_function && original_function_stub); + if (!target_function || !original_function_stub) { + return SIDESTEP_INVALID_PARAMETER; + } + + // We disassemble the preamble of the _stub_ to see how many bytes we + // originally copied to the stub. + MiniDisassembler disassembler; + unsigned int preamble_bytes = 0; + while (preamble_bytes < 5) { + InstructionType instruction_type = + disassembler.Disassemble( + reinterpret_cast<unsigned char*>(original_function_stub) + preamble_bytes, + preamble_bytes); + if (IT_GENERIC != instruction_type) { + ASSERT(false, "Should only have generic instructions in stub!!"); + return SIDESTEP_UNSUPPORTED_INSTRUCTION; + } + } + + // Before unpatching, target_function should be a JMP to + // replacement_function. If it's not, then either it's an error, or + // we're falling into the case where the original instruction was a + // JMP, and we patched the jumped_to address rather than the JMP + // itself. (For instance, if malloc() is just a JMP to __malloc(), + // we patched __malloc() and not malloc().) + unsigned char* target = reinterpret_cast<unsigned char*>(target_function); + while (1) { // we stop when target is a JMP to replacement_function + if (target[0] != ASM_JMP32REL) { + ASSERT(false, "target_function does not look like it was patched."); + return SIDESTEP_INVALID_PARAMETER; + } + int relative_offset; // Windows guarantees int is 4 bytes + ASSERT1(sizeof(relative_offset) == 4); + memcpy(reinterpret_cast<void*>(&relative_offset), + reinterpret_cast<void*>(target + 1), 4); + unsigned char* jump_to = target + 5 + relative_offset; + if (jump_to == replacement_function) + break; + target = jump_to; // follow the jmp + } + + // We need to be able to write to a process-local copy of the first + // MAX_PREAMBLE_STUB_SIZE bytes of target_function. We may be giving execute + // privilege to something that doesn't have it, but that's the price to pay + // for tools. + DWORD old_target_function_protect = 0; + BOOL succeeded = ::VirtualProtect(reinterpret_cast<void*>(target), + MAX_PREAMBLE_STUB_SIZE, + PAGE_EXECUTE_READWRITE, + &old_target_function_protect); + if (!succeeded) { + ASSERT(false, "Failed to make page containing target function " + "copy-on-write."); + return SIDESTEP_ACCESS_DENIED; + } + + // Replace the first few bytes of the original function with the bytes we + // previously moved to the preamble stub. + memcpy(reinterpret_cast<void*>(target), + original_function_stub, preamble_bytes); + + // Stub is now useless so delete it. + // [csilvers: Commented out for perftools because it causes big problems + // when we're unpatching malloc. We just let this live on as a leak.] + //delete original_function_stub; + + // Restore the protection of the first MAX_PREAMBLE_STUB_SIZE bytes of + // target to what they were before we started goofing around. + succeeded = ::VirtualProtect(reinterpret_cast<void*>(target), + MAX_PREAMBLE_STUB_SIZE, + old_target_function_protect, + &old_target_function_protect); + + // Flush the instruction cache to make sure the processor doesn't execute the + // old version of the instructions (before our patch). + // + // See comment on FlushInstructionCache elsewhere in this file. + succeeded = ::FlushInstructionCache(::GetCurrentProcess(), + target, + MAX_PREAMBLE_STUB_SIZE); + if (!succeeded) { + ASSERT(false, "Failed to flush instruction cache."); + return SIDESTEP_UNEXPECTED; + } + + LOG(INFO) << "PreamblePatcher::Unpatch successfully unpatched 0x" << + target_function; + return SIDESTEP_SUCCESS; +} + +}; // namespace sidestep diff --git a/tools/memory_watcher/preamble_patcher.h b/tools/memory_watcher/preamble_patcher.h new file mode 100644 index 0000000..eafe88a --- /dev/null +++ b/tools/memory_watcher/preamble_patcher.h @@ -0,0 +1,319 @@ +/* Copyright (c) 2007, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * + * Definition of PreamblePatcher + */ + +#ifndef MEMORY_WATCHER_PREAMBLE_PATCHER_H__ +#define MEMORY_WATCHER_PREAMBLE_PATCHER_H__ + +#include <windows.h> + +// compatibility shim +#include "base/logging.h" +#define ASSERT(cond, msg) DCHECK(cond) +#define ASSERT1(cond) DCHECK(cond) + +// Maximum size of the preamble stub. We overwrite at least the first 5 +// bytes of the function. Considering the worst case scenario, we need 4 +// bytes + the max instruction size + 5 more bytes for our jump back to +// the original code. With that in mind, 32 is a good number :) +#define MAX_PREAMBLE_STUB_SIZE (32) + +namespace sidestep { + +// Possible results of patching/unpatching +enum SideStepError { + SIDESTEP_SUCCESS = 0, + SIDESTEP_INVALID_PARAMETER, + SIDESTEP_INSUFFICIENT_BUFFER, + SIDESTEP_JUMP_INSTRUCTION, + SIDESTEP_FUNCTION_TOO_SMALL, + SIDESTEP_UNSUPPORTED_INSTRUCTION, + SIDESTEP_NO_SUCH_MODULE, + SIDESTEP_NO_SUCH_FUNCTION, + SIDESTEP_ACCESS_DENIED, + SIDESTEP_UNEXPECTED, +}; + +#define SIDESTEP_TO_HRESULT(error) \ + MAKE_HRESULT(SEVERITY_ERROR, FACILITY_NULL, error) + +// Implements a patching mechanism that overwrites the first few bytes of +// a function preamble with a jump to our hook function, which is then +// able to call the original function via a specially-made preamble-stub +// that imitates the action of the original preamble. +// +// NOTE: This patching mechanism should currently only be used for +// non-production code, e.g. unit tests, because it is not threadsafe. +// See the TODO in preamble_patcher_with_stub.cc for instructions on what +// we need to do before using it in production code; it's fairly simple +// but unnecessary for now since we only intend to use it in unit tests. +// +// To patch a function, use either of the typesafe Patch() methods. You +// can unpatch a function using Unpatch(). +// +// Typical usage goes something like this: +// @code +// typedef int (*MyTypesafeFuncPtr)(int x); +// MyTypesafeFuncPtr original_func_stub; +// int MyTypesafeFunc(int x) { return x + 1; } +// int HookMyTypesafeFunc(int x) { return 1 + original_func_stub(x); } +// +// void MyPatchInitializingFunction() { +// original_func_stub = PreamblePatcher::Patch( +// MyTypesafeFunc, HookMyTypesafeFunc); +// if (!original_func_stub) { +// // ... error handling ... +// } +// +// // ... continue - you have patched the function successfully ... +// } +// @endcode +// +// Note that there are a number of ways that this method of patching can +// fail. The most common are: +// - If there is a jump (jxx) instruction in the first 5 bytes of +// the function being patched, we cannot patch it because in the +// current implementation we do not know how to rewrite relative +// jumps after relocating them to the preamble-stub. Note that +// if you really really need to patch a function like this, it +// would be possible to add this functionality (but at some cost). +// - If there is a return (ret) instruction in the first 5 bytes +// we cannot patch the function because it may not be long enough +// for the jmp instruction we use to inject our patch. +// - If there is another thread currently executing within the bytes +// that are copied to the preamble stub, it will crash in an undefined +// way. +// +// If you get any other error than the above, you're either pointing the +// patcher at an invalid instruction (e.g. into the middle of a multi- +// byte instruction, or not at memory containing executable instructions) +// or, there may be a bug in the disassembler we use to find +// instruction boundaries. +// +// NOTE: In optimized builds, when you have very trivial functions that +// the compiler can reason do not have side effects, the compiler may +// reuse the result of calling the function with a given parameter, which +// may mean if you patch the function in between your patch will never get +// invoked. See preamble_patcher_test.cc for an example. +class PreamblePatcher { + public: + + // This is a typesafe version of RawPatch(), identical in all other + // ways than it takes a template parameter indicating the type of the + // function being patched. + // + // @param T The type of the function you are patching. Usually + // you will establish this type using a typedef, as in the following + // example: + // @code + // typedef BOOL (WINAPI *MessageBoxPtr)(HWND, LPCTSTR, LPCTSTR, UINT); + // MessageBoxPtr original = NULL; + // PreamblePatcher::Patch(MessageBox, Hook_MessageBox, &original); + // @endcode + template <class T> + static SideStepError Patch(T target_function, + T replacement_function, + T* original_function_stub) { + // NOTE: casting from a function to a pointer is contra the C++ + // spec. It's not safe on IA64, but is on i386. We use + // a C-style cast here to emphasize this is not legal C++. + return RawPatch((void*)(target_function), + (void*)(replacement_function), + (void**)(original_function_stub)); + } + + // Patches a named function imported from the named module using + // preamble patching. Uses RawPatch() to do the actual patching + // work. + // + // @param T The type of the function you are patching. Must + // exactly match the function you specify using module_name and + // function_name. + // + // @param module_name The name of the module from which the function + // is being imported. Note that the patch will fail if this module + // has not already been loaded into the current process. + // + // @param function_name The name of the function you wish to patch. + // + // @param replacement_function Your replacement function which + // will be called whenever code tries to call the original function. + // + // @param original_function_stub Pointer to memory that should receive a + // pointer that can be used (e.g. in the replacement function) to call the + // original function, or NULL to indicate failure. + // + // @return One of the EnSideStepError error codes; only SIDESTEP_SUCCESS + // indicates success. + template <class T> + static SideStepError Patch(LPCTSTR module_name, + LPCSTR function_name, + T replacement_function, + T* original_function_stub) { + ASSERT1(module_name && function_name); + if (!module_name || !function_name) { + ASSERT(false, + "You must specify a module name and function name."); + return SIDESTEP_INVALID_PARAMETER; + } + HMODULE module = ::GetModuleHandle(module_name); + ASSERT1(module != NULL); + if (!module) { + ASSERT(false, "Invalid module name."); + return SIDESTEP_NO_SUCH_MODULE; + } + FARPROC existing_function = ::GetProcAddress(module, function_name); + if (!existing_function) { + return SIDESTEP_NO_SUCH_FUNCTION; + } + // NOTE: casting from a function to a pointer is contra the C++ + // spec. It's not safe on IA64, but is on i386. We use + // a C-style cast here to emphasize this is not legal C++. + return RawPatch((void*)existing_function, (void*)replacement_function, + (void**)(original_function_stub)); + } + + // Patches a function by overwriting its first few bytes with + // a jump to a different function. This is the "worker" function + // for each of the typesafe Patch() functions. In most cases, + // it is preferable to use the Patch() functions rather than + // this one as they do more checking at compile time. + // + // @param target_function A pointer to the function that should be + // patched. + // + // @param replacement_function A pointer to the function that should + // replace the target function. The replacement function must have + // exactly the same calling convention and parameters as the original + // function. + // + // @param original_function_stub Pointer to memory that should receive a + // pointer that can be used (e.g. in the replacement function) to call the + // original function, or NULL to indicate failure. + // + // @param original_function_stub Pointer to memory that should receive a + // pointer that can be used (e.g. in the replacement function) to call the + // original function, or NULL to indicate failure. + // + // @return One of the EnSideStepError error codes; only SIDESTEP_SUCCESS + // indicates success. + // + // @note The preamble-stub (the memory pointed to by + // *original_function_stub) is allocated on the heap, and (in + // production binaries) never destroyed, resulting in a memory leak. This + // will be the case until we implement safe unpatching of a method. + // However, it is quite difficult to unpatch a method (because other + // threads in the process may be using it) so we are leaving it for now. + // See however UnsafeUnpatch, which can be used for binaries where you + // know only one thread is running, e.g. unit tests. + static SideStepError RawPatch(void* target_function, + void* replacement_function, + void** original_function_stub); + + // Unpatches target_function and deletes the stub that previously could be + // used to call the original version of the function. + // + // DELETES the stub that is passed to the function. + // + // @param target_function Pointer to the target function which was + // previously patched, i.e. a pointer which value should match the value + // of the symbol prior to patching it. + // + // @param replacement_function Pointer to the function target_function + // was patched to. + // + // @param original_function_stub Pointer to the stub returned when + // patching, that could be used to call the original version of the + // patched function. This function will also delete the stub, which after + // unpatching is useless. + // + // If your original call was + // origptr = Patch(VirtualAlloc, MyVirtualAlloc) + // then to undo it you would call + // Unpatch(VirtualAlloc, MyVirtualAlloc, origptr); + // + // @return One of the EnSideStepError error codes; only SIDESTEP_SUCCESS + // indicates success. + static SideStepError Unpatch(void* target_function, + void* replacement_function, + void* original_function_stub); + + private: + + // Patches a function by overwriting its first few bytes with + // a jump to a different function. This is similar to the RawPatch + // function except that it uses the stub allocated by the caller + // instead of allocating it. + // + // We call VirtualProtect to make the + // target function writable at least for the duration of the call. + // + // @param target_function A pointer to the function that should be + // patched. + // + // @param replacement_function A pointer to the function that should + // replace the target function. The replacement function must have + // exactly the same calling convention and parameters as the original + // function. + // + // @param preamble_stub A pointer to a buffer where the preamble stub + // should be copied. The size of the buffer should be sufficient to + // hold the preamble bytes. + // + // @param stub_size Size in bytes of the buffer allocated for the + // preamble_stub + // + // @param bytes_needed Pointer to a variable that receives the minimum + // number of bytes required for the stub. Can be set to NULL if you're + // not interested. + // + // @return An error code indicating the result of patching. + static SideStepError RawPatchWithStubAndProtections(void* target_function, + void *replacement_function, + unsigned char* preamble_stub, + unsigned long stub_size, + unsigned long* bytes_needed); + + // A helper function used by RawPatchWithStubAndProtections -- it does + // everything but the VirtualProtect wsork. Defined in + // preamble_patcher_with_stub.cc. + static SideStepError RawPatchWithStub(void* target_function, + void *replacement_function, + unsigned char* preamble_stub, + unsigned long stub_size, + unsigned long* bytes_needed); +}; + +}; // namespace sidestep + +#endif // MEMORY_WATCHER_PREAMBLE_PATCHER_H__ diff --git a/tools/memory_watcher/preamble_patcher_with_stub.cc b/tools/memory_watcher/preamble_patcher_with_stub.cc new file mode 100644 index 0000000..d2ce6af --- /dev/null +++ b/tools/memory_watcher/preamble_patcher_with_stub.cc @@ -0,0 +1,200 @@ +/* Copyright (c) 2007, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * + * Implementation of PreamblePatcher + */ + +#include "preamble_patcher.h" + +#include "mini_disassembler.h" + +// Definitions of assembly statements we need +#define ASM_JMP32REL 0xE9 +#define ASM_INT3 0xCC + +namespace sidestep { + +SideStepError PreamblePatcher::RawPatchWithStub( + void* target_function, + void *replacement_function, + unsigned char* preamble_stub, + unsigned long stub_size, + unsigned long* bytes_needed) { + if ((NULL == target_function) || + (NULL == replacement_function) || + (NULL == preamble_stub)) { + ASSERT(false, "Invalid parameters - either pTargetFunction or " + "pReplacementFunction or pPreambleStub were NULL."); + return SIDESTEP_INVALID_PARAMETER; + } + + // TODO(V7:joi) Siggi and I just had a discussion and decided that both + // patching and unpatching are actually unsafe. We also discussed a + // method of making it safe, which is to freeze all other threads in the + // process, check their thread context to see if their eip is currently + // inside the block of instructions we need to copy to the stub, and if so + // wait a bit and try again, then unfreeze all threads once we've patched. + // Not implementing this for now since we're only using SideStep for unit + // testing, but if we ever use it for production code this is what we + // should do. + // + // NOTE: Stoyan suggests we can write 8 or even 10 bytes atomically using + // FPU instructions, and on newer processors we could use cmpxchg8b or + // cmpxchg16b. So it might be possible to do the patching/unpatching + // atomically and avoid having to freeze other threads. Note though, that + // doing it atomically does not help if one of the other threads happens + // to have its eip in the middle of the bytes you change while you change + // them. + unsigned char* target = reinterpret_cast<unsigned char*>(target_function); + + // First, deal with a special case that we see with functions that + // point into an IAT table (including functions linked statically + // into the application): these function already starts with + // ASM_JMP32REL. For instance, malloc() might be implemented as a + // JMP to __malloc(). In that case, we replace the destination of + // the JMP (__malloc), rather than the JMP itself (malloc). This + // way we get the correct behavior no matter how malloc gets called. + if (target[0] == ASM_JMP32REL) { + // target[1-4] holds the place the jmp goes to, but it's + // relative to the next instruction. + int relative_offset; // Windows guarantees int is 4 bytes + ASSERT1(sizeof(relative_offset) == 4); + memcpy(reinterpret_cast<void*>(&relative_offset), + reinterpret_cast<void*>(target + 1), 4); + // I'd like to just say "target = target + 5 + relative_offset" here, but + // I can't, because the new target will need to have its protections set. + return RawPatchWithStubAndProtections(target + 5 + relative_offset, + replacement_function, preamble_stub, + stub_size, bytes_needed); + } + + // Let's disassemble the preamble of the target function to see if we can + // patch, and to see how much of the preamble we need to take. We need 5 + // bytes for our jmp instruction, so let's find the minimum number of + // instructions to get 5 bytes. + MiniDisassembler disassembler; + unsigned int preamble_bytes = 0; + while (preamble_bytes < 5) { + InstructionType instruction_type = + disassembler.Disassemble(target + preamble_bytes, preamble_bytes); + if (IT_JUMP == instruction_type) { + ASSERT(false, "Unable to patch because there is a jump instruction " + "in the first 5 bytes."); + return SIDESTEP_JUMP_INSTRUCTION; + } else if (IT_RETURN == instruction_type) { + ASSERT(false, "Unable to patch because function is too short"); + return SIDESTEP_FUNCTION_TOO_SMALL; + } else if (IT_GENERIC != instruction_type) { + ASSERT(false, "Disassembler encountered unsupported instruction " + "(either unused or unknown)"); + return SIDESTEP_UNSUPPORTED_INSTRUCTION; + } + } + + if (NULL != bytes_needed) + *bytes_needed = preamble_bytes + 5; + + // Inv: cbPreamble is the number of bytes (at least 5) that we need to take + // from the preamble to have whole instructions that are 5 bytes or more + // in size total. The size of the stub required is cbPreamble + size of + // jmp (5) + if (preamble_bytes + 5 > stub_size) { + ASSERT1(false); + return SIDESTEP_INSUFFICIENT_BUFFER; + } + + // First, copy the preamble that we will overwrite. + memcpy(reinterpret_cast<void*>(preamble_stub), + reinterpret_cast<void*>(target), preamble_bytes); + + // Now, make a jmp instruction to the rest of the target function (minus the + // preamble bytes we moved into the stub) and copy it into our preamble-stub. + // find address to jump to, relative to next address after jmp instruction +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable:4244) +#endif + int relative_offset_to_target_rest + = ((reinterpret_cast<unsigned char*>(target) + preamble_bytes) - + (preamble_stub + preamble_bytes + 5)); +#ifdef _MSC_VER +#pragma warning(pop) +#endif + // jmp (Jump near, relative, displacement relative to next instruction) + preamble_stub[preamble_bytes] = ASM_JMP32REL; + // copy the address + memcpy(reinterpret_cast<void*>(preamble_stub + preamble_bytes + 1), + reinterpret_cast<void*>(&relative_offset_to_target_rest), 4); + + // Inv: preamble_stub points to assembly code that will execute the + // original function by first executing the first cbPreamble bytes of the + // preamble, then jumping to the rest of the function. + + // Overwrite the first 5 bytes of the target function with a jump to our + // replacement function. + // (Jump near, relative, displacement relative to next instruction) + target[0] = ASM_JMP32REL; + + // Find offset from instruction after jmp, to the replacement function. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable:4244) +#endif + int offset_to_replacement_function = + reinterpret_cast<unsigned char*>(replacement_function) - + reinterpret_cast<unsigned char*>(target) - 5; +#ifdef _MSC_VER +#pragma warning(pop) +#endif + // complete the jmp instruction + memcpy(reinterpret_cast<void*>(target + 1), + reinterpret_cast<void*>(&offset_to_replacement_function), 4); + // Set any remaining bytes that were moved to the preamble-stub to INT3 so + // as not to cause confusion (otherwise you might see some strange + // instructions if you look at the disassembly, or even invalid + // instructions). Also, by doing this, we will break into the debugger if + // some code calls into this portion of the code. If this happens, it + // means that this function cannot be patched using this patcher without + // further thought. + if (preamble_bytes > 5) { + memset(reinterpret_cast<void*>(target + 5), ASM_INT3, preamble_bytes - 5); + } + + // Inv: The memory pointed to by target_function now points to a relative + // jump instruction that jumps over to the preamble_stub. The preamble + // stub contains the first stub_size bytes of the original target + // function's preamble code, followed by a relative jump back to the next + // instruction after the first cbPreamble bytes. + + return SIDESTEP_SUCCESS; +} + +}; // namespace sidestep diff --git a/tools/purify/chrome_tests.py b/tools/purify/chrome_tests.py new file mode 100644 index 0000000..9ae22dd --- /dev/null +++ b/tools/purify/chrome_tests.py @@ -0,0 +1,259 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# chrome_tests.py + +''' Runs various chrome tests through purify_test.py +''' + +import logging +import optparse +import os +import stat +import sys + +import google.logging_utils +import google.path_utils +import google.platform_utils + +import common + +class TestNotFound(Exception): pass + +class ChromeTests: + + def __init__(self, options, args, test): + # the known list of tests + self._test_list = {"test_shell": self.TestTestShell, + "unit": self.TestUnit, + "net": self.TestNet, + "ipc": self.TestIpc, + "base": self.TestBase, + "layout": self.TestLayout, + "ui": self.TestUI} + + if test not in self._test_list: + raise TestNotFound("Unknown test: %s" % test) + + self._options = options + self._args = args + self._test = test + + script_dir = google.path_utils.ScriptDir() + utility = google.platform_utils.PlatformUtility(script_dir) + # Compute the top of the tree (the "source dir") from the script dir (where + # this script lives). We assume that the script dir is in tools/purify + # relative to the top of the tree. + self._source_dir = os.path.dirname(os.path.dirname(script_dir)) + # since this path is used for string matching, make sure it's always + # an absolute Windows-style path + self._source_dir = utility.GetAbsolutePath(self._source_dir) + purify_test = os.path.join(script_dir, "purify_test.py") + self._command_preamble = ["python.exe", purify_test, "--echo_to_stdout", + "--source_dir=%s" % (self._source_dir), + "--save_cache"] + + def _DefaultCommand(self, module, exe=None): + '''Generates the default command array that most tests will use.''' + module_dir = os.path.join(self._source_dir, module) + if module == "chrome": + # unfortunately, not all modules have the same directory structure + self._data_dir = os.path.join(module_dir, "test", "data", "purify") + else: + self._data_dir = os.path.join(module_dir, "data", "purify") + if not self._options.build_dir: + dir_chrome = os.path.join(self._source_dir, "chrome", "Release") + dir_module = os.path.join(module_dir, "Release") + if exe: + exe_chrome = os.path.join(dir_chrome, exe) + exe_module = os.path.join(dir_module, exe) + if os.path.isfile(exe_chrome) and not os.path.isfile(exe_module): + self._options.build_dir = dir_chrome + elif os.path.isfile(exe_module) and not os.path.isfile(exe_chrome): + self._options.build_dir = dir_module + elif os.stat(exe_module)[stat.ST_MTIME] > os.stat(exe_chrome)[stat.ST_MTIME]: + self._options.build_dir = dir_module + else: + self._options.build_dir = dir_chrome + else: + if os.path.isdir(dir_chrome) and not os.path.isdir(dir_module): + self._options.build_dir = dir_chrome + elif os.path.isdir(dir_module) and not os.path.isdir(dir_chrome): + self._options.build_dir = dir_module + elif os.stat(dir_module)[stat.ST_MTIME] > os.stat(dir_chrome)[stat.ST_MTIME]: + self._options.build_dir = dir_module + else: + self._options.build_dir = dir_chrome + + cmd = self._command_preamble + cmd.append("--data_dir=%s" % self._data_dir) + if self._options.baseline: + cmd.append("--baseline") + if self._options.verbose: + cmd.append("--verbose") + if exe: + cmd.append(os.path.join(self._options.build_dir, exe)) + return cmd + + def Run(self): + ''' Runs the test specified by command-line argument --test ''' + logging.info("running test %s" % (self._test)) + return self._test_list[self._test]() + + def _ReadGtestFilterFile(self, name, cmd): + '''Read a file which is a list of tests to filter out with --gtest_filter + and append the command-line option to cmd. + ''' + filters = [] + filename = os.path.join(self._data_dir, name + ".gtest.txt") + if os.path.exists(filename): + f = open(filename, 'r') + for line in f.readlines(): + if line.startswith("#") or line.startswith("//") or line.isspace(): + continue + line = line.rstrip() + filters.append(line) + gtest_filter = self._options.gtest_filter + if len(filters): + if gtest_filter: + gtest_filter += ":" + if gtest_filter.find("-") < 0: + gtest_filter += "-" + else: + gtest_filter = "-" + gtest_filter += ":".join(filters) + if gtest_filter: + cmd.append("--gtest_filter=%s" % gtest_filter) + + def SimpleTest(self, module, name): + cmd = self._DefaultCommand(module, name) + self._ReadGtestFilterFile(name, cmd) + return common.RunSubprocess(cmd, 0) + + def ScriptedTest(self, module, exe, name, script, multi=False, cmd_args=None): + '''Purify a target exe, which will be executed one or more times via a + script or driver program. + Args: + module - which top level component this test is from (webkit, base, etc.) + exe - the name of the exe (it's assumed to exist in build_dir) + name - the name of this test (used to name output files) + script - the driver program or script. If it's python.exe, we use + search-path behavior to execute, otherwise we assume that it is in + build_dir. + multi - a boolean hint that the exe will be run multiple times, generating + multiple output files (without this option, only the last run will be + recorded and analyzed) + cmd_args - extra arguments to pass to the purify_test.py script + ''' + cmd = self._DefaultCommand(module) + exe = os.path.join(self._options.build_dir, exe) + cmd.append("--exe=%s" % exe) + cmd.append("--name=%s" % name) + if multi: + out = os.path.join(google.path_utils.ScriptDir(), + "latest", "%s%%5d.txt" % name) + cmd.append("--out_file=%s" % out) + if cmd_args: + cmd.extend(cmd_args) + if script[0] != "python.exe" and not os.path.exists(script[0]): + script[0] = os.path.join(self._options.build_dir, script[0]) + cmd.extend(script) + self._ReadGtestFilterFile(name, cmd) + return common.RunSubprocess(cmd, 0) + + def TestBase(self): + return self.SimpleTest("base", "base_unittests.exe") + + def TestIpc(self): + return self.SimpleTest("chrome", "ipc_tests.exe") + + def TestNet(self): + return self.SimpleTest("net", "net_unittests.exe") + + def TestTestShell(self): + return self.SimpleTest("webkit", "test_shell_tests.exe") + + def TestUnit(self): + return self.SimpleTest("chrome", "unit_tests.exe") + + def TestLayout(self): + script = os.path.join(self._source_dir, "webkit", "tools", "layout_tests", + "run_webkit_tests.py") + script_cmd = ["python.exe", script, "--run-singly", "-v", + "--noshow-results", "--time-out-ms=200000"] + if len(self._args): + # if the arg is a txt file, then treat it as a list of tests + if os.path.isfile(self._args[0]) and self._args[0][-4:] == ".txt": + script_cmd.append("--test-list=%s" % self._args[0]) + else: + script_cmd.extend(self._args) + self.ScriptedTest("webkit", "test_shell.exe", "layout", + script_cmd, multi=True, cmd_args=["--timeout=0"]) + # since layout tests take so long to run, having the test red on buildbot + # isn't very useful + return 0 + + def TestUI(self): + return self.ScriptedTest("chrome", "chrome.exe", "ui_tests", + ["ui_tests.exe", "--single-process", "--test-timeout=100000000"], multi=True) + +def _main(argv): + parser = optparse.OptionParser("usage: %prog -b <dir> -t <test> " + "[-t <test> ...]") + parser.disable_interspersed_args() + parser.add_option("-b", "--build_dir", + help="the location of the output of the compiler output") + parser.add_option("-t", "--test", action="append", + help="which test to run") + parser.add_option("", "--baseline", action="store_true", default=False, + help="generate baseline data instead of validating") + parser.add_option("", "--gtest_filter", + help="additional arguments to --gtest_filter") + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="verbose output - enable debug log messages") + (options, args) = parser.parse_args() + + if options.verbose: + google.logging_utils.config_root(logging.DEBUG) + else: + google.logging_utils.config_root() + + if not options.test or not len(options.test): + parser.error("--test not specified") + + for t in options.test: + tests = ChromeTests(options, args, t) + ret = tests.Run() + if ret: return ret + return 0 + +if __name__ == "__main__": + ret = _main(sys.argv) + sys.exit(ret) diff --git a/tools/purify/chrome_tests.sh b/tools/purify/chrome_tests.sh new file mode 100755 index 0000000..3c9c7da --- /dev/null +++ b/tools/purify/chrome_tests.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +system_root=`cygpath "$SYSTEMROOT"` +export PATH="/usr/bin:$system_root/system32:$system_root:$system_root/system32/WBEM" + +exec_dir=$(dirname $0) + +"$exec_dir/../../third_party/python_24/python.exe" \ + "$exec_dir/chrome_tests.py" "$@" diff --git a/tools/purify/common.py b/tools/purify/common.py new file mode 100644 index 0000000..3dd72af --- /dev/null +++ b/tools/purify/common.py @@ -0,0 +1,293 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# common.py + +""" Common code used by purify_test.py and quantify_test.py in order to automate +running of Rational Purify and Quantify in a consistent manner. +""" + +# Purify and Quantify have a front-end (e.g. quantifyw.exe) which talks to a +# back-end engine (e.g. quantifye.exe). The back-end seems to handle +# instrumentation, while the front-end controls program execution and +# measurement. The front-end will dynamically launch the back-end if +# instrumentation is needed (sometimes in the middle of a run if a dll is +# loaded dynamically). +# In an ideal world, this script would simply execute the front-end and check +# the output. However, purify is not the most reliable or well-documented app +# on the planet, and my attempts to get it to run this way led to the back-end +# engine hanging during instrumentation. The workaround to this was to run two +# passes, first running the engine to do instrumentation rather than letting +# the front-end do it for you, then running the front-end to actually do the +# run. Each time through we're deleting all of the instrumented files in the +# cache to ensure that we're testing that instrumentation works from scratch. +# (although this can be changed with an option) + +import datetime +import logging +import optparse +import os +import subprocess +import sys +import tempfile +import time + +import google.logging_utils + +# hard-coded location of Rational files and directories +RATIONAL_PATH = os.path.join("C:\\", "Program Files", "Rational") +COMMON_PATH = os.path.join(RATIONAL_PATH, "common") +PPLUS_PATH = os.path.join(RATIONAL_PATH, "PurifyPlus") +PURIFY_PATH = os.path.join(COMMON_PATH, "purify.exe") +PURIFYW_PATH = os.path.join(PPLUS_PATH, "purifyW.exe") +PURIFYE_PATH = os.path.join(PPLUS_PATH, "purifye.exe") +QUANTIFYE_PATH = os.path.join(PPLUS_PATH, "quantifye.exe") +QUANTIFYW_PATH = os.path.join(PPLUS_PATH, "quantifyw.exe") + +class TimeoutError(Exception): pass + +def RunSubprocess(proc, timeout=0, detach=False): + """ Runs a subprocess, polling every .2 seconds until it finishes or until + timeout is reached. Then kills the process with taskkill. A timeout <= 0 + means no timeout. + + Args: + proc: list of process components (exe + args) + timeout: how long to wait before killing, <= 0 means wait forever + detach: Whether to pass the DETACHED_PROCESS argument to CreateProcess + on Windows. This is used by Purify subprocesses on buildbot which + seem to get confused by the parent console that buildbot sets up. + """ + logging.info("running %s" % (" ".join(proc))) + if detach: + # see MSDN docs for "Process Creation Flags" + DETACHED_PROCESS = 0x8 + p = subprocess.Popen(proc, creationflags=DETACHED_PROCESS) + else: + p = subprocess.Popen(proc) + if timeout <= 0: + while p.poll() is None: + time.sleep(0.2) + else: + wait_until = time.time() + timeout + while p.poll() is None and time.time() < wait_until: + time.sleep(0.2) + result = p.poll() + if result is None: + subprocess.call(["taskkill", "/T", "/F", "/PID", str(p.pid)]) + logging.error("KILLED %d" % (p.pid)) + # give the process a chance to actually die before continuing + # so that cleanup can happen safely + time.sleep(1.0) + logging.error("TIMEOUT waiting for %s" % (proc[0])) + raise TimeoutError(proc[0]) + if result: + logging.error("%s exited with non-zero result code %d" % (proc[0], result)) + return result + +def FixPath(path): + """We pass computed paths to Rational as arguments, so these paths must be + valid windows paths. When running in cygwin's python, computed paths + wind up looking like /cygdrive/c/..., so we need to call out to cygpath + to fix them up. + """ + if sys.platform != "cygwin": + return path + p = subprocess.Popen(["cygpath", "-a", "-m", path], stdout=subprocess.PIPE) + return p.communicate()[0].rstrip() + +class Rational(object): + ''' Common superclass for Purify and Quantify automation objects. Handles + common argument parsing as well as the general program flow of Instrument, + Execute, Analyze. + ''' + + def __init__(self): + google.logging_utils.config_root() + self._out_file = None + + def Run(self): + '''Call this to run through the whole process: + Setup, Instrument, Execute, Analyze''' + start = datetime.datetime.now() + retcode = -1 + if self.Setup(): + if self.Instrument(): + if self.Execute(): + retcode = self.Analyze() + if not retcode: + logging.info("instrumentation and execution completed successfully.") + else: + logging.error("Analyze failed") + else: + logging.error("Execute failed") + else: + logging.error("Instrument failed") + self.Cleanup() + else: + logging.error("Setup failed") + end = datetime.datetime.now() + seconds = (end - start).seconds + hours = seconds / 3600 + seconds = seconds % 3600 + minutes = seconds / 60 + seconds = seconds % 60 + logging.info("elapsed time: %02d:%02d:%02d" % (hours, minutes, seconds)) + return retcode + + def CreateOptionParser(self): + '''Creates OptionParser with shared arguments. Overridden by subclassers + to add custom arguments.''' + parser = optparse.OptionParser("usage: %prog [options] <program to test>") + # since the trailing program likely has command-line args of itself + # we need to stop parsing when we reach the first positional arg + parser.disable_interspersed_args() + parser.add_option("-o", "--out_file", dest="out_file", metavar="OUTFILE", + default="", + help="output data is written to OUTFILE") + parser.add_option("-s", "--save_cache", + dest="save_cache", action="store_true", default=False, + help="don't delete instrumentation cache") + parser.add_option("-c", "--cache_dir", dest="cache_dir", metavar="CACHEDIR", + default="", + help="location of instrumentation cache is CACHEDIR") + parser.add_option("-m", "--manual", + dest="manual_run", action="store_true", default=False, + help="target app is being run manually, don't timeout") + parser.add_option("-t", "--timeout", + dest="timeout", metavar="TIMEOUT", default=10000, + help="timeout in seconds for the run (default 10000)") + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="verbose output - enable debug log messages") + self._parser = parser + + def Setup(self): + if self.ParseArgv(): + logging.info("instrumentation cache in %s" % self._cache_dir) + logging.info("output saving to %s" % self._out_file) + # Ensure that Rational's common dir and cache dir are in the front of the + # path. The common dir is required for purify to run in any case, and + # the cache_dir is required when using the /Replace=yes option. + os.environ["PATH"] = (COMMON_PATH + ";" + self._cache_dir + ";" + + os.environ["PATH"]) + # clear the cache to make sure we're starting clean + self.__ClearInstrumentationCache() + return True + return False + + def Instrument(self, proc): + '''Instrument the app to be tested. Full instrumentation command-line + provided by subclassers via proc.''' + logging.info("starting instrumentation...") + if RunSubprocess(proc, self._timeout, detach=True) == 0: + if "/Replace=yes" in proc: + if os.path.exists(self._exe + ".Original"): + return True + elif os.path.isdir(self._cache_dir): + for cfile in os.listdir(self._cache_dir): + # TODO(erikkay): look for the actual munged purify filename + ext = os.path.splitext(cfile)[1] + if ext == ".exe": + return True + logging.error("no instrumentation data generated") + return False + + def Execute(self, proc): + ''' Execute the app to be tested after successful instrumentation. + Full execution command-line provided by subclassers via proc.''' + logging.info("starting execution...") + # note that self._args begins with the exe to be run + proc += self._args + if RunSubprocess(proc, self._timeout) == 0: + return True + return False + + def Analyze(self): + '''Analyze step after a successful Execution. Should be overridden + by the subclasser if instrumentation is desired. + Returns 0 for success, -88 for warning (see ReturnCodeCommand) and anything + else for error + ''' + return -1 + + def ParseArgv(self): + '''Parses arguments according to CreateOptionParser + Subclassers must override if they have extra arguments.''' + self.CreateOptionParser() + (self._options, self._args) = self._parser.parse_args() + if self._options.verbose: + google.logging_utils.config_root(logging.DEBUG) + self._save_cache = self._options.save_cache + self._manual_run = self._options.manual_run + if self._manual_run: + logging.info("manual run - timeout disabled") + self._timeout = 0 + else: + self._timeout = int(self._options.timeout) + logging.info("timeout set to %ds" % (self._timeout)) + if self._save_cache: + logging.info("saving instrumentation cache") + if not self._options.cache_dir: + try: + temp_dir = os.environ["TEMP"] + except KeyError: + temp_dir = tempfile.mkdtemp() + self._cache_dir = os.path.join(FixPath(temp_dir), + "instrumentation_cache") + else: + self._cache_dir = FixPath(os.path.abspath(self._options.cache_dir)) + if self._options.out_file: + self._out_file = FixPath(os.path.abspath(self._options.out_file)) + if len(self._args) == 0: + self._parser.error("missing program to %s" % (self.__class__.__name__,)) + return False + self._exe = self._args[0] + self._exe_dir = FixPath(os.path.abspath(os.path.dirname(self._exe))) + return True + + def Cleanup(self): + # delete the cache to avoid filling up the hard drive when we're using + # temporary directory names + self.__ClearInstrumentationCache() + + def __ClearInstrumentationCache(self): + if not self._save_cache: + logging.info("clearing instrumentation cache %s" % self._cache_dir) + if os.path.isdir(self._cache_dir): + for cfile in os.listdir(self._cache_dir): + file = os.path.join(self._cache_dir, cfile); + if os.path.isfile(file): + try: + os.remove(file) + except: + logging.warning("unable to delete file %s: %s" % (file, + sys.exc_info()[0])) + + diff --git a/tools/purify/data/filters.pft b/tools/purify/data/filters.pft Binary files differnew file mode 100644 index 0000000..a353c12 --- /dev/null +++ b/tools/purify/data/filters.pft diff --git a/tools/purify/data/ignore.txt b/tools/purify/data/ignore.txt new file mode 100644 index 0000000..16a11b2 --- /dev/null +++ b/tools/purify/data/ignore.txt @@ -0,0 +1,12 @@ +# See bug 1157381 +Pure: Trap bits found in live chunk + +# See bugs 1151263 and 1164562 +Memory leak .+ allocated in InitSecurityInterfaceA \[SECUR32\.DLL\] + +# See bug 1163766 +# Ugly regexps are trying to deal with Purify's demangling bugs. +Memory leak .+ allocated in \?NewRunnableMethod.+ExpireHistoryBackend.+ScopedRunnableMethodFactory +Memory leak .+ allocated in RevocableStore::RevokeAll\(void\) +Memory leak .+ allocated in \?NewRunnableMethod.+CommitLaterTask.+CancelableTask.+CommitLaterTask +Memory leak .+ allocated in history::HistoryBackend::ScheduleCommit\(void\) diff --git a/tools/purify/purify_analyze.py b/tools/purify/purify_analyze.py new file mode 100644 index 0000000..4625a70 --- /dev/null +++ b/tools/purify/purify_analyze.py @@ -0,0 +1,874 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# purify_analyze.py + +''' Given a Purify text file, parses messages, normalizes and uniques them. +If there's an existing baseline of this data, it can compare against that +baseline and return an error code if there are any new errors not in the +baseline. ''' + +import logging +import optparse +import os +import re +import sys + +import google.logging_utils +import google.path_utils + +import purify_message + +class MemoryTreeNode(object): + ''' A node in a tree representing stack traces of memory allocation. + Essentially, each node in the tree is a hashtable mapping a child + function name to a child node. Each node contains the total number + of bytes of all of its descendants. + See also: PurifyAnalyze.PrintMemoryInUse() + ''' + + pat_initializer = re.compile('(.*)\`dynamic initializer for \'(.*)\'\'') + + @classmethod + def CreateTree(cls, message_list): + '''Creates a tree from message_list. All of the Message objects are built + into a tree with a default "ROOT" root node that is then returned. + Args: + message_list: a MessageList object. + ''' + root = MemoryTreeNode("ROOT", 0, 0) + msgs = message_list.AllMessages() + for msg in msgs: + bytes = msg._bytes + blocks = msg._blocks + stack = msg.GetAllocStack() + stack_lines = stack.GetLines() + size = len(stack_lines) + node = root + node._AddAlloc(bytes, blocks) + counted = False + # process stack lines from the bottom up to build a call-stack tree + functions = [line["function"] for line in stack_lines] + functions.reverse() + for func in functions: + if node == root: + m = MemoryTreeNode.pat_initializer.match(func) + if m: + node = node._AddChild("INITIALIZERS", bytes, blocks) + func = m.group(1) + m.group(2) + # don't process ellided or truncated stack lines + if func: + node = node._AddChild(func, bytes, blocks) + counted = True + if not counted: + # Nodes with no stack frames in our code wind up not being counted + # above. These seem to be attributable to Windows DLL + # initialization, so just throw them into that bucket. + node._AddChild("WINDOWS", bytes, blocks) + return root + + def __init__(self, function, bytes, blocks): + ''' + Args: + function: A string representing a unique method or function. + bytes: initial number of bytes allocated in this node + blocks: initial number of blocks allocated in this node + ''' + self._function = function + self._bytes = bytes + self._blocks = blocks + self._allocs = 1 + self._children = {} + + def _AddAlloc(self, bytes, blocks): + '''Adds bytes and blocks to this node's allocation totals + ''' + self._allocs += 1 + self._bytes += bytes + self._blocks += blocks + + def _AddChild(self, function, bytes, blocks): + '''Adds a child node if not present. Otherwise, adds + bytes and blocks to it's allocation total. + ''' + if function not in self._children: + self._children[function] = MemoryTreeNode(function, bytes, blocks) + else: + self._children[function]._AddAlloc(bytes, blocks) + return self._children[function] + + def __cmp__(self, other): + # sort by size, then blocks, then function name + return cmp((self._bytes, self._blocks, self._function), + (other._bytes, other._blocks, other._function)) + + def __str__(self): + return "(%d bytes, %d blocks, %d allocs) %s" % ( + self._bytes, self._blocks, self._allocs, self._function) + + def PrintRecursive(self, padding="", byte_filter=0): + '''Print the tree and all of its children recursively (depth-first). All + nodes at a given level of the tree are sorted in descending order by size. + + Args: + padding: Printed at the front of the line. Each recursive call adds a + single space character. + byte_filter: a number of bytes below which we'll prune the tree + ''' + print "%s%s" % (padding, self) + padding = padding + " " + # sort the children in descending order (see __cmp__) + swapped = self._children.values() + swapped.sort(reverse=True) + rest_bytes = 0 + rest_blocks = 0 + rest_allocs = 0 + for node in swapped: + if node._bytes < byte_filter: + rest_bytes += node._bytes + rest_blocks += node._blocks + rest_allocs += node._allocs + else: + node.PrintRecursive(padding, byte_filter) + if rest_bytes: + print "%s(%d bytes, %d blocks, %d allocs) PRUNED" % (padding, + rest_bytes, rest_blocks, rest_allocs) + +class PurifyAnalyze: + ''' Given a Purify text file, parses all of the messages inside of it and + normalizes them. Provides a mechanism for comparing this normalized set + against a baseline and detecting if new errors have been introduced. ''' + + # a line which is the start of a new message + pat_msg_start = re.compile('^\[([A-Z])\] (.*)$') + # a message with a specific type + pat_msg_type = re.compile('^([A-Z]{3}): (.*)$') + pat_leak_summary = re.compile("Summary of ... memory leaks") + pat_miu_summary = re.compile("Summary of ... memory in use") + pat_starting = re.compile("Starting Purify'd ([^\\s]+\\\\[^\\s]+)") + pat_arguments = re.compile("\s+Command line arguments:\s+([^\s].*)") + pat_terminate = re.compile('Message: TerminateProcess called with code') + # Purify treats this as a warning, but for us it's a fatal error. + pat_instrumentation_failed = re.compile('^.* file not instrumented') + # misc to ignore + pat_ignore = (re.compile('^(Start|Exit)ing'), + re.compile('^Program terminated'), + re.compile('^Terminating thread'), + re.compile('^Message: CryptVerifySignature')) + # message types that aren't analyzed + # handled, ignored and continued exceptions will likely never be interesting + # TODO(erikkay): MPK ("potential" memory leaks) may be worth turning on + types_excluded = ("EXH", "EXI", "EXC", "MPK") + + + def __init__(self, files, echo, name=None, source_dir=None, data_dir=None): + # The input file we're analyzing. + self._files = files + # Whether the input file contents should be echoed to stdout. + self._echo = echo + # A symbolic name for the run being analyzed, often the name of the + # exe which was purified. + self._name = name + # The top of the source code tree of the code we're analyzing. + # This prefix is stripped from all filenames in stacks for normalization. + if source_dir: + purify_message.Stack.SetSourceDir(source_dir) + if data_dir: + self._data_dir = data_dir + else: + self._data_dir = os.path.join(google.path_utils.ScriptDir(), "data") + # A map of message_type to a MessageList of that type. + self._message_lists = {} + self._ReadIgnoreFile() + + def _ReadIgnoreFile(self): + '''Read a file which is a list of regexps for either the title or the + top-most visible stack line. + ''' + self._pat_ignore = [] + filenames = [os.path.join(self._data_dir, "ignore.txt"), + os.path.join(google.path_utils.ScriptDir(), "data", "ignore.txt")] + for filename in filenames: + if os.path.exists(filename): + f = open(filename, 'r') + for line in f.readlines(): + if line.startswith("#") or line.startswith("//") or line.isspace(): + continue + line = line.rstrip() + pat = re.compile(line) + if pat: + self._pat_ignore.append(pat) + + def ShouldIgnore(self, msg): + '''Should the message be ignored as irrelevant to analysis ''' + # never ignore memory in use + if msg.Type() == "MIU": + return False + + # check ignore patterns against title and top-most visible stack frames + strings = [msg._title] + err = msg.GetErrorStack() + if err: + line = err.GetTopVisibleStackLine().get('function', None) + if line: + strings.append(line) + alloc = msg.GetAllocStack() + if alloc: + line = alloc.GetTopVisibleStackLine().get('function', None) + if line: + strings.append(line) + for pat in self._pat_ignore: + for str in strings: + if pat.match(str): + logging.debug("Igorning message based on ignore.txt") + logging.debug(msg.NormalizedStr(verbose=True)) + return True + + # unless it's explicitly in the ignore file, never ignore these + if msg.Type() == purify_message.FATAL: + return False + + # certain message types aren't that interesting + if msg.Type() in PurifyAnalyze.types_excluded: + logging.debug("Igorning message because type is excluded") + logging.debug(msg.NormalizedStr(verbose=True)) + return True + # if the message stacks have no local stack frames, we can ignore them + if msg.StacksAllExternal(): + logging.debug("Igorning message because stacks are all external") + logging.debug(msg.NormalizedStr(verbose=True)) + return True + + # Microsoft's STL has a bunch of non-harmful UMRs in it. Most of them + # are filtered out by Purify's default filters and by our explicit ignore + # list. This code notices ones that have made it through so we can add + # them to the ignore list later. + if msg.Type() == "UMR": + if err.GetTopStackLine()['file'].endswith('.'): + logging.debug("non-ignored UMR in STL: %s" % msg._title) + + return False + + def AddMessage(self, msg): + ''' Append the message to an array for its type. Returns boolean indicating + whether the message was actually added or was ignored.''' + if msg: + if self.ShouldIgnore(msg): + return False + if msg.Type() not in self._message_lists: + self._message_lists[msg.Type()] = purify_message.MessageList(msg.Type()) + self._message_lists[msg.Type()].AddMessage(msg) + return True + return False + + def _BeginNewSublist(self, key): + '''See MessageList.BeginNewSublist for details. + ''' + if key in self._message_lists: + self._message_lists[key].BeginNewSublist() + + def ReadFile(self): + ''' Reads a Purify ASCII file and parses and normalizes the messages in + the file. + Returns False if a fatal error was detected, True otherwise. + ''' + # Purify files consist of a series of "messages". These messages have a type + # (designated as a three letter code - see message_type), a severity + # (designated by a one letter code - see message_severity) and some + # textual details. It will often also have a stack trace of the error + # location, and (for memory errors) may also have a stack trace of the + # allocation location. + + fatal_errors = 0 + fatal_exe = "" + + for file in self._files: + exe = "" + error = None + message = None + for line in open(file, mode='rb'): + line = line.rstrip() + m = PurifyAnalyze.pat_msg_start.match(line) + if m: + if exe == fatal_exe: + # since we hit a fatal error in this program, ignore all messages + # until the program changes + continue + # we matched a new message, so if there's an existing one, it's time + # to finish processing it + if message: + message.SetProgram(exe) + if not self.AddMessage(message): + # error is only set if the message we just tried to add would + # otherwise be considered a fatal error. Since AddMessage failed + # (presumably the messages matched the ignore list), we reset + # error to None + error = None + message = None + if error: + if error.Type() == "EXU": + # Don't treat EXU as fatal, since unhandled exceptions + # in other threads don't necessarily lead to the app to exit. + # TODO(erikkay): verify that we still do trap exceptions that lead + # to early termination. + logging.warning(error.NormalizedStr(verbose=True)) + error = None + else: + if len(self._files) > 1: + logging.error("Fatal error in program: %s" % error.Program()) + logging.error(error.NormalizedStr(verbose=True)) + fatal_errors += 1 + error = None + fatal_exe = exe + continue + severity = m.group(1) + line = m.group(2) + m = PurifyAnalyze.pat_msg_type.match(line) + if m: + type = m.group(1) + message = purify_message.Message(severity, type, m.group(2)) + if type == "EXU": + error = message + elif severity == "O": + message = purify_message.Message(severity, purify_message.FATAL, + line) + # This is an internal Purify error, and it means that this run can't + # be trusted and analysis should be aborted. + error = message + elif PurifyAnalyze.pat_instrumentation_failed.match(line): + message = purify_message.Message(severity, purify_message.FATAL, + line) + error = message + elif PurifyAnalyze.pat_terminate.match(line): + message = purify_message.Message(severity, purify_message.FATAL, + line) + error = message + elif PurifyAnalyze.pat_leak_summary.match(line): + # TODO(erikkay): should we do sublists for MLK and MPK too? + # Maybe that means we need to handle "new" and "all" messages + # separately. + #self._BeginNewSublist("MLK") + #self._BeginNewSublist("MPK") + pass + elif PurifyAnalyze.pat_miu_summary.match(line): + # Each time Purify is asked to do generate a list of all memory in use + # or new memory in use, it first emits this summary line. Since the + # different lists can overlap, we need to tell MessageList to begin + # a new sublist. + # TODO(erikkay): should we tag "new" and "all" sublists explicitly + # somehow? + self._BeginNewSublist("MIU") + elif PurifyAnalyze.pat_starting.match(line): + m = PurifyAnalyze.pat_starting.match(line) + exe = m.group(1) + last_slash = exe.rfind("\\") + if not purify_message.Stack.source_dir: + path = os.path.abspath(os.path.join(exe[:last_slash], "..", "..")) + purify_message.Stack.SetSourceDir(path) + if not self._name: + self._name = exe[(last_slash+1):] + else: + unknown = True + for pat in PurifyAnalyze.pat_ignore: + if pat.match(line): + unknown = False + break + if unknown: + logging.error("unknown line " + line) + else: + if message: + message.AddLine(line) + elif PurifyAnalyze.pat_arguments.match(line): + m = PurifyAnalyze.pat_arguments.match(line) + exe += " " + m.group(1) + + # Purify output should never end with a real message + if message: + logging.error("Unexpected message at end of file %s" % file) + + return fatal_errors == 0 + + def GetMessageList(self, key): + if key in self._message_lists: + return self._message_lists[key] + else: + return None + + def PrintSummary(self, echo=None): + ''' Print a summary of how many messages of each type were found. ''' + # make sure everyone else is done first + sys.stderr.flush() + sys.stdout.flush() + if echo == None: + echo = self._echo + logging.info("summary of Purify messages:") + for key in self._message_lists: + list = self._message_lists[key] + unique = list.UniqueMessages() + all = list.AllMessages() + count = 0 + for msg in all: + count += msg._count + logging.info("%s(%s) unique:%d total:%d" % (self._name, + purify_message.GetMessageType(key), len(unique), count)) + if key not in ["MIU"]: + ignore_file = "%s_%s_ignore.txt" % (self._name, key) + ignore_hashes = self._MessageHashesFromFile(ignore_file) + ignored = 0 + + groups = list.UniqueMessageGroups() + group_keys = groups.keys() + group_keys.sort(cmp=lambda x,y: len(groups[y]) - len(groups[x])) + for group in group_keys: + # filter out ignored messages + kept_msgs= [x for x in groups[group] if hash(x) not in ignore_hashes] + ignored += len(groups[group]) - len(kept_msgs) + groups[group] = kept_msgs + if ignored: + logging.info("%s(%s) ignored:%d" % (self._name, + purify_message.GetMessageType(key), ignored)) + total = reduce(lambda x, y: x + len(groups[y]), group_keys, 0) + if total: + print "%s(%s) group summary:" % (self._name, + purify_message.GetMessageType(key)) + print " TOTAL: %d" % total + for group in group_keys: + if len(groups[group]): + print " %s: %d" % (group, len(groups[group])) + if echo: + for group in group_keys: + msgs = groups[group] + if len(msgs) == 0: + continue + print "messages from %s (%d)" % (group, len(msgs)) + print "="*79 + for msg in msgs: + # for the summary output, line numbers are useful + print msg.NormalizedStr(verbose=True) + # make sure stdout is flushed to avoid weird overlaps with logging + sys.stdout.flush() + + def PrintMemoryInUse(self, byte_filter=16384): + ''' Print one or more trees showing a hierarchy of memory allocations. + Args: + byte_filter: a number of bytes below which we'll prune the tree + ''' + list = self.GetMessageList("MIU") + sublists = list.GetSublists() + if not sublists: + sublists = [list] + trees = [] + summaries = [] + # create the trees and summaries + for sublist in sublists: + tree = MemoryTreeNode.CreateTree(sublist) + trees.append(tree) + + # while the tree is a hierarchical assignment from the root/bottom of the + # stack down, the summary is simply adding the total of the top-most + # stack item from our code + summary = {} + total = 0 + summaries.append(summary) + for msg in sublist.AllMessages(): + total += msg._bytes + stack = msg.GetAllocStack() + if stack._all_external: + alloc_caller = "WINDOWS" + else: + lines = stack.GetLines() + for line in lines: + alloc_caller = line["function"] + if alloc_caller: + break + summary[alloc_caller] = summary.get(alloc_caller, 0) + msg._bytes + summary["TOTAL"] = total + + # print out the summaries and trees. + # TODO(erikkay): perhaps we should be writing this output to a file + # instead? + tree_number = 1 + num_trees = len(trees) + for tree, summary in zip(trees, summaries): + print "MEMORY SNAPSHOT %d of %d" % (tree_number, num_trees) + lines = summary.keys() + lines.sort(cmp=lambda x,y: summary[y] - summary[x]) + rest = 0 + for line in lines: + bytes = summary[line] + if bytes < byte_filter: + rest += bytes + else: + print "%d: %s" % (bytes, line) + print "%d: REST" % rest + print + print "BEGIN TREE" + tree.PrintRecursive(byte_filter=byte_filter) + tree_number += 1 + + # make sure stdout is flushed to avoid weird overlaps with logging + sys.stdout.flush() + + def PrintBugReport(self): + ''' Print a summary of how many messages of each type were found. ''' + # make sure everyone else is done first + sys.stderr.flush() + sys.stdout.flush() + logging.info("summary of Purify bugs:") + # This is a specialized set of counters for layout tests, with some + # unfortunate hard-coded knowledge. + layout_test_counts = {} + for key in self._message_lists: + bug = {} + list = self._message_lists[key] + unique = list.UniqueMessages() + all = list.AllMessages() + count = 0 + for msg in all: + if msg._title not in bug: + # use a single sample message to represent all messages + # that match this title + bug[msg._title] = {"message":msg, + "total":0, + "count":0, + "programs":set()} + this_bug = bug[msg._title] + this_bug["total"] += msg._count + this_bug["count"] += 1 + this_bug["programs"].add(msg.Program()) + # try to summarize the problem areas for layout tests + if self._name == "layout": + prog = msg.Program() + prog_args = prog.split(" ") + if len(prog_args): + path = prog_args[-1].replace('\\', '/') + index = path.rfind("layout_tests/") + if index >= 0: + path = path[(index + len("layout_tests/")):] + else: + index = path.rfind("127.0.0.1:") + if index >= 0: + # the port number is 8000 or 9000, but length is the same + path = "http: " + path[(index + len("127.0.0.1:8000/")):] + path = "/".join(path.split('/')[0:-1]) + count = 1 + layout_test_counts.get(path, 0) + layout_test_counts[path] = count + for title in bug: + b = bug[title] + print "[%s] %s" % (key, title) + print "%d tests, %d stacks, %d instances" % (len(b["programs"]), + b["count"], b["total"]) + print "Reproducible with:" + for program in b["programs"]: + print " %s" % program + print "Sample error details:" + print "=====================" + print b["message"].NormalizedStr(verbose=True) + if len(layout_test_counts): + print + print "Layout test error counts" + print "========================" + paths = layout_test_counts.keys() + paths.sort() + for path in paths: + print "%s: %d" % (path, layout_test_counts[path]) + # make sure stdout is flushed to avoid weird overlaps with logging + sys.stdout.flush() + + def SaveLatestStrings(self, string_list, key, fname_extra=""): + '''Output a list of strings to a file in the "latest" dir. + ''' + script_dir = google.path_utils.ScriptDir() + path = os.path.join(script_dir, "latest") + out = os.path.join(path, "%s_%s%s.txt" % (self._name, key, fname_extra)) + logging.info("saving %s" % (out)) + try: + f = open(out, "w+") + f.write('\n'.join(string_list)) + except IOError, (errno, strerror): + logging.error("error writing to file %s (%d, %s)" % out, errno, strerror) + if f: + f.close() + return True + + def SaveResults(self, path=None, verbose=False): + ''' Output normalized data to baseline files for future comparison runs. + Messages are saved in sorted order into a separate file for each message + type. See Message.NormalizedStr() for details of what's written. + ''' + if not path: + path = self._data_dir + for key in self._message_lists: + out = os.path.join(path, "%s_%s.txt" % (self._name, key)) + logging.info("saving %s" % (out)) + f = open(out, "w+") + list = self._message_lists[key].UniqueMessages() + # TODO(erikkay): should the count of each message be a diff factor? + # (i.e. the same error shows up, but more frequently) + for message in list: + f.write(message.NormalizedStr(verbose=verbose)) + f.write("\n") + f.close() + return True + + def _MessageHashesFromFile(self, filename): + ''' Reads a file of normalized messages (see SaveResults) and creates a + dictionary mapping the hash of each message to its text. + ''' + # NOTE: this uses the same hashing algorithm as Message.__hash__. + # Unfortunately, we can't use the same code easily since Message is based + # on parsing an original Purify output file and this code is reading a file + # of already normalized messages. This means that these two bits of code + # need to be kept in sync. + msgs = {} + if not os.path.isabs(filename): + filename = os.path.join(self._data_dir, filename) + if os.path.exists(filename): + logging.info("reading messages from %s" % filename) + file = open(filename, "r") + msg = "" + title = None + lines = file.readlines() + # in case the file doesn't end in a blank line + lines.append("\n") + for line in lines: + # allow these files to have comments in them + if line.startswith('#') or line.startswith('//'): + continue + if not title: + if not line.isspace(): + # first line of each message is a title + title = line + continue + elif not line.isspace(): + msg += line + else: + # note that the hash doesn't include the title, see Message.__hash__ + h = hash(msg) + msgs[h] = title + msg + title = None + msg = "" + logging.info("%s: %d msgs" % (filename, len(msgs))) + return msgs + + def _SaveLatestGroupSummary(self, message_list): + '''Save a summary of message groups and their counts to a file in "latest" + ''' + string_list = [] + groups = message_list.UniqueMessageGroups() + group_keys = groups.keys() + + group_keys.sort(cmp=lambda x,y: len(groups[y]) - len(groups[x])) + for group in group_keys: + string_list.append("%s: %d" % (group, len(groups[group]))) + + self.SaveLatestStrings(string_list, message_list.GetType(), "_GROUPS") + + def CompareResults(self): + ''' Compares the results from the current run with the baseline data + stored in data/<name>_<key>.txt returning False if it finds new errors + that are not in the baseline. See ReadFile() and SaveResults() for + details of what's in the original file and what's in the baseline. + Errors that show up in the baseline but not the current run are not + considered errors (they're considered "fixed"), but they do suggest + that the baseline file could be re-generated.''' + errors = 0 + fixes = 0 + for type in purify_message.message_type: + if type in ["MIU"]: + continue + # number of new errors for this message type + type_errors = [] + # number of new unexpected fixes for this message type + type_fixes = [] + # the messages from the current run that are in the baseline + new_baseline = [] + # a common prefix used to describe the program being analyzed and the + # type of message which is used to generate filenames and descriptive + # error messages + type_name = "%s_%s" % (self._name, type) + + # open the baseline file to compare against + baseline_file = "%s.txt" % type_name + baseline_hashes = self._MessageHashesFromFile(baseline_file) + + # read the flakey file if it exists + flakey_file = "%s_flakey.txt" % type_name + flakey_hashes = self._MessageHashesFromFile(flakey_file) + + # read the ignore file if it exists + ignore_file = "%s_ignore.txt" % type_name + ignore_hashes = self._MessageHashesFromFile(ignore_file) + + # messages from the current run + current_list = self.GetMessageList(type) + if current_list: + # Since we're looking at the list of unique messages, + # if the number of occurrances of a given unique message + # changes, it won't show up as an error. + current_messages = current_list.UniqueMessages() + else: + current_messages = [] + current_hashes = {} + # compute errors and new baseline + for message in current_messages: + msg_hash = hash(message) + current_hashes[msg_hash] = message + if msg_hash in ignore_hashes or msg_hash in flakey_hashes: + continue + if msg_hash in baseline_hashes: + new_baseline.append(msg_hash) + continue + type_errors.append(msg_hash) + # compute unexpected fixes + for msg_hash in baseline_hashes: + if (msg_hash not in current_hashes and + msg_hash not in ignore_hashes and + msg_hash not in flakey_hashes): + type_fixes.append(baseline_hashes[msg_hash]) + + if len(current_messages) or len(type_errors) or len(type_fixes): + logging.info("%d '%s(%s)' messages " + "(%d new, %d unexpectedly fixed)" % (len(current_messages), + purify_message.GetMessageType(type), type, + len(type_errors), len(type_fixes))) + + if len(type_errors): + strs = [current_hashes[x].NormalizedStr(verbose=True) + for x in type_errors] + logging.error("%d new '%s(%s)' errors found\n%s" % (len(type_errors), + purify_message.GetMessageType(type), type, + '\n'.join(strs))) + strs = [current_hashes[x].NormalizedStr() for x in type_errors] + self.SaveLatestStrings(strs, type, "_NEW") + errors += len(type_errors) + + if len(type_fixes): + # we don't have access to the original message, so all we can do is log + # the non-verbose normalized text + logging.warning("%d new '%s(%s)' unexpected fixes found\n%s" % ( + len(type_fixes), purify_message.GetMessageType(type), + type, '\n'.join(type_fixes))) + self.SaveLatestStrings(type_fixes, type, "_FIXED") + fixes += len(type_fixes) + if len(current_messages) == 0: + logging.warning("all errors fixed in %s" % baseline_file) + + if len(type_fixes) or len(type_errors): + strs = [baseline_hashes[x] for x in new_baseline] + self.SaveLatestStrings(strs, type, "_BASELINE") + + if current_list: + self._SaveLatestGroupSummary(current_list) + + if errors: + logging.error("%d total new errors found" % errors) + return -1 + else: + logging.info("no new errors found - yay!") + if fixes: + logging.warning("%d total errors unexpectedly fixed" % fixes) + # magic return code to turn the builder orange (via ReturnCodeCommand) + return -88 + return 0 + + +# The following code is here for testing and development purposes. + +def _main(): + retcode = 0 + + parser = optparse.OptionParser("usage: %prog [options] <files to analyze>") + parser.add_option("-b", "--baseline", action="store_true", default=False, + help="save output to baseline files") + parser.add_option("-m", "--memory_in_use", + action="store_true", default=False, + help="print memory in use summary") + parser.add_option("", "--validate", + action="store_true", default=False, + help="validate results vs. baseline") + parser.add_option("-e", "--echo_to_stdout", + action="store_true", default=False, + help="echo purify output to standard output") + parser.add_option("", "--source_dir", + help="path to top of source tree for this build" + "(used to normalize source paths in output)") + parser.add_option("", "--byte_filter", default=16384, + help="prune the tree below this number of bytes") + parser.add_option("-n", "--name", + help="name of the test being run " + "(used for output filenames)") + parser.add_option("", "--data_dir", + help="path to where purify data files live") + parser.add_option("", "--bug_report", default=False, + action="store_true", + help="print output as an attempted summary of bugs") + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="verbose output - enable debug log messages") + + (options, args) = parser.parse_args() + if not len(args) >= 1: + parser.error("no filename specified") + filenames = args + + if options.verbose: + google.logging_utils.config_root(level=logging.DEBUG) + else: + google.logging_utils.config_root(level=logging.INFO) + pa = PurifyAnalyze(filenames, options.echo_to_stdout, options.name, + options.source_dir, options.data_dir) + execute_crash = not pa.ReadFile() + if options.bug_report: + pa.PrintBugReport() + pa.PrintSummary(False) + elif options.memory_in_use: + pa.PrintMemoryInUse(int(options.byte_filter)) + elif execute_crash: + retcode = -1 + logging.error("Fatal error during test execution. Analysis skipped.") + elif options.validate: + if pa.CompareResults() != 0: + retcode = -1 + script_dir = google.path_utils.ScriptDir() + latest_dir = os.path.join(script_dir, "latest") + pa.SaveResults(latest_dir) + pa.PrintSummary() + elif options.baseline: + if not pa.SaveResults(verbose=True): + retcode = -1 + pa.PrintSummary(False) + else: + pa.PrintSummary(False) + + sys.exit(retcode) + +if __name__ == "__main__": + _main() + diff --git a/tools/purify/purify_coverage.py b/tools/purify/purify_coverage.py new file mode 100644 index 0000000..0828e22 --- /dev/null +++ b/tools/purify/purify_coverage.py @@ -0,0 +1,111 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# purify_coverage.py + +import logging +import optparse +import os +import re +import sys + +import google.path_utils + +# local modules +import common +import purify_analyze +import purify_message + + +class PurifyCoverage(common.Rational): + def __init__(self): + common.Rational.__init__(self) + script_dir = google.path_utils.ScriptDir() + self._latest_dir = os.path.join(script_dir, "latest") + + def CreateOptionParser(self): + common.Rational.CreateOptionParser(self) + self._parser.description = __doc__ + self._parser.add_option("-n", "--name", + dest="name", default=None, + help="name of the test being run " + "(used for output filenames)") + self._parser.add_option("", "--source_dir", + help="path to top of source tree for this build" + "(used to normalize source paths in baseline)") + + def ParseArgv(self): + if common.Rational.ParseArgv(self): + self._name = self._options.name + if not self._name: + self._name = os.path.basename(self._exe) + # _out_file can be set in common.Rational.ParseArgv + if not self._out_file: + self._out_file = os.path.join(self._latest_dir, + "%s_coverage.txt" % (self._name)) + self._source_dir = self._options.source_dir + return True + return False + + def _PurifyCommand(self): + cmd = [common.PURIFYW_PATH, "/CacheDir=" + self._cache_dir, + "/ShowInstrumentationProgress=no", "/ShowLoadLibraryProgress=no", + "/AllocCallStackLength=30", "/Coverage", + "/CoverageDefaultInstrumentationType=line"] + return cmd + + def Instrument(self): + cmd = self._PurifyCommand() + # /Run=no means instrument only + cmd.append("/Run=no") + cmd.append(os.path.abspath(self._exe)) + return common.Rational.Instrument(self, cmd) + + def Execute(self): + cmd = self._PurifyCommand() + cmd.append("/SaveTextData=" + self._out_file) + # TODO(erikkay): should we also do /SaveMergeTextData? + return common.Rational.Execute(self, cmd) + + def Analyze(self): + if not os.path.isfile(self._out_file): + logging.info("no output file %s" % self._out_file) + return -1 + # TODO(erikkay): parse the output into a form we could use on the buildbots + return 0 + +if __name__ == "__main__": + rational = PurifyCoverage() + if rational.Run(): + retcode = 0 + else: + retcode = -1 + sys.exit(retcode) + diff --git a/tools/purify/purify_inuse.py b/tools/purify/purify_inuse.py new file mode 100644 index 0000000..d527189 --- /dev/null +++ b/tools/purify/purify_inuse.py @@ -0,0 +1,116 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# purify_inuse.py + +import logging +import optparse +import os +import re +import sys + +import google.path_utils + +# local modules +import common +import purify_analyze +import purify_message + + +class PurifyInUse(common.Rational): + def __init__(self): + common.Rational.__init__(self) + script_dir = google.path_utils.ScriptDir() + self._latest_dir = os.path.join(script_dir, "latest") + + def CreateOptionParser(self): + common.Rational.CreateOptionParser(self) + self._parser.description = __doc__ + self._parser.add_option("-n", "--name", + dest="name", default=None, + help="name of the test being run " + "(used for output filenames)") + self._parser.add_option("", "--source_dir", + help="path to top of source tree for this build" + "(used to normalize source paths in baseline)") + self._parser.add_option("", "--byte_filter", default=16384, + help="prune the tree below this number of bytes") + + def ParseArgv(self): + if common.Rational.ParseArgv(self): + self._name = self._options.name + if not self._name: + self._name = os.path.basename(self._exe) + # _out_file can be set in common.Rational.ParseArgv + if not self._out_file: + self._out_file = os.path.join(self._latest_dir, "%s.txt" % (self._name)) + self._source_dir = self._options.source_dir + self._byte_filter = int(self._options.byte_filter) + return True + return False + + def _PurifyCommand(self): + cmd = [common.PURIFYW_PATH, "/CacheDir=" + self._cache_dir, + "/ShowInstrumentationProgress=no", "/ShowLoadLibraryProgress=no", + "/AllocCallStackLength=30", "/ErrorCallStackLength=30", + "/LeaksAtExit=no", "/InUseAtExit=yes"] + return cmd + + def Instrument(self): + cmd = self._PurifyCommand() + # /Run=no means instrument only + cmd.append("/Run=no") + cmd.append(os.path.abspath(self._exe)) + return common.Rational.Instrument(self, cmd) + + def Execute(self): + cmd = self._PurifyCommand() + cmd.append("/SaveTextData=" + self._out_file) + return common.Rational.Execute(self, cmd) + + def Analyze(self): + if not os.path.isfile(self._out_file): + logging.info("no output file %s" % self._out_file) + return -1 + pa = purify_analyze.PurifyAnalyze(self._out_file, False, + self._name, self._source_dir) + if not pa.ReadFile(): + logging.warning("inuse summary suspect due to fatal error during run") + pa.PrintMemoryInUse(byte_filter=self._byte_filter) + return 0 + +if __name__ == "__main__": + rational = PurifyInUse() + if rational.Run(): + retcode = 0 + else: + retcode = -1 + sys.exit(retcode) + diff --git a/tools/purify/purify_message.py b/tools/purify/purify_message.py new file mode 100644 index 0000000..d093461 --- /dev/null +++ b/tools/purify/purify_message.py @@ -0,0 +1,634 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# purify_message.py + +''' Utility objects and functions to parse and unique Purify messages ''' + +import cStringIO +import logging +import re +import sys + +import google.logging_utils + +# used to represent one or more elided frames +ELIDE = "..." +# used to represent stack truncation at a known entry point +TRUNCATE = "^^^" +# a file that's outside of our source directory +EXTERNAL_FILE = "EXTERNAL_FILE" + +# mapping of purify message types to descriptions +message_type = { + "ABR": "Array Bounds Read", + "ABW": "Array Bounds Write", + "ABWL": "Array Bounds Write (late detect)", + "BSR": "Beyond Stack Read", + "BSW": "Beyond Stack Write", + "COM": "COM API/Interface Failure", + "EXC": "Continued Exception", + "EXH": "Handled Exception", + "EXI": "Ignored Exception", + "EXU": "Unhandled Exception", + "FFM": "Freeing Freed Memory", + "FIM": "Freeing Invalid Memory", + "FMM": "Freeing Mismatched Memory", + "FMR": "Free Memory Read", + "FMW": "Free Memory Write", + "FMWL": "Free Memory Write (late detect)", + "HAN": "Invalid Handle", + "HIU": "Handle In Use", + "ILK": "COM Interface Leak", + "IPR": "Invalid Pointer Read", + "IPW": "Invalid Pointer Write", + "MAF": "Memory Allocation Failure", + "MIU": "Memory In Use", + "MLK": "Memory Leak", + "MPK": "Potential Memory Leak", + "NPR": "Null Pointer Read", + "NPW": "Null Pointer Write", + "PAR": "Bad Parameter", + "UMC": "Uninitialized Memory Copy", + "UMR": "Uninitialized Memory Read", +} + +# a magic message type which is not enumerated with the normal message type dict +FATAL = "FATAL" + +def GetMessageType(key): + if key in message_type: + return message_type[key] + elif key == FATAL: + return key + logging.warn("unknown message type %s" % key) + return "UNKNOWN" + +# currently unused, but here for documentation purposes +message_severity = { + "I": "Informational", + "E": "Error", + "W": "Warning", + "O": "Internal Purify Error", +} + + +class Stack: + ''' A normalized Purify Stack. The stack is constructed by adding one line + at a time from a stack in a Purify text file via AddLine. + Supports cmp and hash so that stacks which normalize the same can be sorted + and uniqued. + The original stack contents are preserved so that it's possible to drill + down into the full details if necessary. ''' + + # The top of the source tree. This is stripped from the filename as part + # of normalization. + source_dir = "" + + @classmethod + def SetSourceDir(cls, dir): + # normalize the dir + cls.source_dir = dir.replace("\\", "/").lower() + logging.debug("Stack.source_dir = %s" % cls.source_dir) + + # a line in a stack trace + pat_stack_line = re.compile('(.*)\[(\w:)?([^\:\s]*)(:\d+)?(\s+.*)?]') + + # Known stack entry points that allow us to truncate the rest of the stack + # below that point. + pat_known_entries = ( + re.compile('RunnableMethod::Run\(void\)'), + re.compile('ChromeMain'), + re.compile('BrowserMain'), + re.compile('wWinMain'), + re.compile('TimerManager::ProcessPendingTimer\(void\)'), + re.compile('RunnableMethod::RunableMethod\(.*\)'), + re.compile('RenderViewHost::OnMessageReceived\(Message::IPC const&\)'), + re.compile('testing::Test::Run\(void\)'), + re.compile('testing::TestInfoImpl::Run\(void\)'), + re.compile('Thread::ThreadFunc\\(void \*\)'), + re.compile('TimerTask::Run\(void\)'), + re.compile('MessageLoop::RunTask\(Task \*\)'), + re.compile('.DispatchToMethod\@.*'), + ) + + # if functions match the following, elide them from the stack + pat_func_elide = (re.compile('^std::'), re.compile('^new\(')) + # if files match the following, elide them from the stack + pat_file_elide = (re.compile('.*platformsdk_vista.*'), + re.compile('.*.(dll|DLL)$'), + # bug 1069902 + re.compile('webkit/pending/wtf/fastmalloc\.h'), + # When we leak sqlite stuff, we leak a lot, and the stacks + # are all over the place. For now, let's assume that + # sqlite itself is leak free and focus on our calling code. + re.compile('chrome/third_party/sqlite/.*'), + ) + + pat_unit_test = re.compile('^([a-zA-Z0-9]+)_(\w+)_Test::.*') + + def __init__(self, title): + self._title = title.lstrip() + self._stack = [] + self._orig = "" + # are we currently in an eliding block + self._eliding = False + # have we truncated the stack? + self._truncated = False + # is the stack made up completely of external code? (i.e. elided) + self._all_external = True + # a logical group that this stack belongs to + self._group = None + # top stack line (preserved even if elided) + self._top_stack_line = None + + def GetLines(self): + return self._stack + + def GetTopStackLine(self): + return self._top_stack_line + + def GetTopVisibleStackLine(self): + for line in self._stack: + if line['function']: + return line + return {} + + def GetGroup(self): + '''A logical grouping for this stack, allowing related stacks to be grouped + together. Subgroups within a group are separated by ".". + (e.g. group.subgroup.subsubgroup) + ''' + return self._group; + + def _ComputeStackLine(self, line): + line = line.lstrip() + m = Stack.pat_stack_line.match(line) + if m: + func = m.group(1).rstrip() + func = self._DemangleSymbol(func) + func = self._DetemplatizeSymbol(func) + if m.group(2): + file = m.group(2) + m.group(3) + else: + file = m.group(3) + # paths are normalized to use / and be lower case + file = file.replace("\\", "/").lower() + if not file.startswith(Stack.source_dir): + file = EXTERNAL_FILE + else: + file = file[len(Stack.source_dir):] + # trim leading / if present + if file[0] == "/": + file = file[1:] + loc = m.group(4) + if loc: + loc = int(loc[1:]) + else: + loc = 0 + return {'function': func, 'file': file, 'line_number': loc} + return None + + def _ShouldElide(self, stack_line): + func = stack_line['function'] + file = stack_line['file'] + # elide certain common functions from the stack such as the STL + for pat in Stack.pat_func_elide: + if pat.match(func): + logging.debug("eliding due to func pat match: %s" % func) + return True + if file == EXTERNAL_FILE: + # if it's not in our source tree, then elide + logging.debug("eliding due to external file: %s" % file) + return True + # elide certain common file sources from the stack, usually this + # involves system libraries + for pat in Stack.pat_file_elide: + if pat.match(file): + logging.debug("eliding due to file pat match: %s" % file) + return True + + return False + + def AddLine(self, line): + ''' Add one line from a stack in a Purify text file. Lines must be + added in order (top down). Lines are added to two internal structures: + an original string copy and an array of normalized lines, split into + (function, file, line number). + Stack normalization does several things: + * elides sections of the stack that are in external code + * truncates the stack at so called "known entry points" + * removes template type information from symbols + Returns False if the line was elided or otherwise omitted. + ''' + self._orig += line + "\n" + stack_line = self._ComputeStackLine(line) + if stack_line: + if not self._top_stack_line: + self._top_stack_line = stack_line + # Unit test entry points are good groupings. Even if we already have a + # group set, a later unit-test stack line will override. + # Note that we also do this even if the stack has already been truncated + # since this is useful information. + # TODO(erikkay): Maybe in this case, the truncation should be overridden? + test_match = Stack.pat_unit_test.match(stack_line["function"]) + if test_match: + self._group = test_match.group(1) + "." + test_match.group(2) + + if self._truncated: + return False + + if self._ShouldElide(stack_line): + if not self._eliding: + self._eliding = True + self._stack.append({'function': "", 'file': ELIDE, 'line_number': 0}) + return False + else: + self._stack.append(stack_line) + self._eliding = False + self._all_external = False + + # when we reach one of the known common stack entry points, truncate + # the stack to avoid printing overly redundant information + if len(self._stack) > 1: + for f in Stack.pat_known_entries: + if f.match(stack_line["function"]): + if not self._group: + # we're at the end of the stack, so use the path to the file + # as the group if we don't already have one + # This won't be incredibly reliable, but might still be useful. + prev = self._stack[-2] + if prev['file']: + self._group = '.'.join(prev['file'].split('/')[:-1]) + self._stack.append({'function': "", 'file': TRUNCATE, + 'line_number': 0}) + self._truncated = True + return False + return True + else: + # skip these lines + logging.debug(">>>" + line) + return False + + def _DemangleSymbol(self, symbol): + # TODO(erikkay) - I'm not sure why Purify prepends an address on the + # front of some of these as if it were a namespace (?A<addr>::). From an + # analysis standpoint, it seems meaningless and can change from machine to + # machine, so it's best if it's thrown away + if symbol.startswith("?A0x"): + skipto = symbol.find("::") + if skipto >= 0: + symbol = symbol[(skipto+2):] + else: + logging.warn("unable to strip address off of symbol (%s)" % symbol) + # TODO(erikkay) there are more symbols not being properly demangled + # in Purify's output. Some of these look like template-related issues. + return symbol + + def _DetemplatizeSymbol(self, symbol): + ''' remove all of the template arguments and return values from the + symbol, normalizing it, making it more readable, and less precise ''' + ret = "" + nested = 0 + for i in range(len(symbol)): + if nested > 0: + if symbol[i] == '>': + nested -= 1 + elif symbol[i] == '<': + nested += 1 + elif symbol[i] == '<': + nested += 1 + else: + ret += symbol[i] + return ret + + def __hash__(self): + return hash(self.NormalizedStr()) + + def __cmp__(self, other): + if not other: + return 1 + len_self = len(self._stack) + len_other = len(other._stack) + min_len = min(len_self, len_other) + # sort stacks from the bottom up + for i in range(-1, -(min_len + 1), -1): + # compare file, then func, but omit line number + ret = cmp((self._stack[i]['file'], self._stack[i]['function']), + (other._stack[i]['file'], other._stack[i]['function'])) + if ret: + return ret + return cmp(len_self, len_other) + + def NormalizedStr(self, verbose=False): + ''' String version of the normalized stack. See AddLine for normalization + details. ''' + # use cStringIO for more efficient string building + out = cStringIO.StringIO() + for line in self._stack: + out.write(" ") + out.write(line['file']) + if verbose and line['line_number'] > 0: + out.write(":%d" % line['line_number']) + out.write(" ") + out.write(line['function']) + out.write("\n") + ret = out.getvalue() + out.close() + return ret + + def __str__(self): + return self._orig + + +class Message: + '''A normalized message from a Purify text file. Messages all have a + severity, most have a type, and many have an error stack and/or an + allocation stack. + Supports cmp and hash so that messages which normalize the same can be + sorted and uniqued.''' + + pat_count = re.compile('^(.*) \{(\d+) occurrences?\}') + pat_leak = re.compile('(Potential )?[Mm]emory leak of (\d+) bytes? ' + 'from (\d+) blocks? allocated in (.+)') + pat_miu = re.compile('Memory use of (\d+) bytes? ' + '(\((\d+)% initialized\) )?from (\d+) blocks? ' + 'allocated .. (.+)') + # these are headings to different types of stack traces + pat_loc_error = re.compile('\s*(Exception|Error|Call) location') + pat_loc_alloc = re.compile('\s*Allocation location') + pat_loc_free = re.compile('\s*Free location') + pat_loc_free2 = re.compile('\s*Location of free attempt') + + def __init__(self, severity, type, title): + self._severity = severity + self._type = type + self._program = None + self._head = "" + self._loc_alloc = None + self._loc_error = None + self._loc_free = None + self._stack = None + self._count = 1 + self._bytes = 0 + self._blocks = 0 + m = Message.pat_count.match(title) + if m: + self._title = m.group(1) + self._count = int(m.group(2)) + else: + m = Message.pat_leak.match(title) + if m: + self._title = m.group(4) + self._bytes = int(m.group(2)) + self._blocks = int(m.group(3)) + else: + m = Message.pat_miu.match(title) + if m: + self._title = m.group(5) + self._bytes = int(m.group(1)) + self._blocks = int(m.group(4)) + #print "%d/%d - %s" % (self._bytes, self._blocks, title[0:60]) + elif type == "MIU": + logging.error("%s didn't match" % title) + sys.exit(-1) + else: + self._title = title + + def GetAllocStack(self): + return self._loc_alloc + + def GetErrorStack(self): + return self._loc_error + + def GetGroup(self): + '''An attempted logical grouping for this Message computed by the contained + Stack objects. + ''' + group = None + if self._loc_alloc: + group = self._loc_alloc.GetGroup() + if not group and self._loc_error: + group = self._loc_error.GetGroup() + if not group and self._loc_free: + group = self._loc_free.GetGroup() + if not group: + group = "UNKNOWN" + return group + + def AddLine(self, line): + '''Add a line one at a time (in order from the Purify text file) to + build up the message and its associated stacks. ''' + + if Message.pat_loc_error.match(line): + self._stack = Stack(line) + self._loc_error = self._stack + elif Message.pat_loc_alloc.match(line): + self._stack = Stack(line) + self._loc_alloc = self._stack + elif Message.pat_loc_free.match(line) or Message.pat_loc_free2.match(line): + self._stack = Stack(line) + self._loc_free = self._stack + elif self._stack: + if not line.startswith(" "): + logging.debug("*** " + line) + self._stack.AddLine(line) + else: + self._head += line.lstrip() + + def Type(self): + return self._type + + def Program(self): + return self._program + + def SetProgram(self, program): + self._program = program + + def StacksAllExternal(self): + '''Returns True if the stacks it contains are made up completely of + external (elided) symbols''' + return ((not self._loc_error or self._loc_error._all_external) and + (not self._loc_alloc or self._loc_alloc._all_external) and + (not self._loc_free or self._loc_free._all_external)) + + def __hash__(self): + # NOTE: see also _MessageHashesFromFile. If this method changes, then + # _MessageHashesFromFile must be updated to match. + s = "" + if self._loc_error: + s += "Error Location\n" + self._loc_error.NormalizedStr() + if self._loc_alloc: + s += "Alloc Location\n" + self._loc_alloc.NormalizedStr() + if self._loc_free: + s += "Free Location\n" + self._loc_free.NormalizedStr() + return hash(s) + + def NormalizedStr(self, verbose=False): + '''String version of the normalized message. Only includes title + and normalized versions of error and allocation stacks if present. + Example: + Unitialized Memory Read in Foo::Bar() + Error Location + foo/Foo.cc Foo::Bar(void) + foo/main.cc start(void) + foo/main.cc main(void) + Alloc Location + foo/Foo.cc Foo::Foo(void) + foo/main.cc start(void) + foo/main.cc main(void) + ''' + ret = "" + # some of the message types are more verbose than others and we + # don't need to indicate their type + if verbose and self._type not in ["UMR", "IPR", "IPW"]: + ret += GetMessageType(self._type) + ": " + if verbose and self._bytes > 0: + ret += "(%d bytes, %d blocks) " % (self._bytes, self._blocks) + ret += "%s\n" % self._title + if self._loc_error: + ret += "Error Location\n" + self._loc_error.NormalizedStr(verbose) + if self._loc_alloc: + ret += "Alloc Location\n" + self._loc_alloc.NormalizedStr(verbose) + if self._loc_free: + ret += "Free Location\n" + self._loc_free.NormalizedStr(verbose) + return ret + + def __str__(self): + ret = self._title + "\n" + self._head + if self._loc_error: + ret += "Error Location\n" + str(self._loc_error) + if self._loc_alloc: + ret += "Alloc Location\n" + str(self._loc_alloc) + if self._loc_free: + ret += "Free Location\n" + str(self._loc_free) + return ret + + def __cmp__(self, other): + if not other: + return 1 + ret = 0 + if self._loc_error: + ret = cmp(self._loc_error, other._loc_error) + if ret == 0 and self._loc_alloc: + ret = cmp(self._loc_alloc, other._loc_alloc) + if ret == 0 and self._loc_free: + ret = cmp(self._loc_free, other._loc_free) + # since title is often not very interesting, we sort against that last + if ret == 0: + ret = cmp(self._title, other._title) + return ret + + +class MessageList: + '''A collection of Message objects of a given message type.''' + def __init__(self, type): + self._type = type + self._messages = [] + self._unique_messages = None + self._sublists = None + self._bytes = 0 + + def GetType(self): + return self._type + + def BeginNewSublist(self): + '''Some message types are logically grouped into sets of messages which + should not be mixed in the same list. Specifically, Memory In Use (MIU), + Memory Leak (MLK) and Potential Memory Leak (MPK) are generated in a set + all at once, but this generation can happen at multiple distinct times, + either via the Purify UI or through Purify API calls. For example, if + Purify is told to dump a list all memory leaks once, and then a few minutes + later, the two lists will certainly overlap, so they should be kept + in separate lists. + In order to accommodate this, MessageList supports the notion of sublists. + When the caller determines that one list of messages of a type has ended + and a new list has begun, it calls BeginNewSublist() which takes the current + set of messages, puts them into a new MessageList and puts that into the + sublists array. Later, when the caller needs to get at these messages, + GetSublists() should be called. + ''' + if len(self._messages): + # if this is the first list, no need to make a new one + list = MessageList(self._type) + list._messages = self._messages + if not self._sublists: + self._sublists = [list] + else: + self._sublists.append(list) + self._messages = [] + logging.info("total size: %d" % self._bytes) + self._bytes = 0 + + def GetSublists(self): + '''Returns the current list of sublists. If there are currently sublists + and there are any messages that aren't in a sublist, BeginNewSublist() is + called implicitly by this method to force those ungrouped messages into + their own sublist. + ''' + if self._sublists and len(self._sublists) and len(self._messages): + self.BeginNewSublist() + return self._sublists + + def AddMessage(self, msg): + '''Adds a message to this MessageList.''' + # TODO(erikkay): assert if _unique_messages exists + self._messages.append(msg) + self._bytes += msg._bytes + + def AllMessages(self): + '''Returns an array of all Message objects in this MessageList. ''' + # TODO(erikkay): handle case with sublists + return self._messages + + def UniqueMessages(self): + '''Returns an array of the unique normalized Message objects in this + MessageList. + ''' + # the list is lazily computed since we have to create a sorted list, + # which is only valid once all messages have been added + # TODO(erikkay): handle case with sublists + if not self._unique_messages: + self._unique_messages = list(set(self._messages)) + self._unique_messages.sort() + return self._unique_messages + + def UniqueMessageGroups(self): + '''Returns a dictionary mapping Message group names to arrays of uniqued + normalized Message objects in this MessageList. + ''' + unique = self.UniqueMessages() + groups = {} + for msg in unique: + group = msg.GetGroup() + if not group in groups: + groups[group] = [] + groups[group].append(msg) + return groups diff --git a/tools/purify/purify_test.py b/tools/purify/purify_test.py new file mode 100644 index 0000000..c2c1b76 --- /dev/null +++ b/tools/purify/purify_test.py @@ -0,0 +1,249 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# purify_test.py + +'''Runs an exe through Purify and verifies that Purify was +able to successfully instrument and run it. The original purpose was +to be able to identify when a change to our code breaks our ability to Purify +the app. This can happen with seemingly innocuous changes to code due to bugs +in Purify, and is notoriously difficult to track down when it does happen. +Perhaps more importantly in the long run, this can also automate detection of +leaks and other memory bugs. It also may be useful to allow people to run +Purify in a consistent manner without having to worry about broken PATHs, +corrupt instrumentation, or other per-machine flakiness that Purify is +sometimes subject to. +''' + +import glob +import logging +import optparse +import os +import re +import shutil +import sys +import time + +import google.path_utils + +# local modules +import common +import purify_analyze + +class Purify(common.Rational): + def __init__(self): + common.Rational.__init__(self) + self._data_dir = None + + def CreateOptionParser(self): + common.Rational.CreateOptionParser(self) + self._parser.description = __doc__ + self._parser.add_option("-e", "--echo_to_stdout", + dest="echo_to_stdout", action="store_true", default=False, + help="echo purify output to standard output") + self._parser.add_option("-b", "--baseline", + dest="baseline", action="store_true", default=False, + help="create baseline error files") + self._parser.add_option("-n", "--name", + dest="name", default=None, + help="name of the test being run " + "(used for output filenames)") + self._parser.add_option("", "--source_dir", + help="path to top of source tree for this build" + "(used to normalize source paths in baseline)") + self._parser.add_option("", "--exe", + help="The actual exe to instrument which is " + "different than the program being run. This " + "is useful when the exe you want to purify is " + "run by another script or program.") + self._parser.add_option("", "--data_dir", + help="path to where purify data files live") + + def ParseArgv(self): + if common.Rational.ParseArgv(self): + if self._options.exe: + self._exe = self._options.exe; + if not os.path.isfile(self._exe): + logging.error("file doesn't exist " + self._exe) + return False + self._exe_dir = common.FixPath(os.path.abspath(os.path.dirname(self._exe))) + self._echo_to_stdout = self._options.echo_to_stdout + self._baseline = self._options.baseline + self._name = self._options.name + if not self._name: + self._name = os.path.basename(self._exe) + # _out_file can be set in common.Rational.ParseArgv + if not self._out_file: + self._out_file = os.path.join(self._latest_dir, "%s.txt" % self._name) + self._source_dir = self._options.source_dir + self._data_dir = self._options.data_dir + if not self._data_dir: + self._data_dir = os.path.join(script_dir, "data") + return True + return False + + def _PurifyCommand(self): + cmd = [common.PURIFY_PATH, "/CacheDir=" + self._cache_dir] + return cmd + + def Setup(self): + script_dir = google.path_utils.ScriptDir() + self._latest_dir = os.path.join(script_dir, "latest") + if common.Rational.Setup(self): + pft_file = os.path.join(script_dir, "data", "filters.pft") + shutil.copyfile(pft_file, self._exe.replace(".exe", "_exe.pft")) + string_list = [ + "[Purify]", + "option -cache-dir=\"%s\"" % (self._cache_dir), + "option -save-text-data=\"%s\"" % (common.FixPath(self._out_file)), + "option -alloc-call-stack-length=30", + "option -error-call-stack-length=30", + "option -free-call-stack-length=30", + "option -leaks-at-exit=yes", + "option -in-use-at-exit=no" + ] + ini_file = self._exe.replace(".exe", "_pure.ini") + if os.path.isfile(ini_file): + ini_file_orig = ini_file + ".Original" + if not os.path.isfile(ini_file_orig): + os.rename(ini_file, ini_file_orig) + try: + f = open(ini_file, "w+") + f.write('\n'.join(string_list)) + except IOError, (errno, strerror): + logging.error("error writing to file %s (%d, %s)" % ini_file, errno, + strerror) + return False + if f: + f.close() + return True + return False + + def Instrument(self): + if not os.path.isfile(self._exe): + logging.error("file doesn't exist " + self._exe) + return False + cmd = self._PurifyCommand() + # /Run=no means instrument only, /Replace=yes means replace the exe in place + cmd.extend(["/Run=no", "/Replace=yes"]) + cmd.append(os.path.abspath(self._exe)) + return common.Rational.Instrument(self, cmd) + + def _ExistingOutputFiles(self): + pat_multi = re.compile('(.*)%[0-9]+d(.*)') + m = pat_multi.match(self._out_file) + if m: + g = m.group(1) + '[0-9]*' + m.group(2) + out = glob.glob(g) + if os.path.isfile(m.group(1) + m.group(2)): + out.append(m.group(1) + m.group(2)) + return out + if not os.path.isfile(self._out_file): + return [] + return [self._out_file] + + def Execute(self): + # delete the old file(s) to make sure that this run actually generated + # something new + out_files = self._ExistingOutputFiles() + for f in out_files: + os.remove(f) + common.Rational.Execute(self, []) + # Unfortunately, when we replace the exe, there's no way here to figure out + # if purify is actually going to output a file or if the exe just crashed + # badly. The reason is that it takes some small amount of time for purify + # to dump out the file. + count = 60 + while count > 0 and not os.path.isfile(self._out_file): + time.sleep(0.2) + count -= 1 + # Always return true, even if Execute failed - we'll depend on Analyze to + # determine if the run was valid. + return True + + def Analyze(self): + out_files = self._ExistingOutputFiles() + if not len(out_files): + logging.info("no output files matching %s" % self._out_file) + return -1 + pa = purify_analyze.PurifyAnalyze(out_files, self._echo_to_stdout, + self._name, self._source_dir, + self._data_dir) + if not pa.ReadFile(): + # even though there was a fatal error during Purify, it's still useful + # to see the normalized output + pa.PrintSummary() + if self._baseline: + logging.warning("baseline not generated due to fatal error") + else: + logging.warning("baseline comparison skipped due to fatal error") + return -1 + if self._baseline: + pa.PrintSummary(False) + if pa.SaveResults(): + return 0 + return -1 + else: + retcode = pa.CompareResults() + if retcode != 0: + pa.SaveResults(self._latest_dir) + pa.PrintSummary() + # with more than one output file, it's also important to emit the bug + # report which includes info on the arguments that generated each stack + if len(out_files) > 1: + pa.PrintBugReport() + return retcode + + def Cleanup(self): + common.Rational.Cleanup(self); + cmd = self._PurifyCommand() + # undo the /Replace=yes that was done in Instrument(), which means to + # remove the instrumented exe, and then rename exe.Original back to exe. + cmd.append("/UndoReplace") + cmd.append(os.path.abspath(self._exe)) + common.RunSubprocess(cmd, self._timeout, detach=True) + # if we overwrote an existing ini file, restore it + ini_file = self._exe.replace(".exe", "_pure.ini") + if os.path.isfile(ini_file): + os.remove(ini_file) + ini_file_orig = ini_file + ".Original" + if os.path.isfile(ini_file_orig): + os.rename(ini_file_orig, ini_file) + # remove the pft file we wrote out + pft_file = self._exe.replace(".exe", "_exe.pft") + if os.path.isfile(pft_file): + os.remove(pft_file) + + +if __name__ == "__main__": + rational = Purify() + retcode = rational.Run() + sys.exit(retcode) + diff --git a/tools/purify/quantify_test.py b/tools/purify/quantify_test.py new file mode 100644 index 0000000..054af5d --- /dev/null +++ b/tools/purify/quantify_test.py @@ -0,0 +1,85 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# quantify_test.py + +'''Runs an app through Quantify and verifies that Quantify was able to +successfully instrument and run it. The original purpose was to allow people +to run Quantify in a consistent manner without having to worry about broken +PATHs, corrupt instrumentation, or other per-machine flakiness that Quantify is +sometimes subject to. Unlike purify_test, the output from quantify_test is +a binary file, which is much more useful in manual analysis. As such, this +tool is not particularly interesting for automated analysis yet. +''' + +import os +import sys + +# local modules +import common + +class Quantify(common.Rational): + def __init__(self): + common.Rational.__init__(self) + + def CreateOptionParser(self): + common.Rational.CreateOptionParser(self) + self._parser.description = __doc__ + + def ParseArgv(self): + if common.Rational.ParseArgv(self): + if not self._out_file: + self._out_file = os.path.join(self._cache_dir, + "%s.qfy" % (os.path.basename(self._exe))) + return True + return False + + def Instrument(self): + proc = [common.QUANTIFYE_PATH, "-quantify", + '-quantify_home="' + common.PPLUS_PATH + '"' , + "/CacheDir=" + self._cache_dir, + "-first-search-dir=" + self._exe_dir, self._exe] + return common.Rational.Instrument(self, proc) + + def Execute(self): + # TODO(erikkay): add an option to also do /SaveTextData and add an + # Analyze method for automated analysis of that data. + proc = [common.QUANTIFYW_PATH, "/CacheDir=" + self._cache_dir, + "/ShowInstrumentationProgress=no", "/ShowLoadLibraryProgress=no", + "/SaveData=" + self._out_file] + return common.Rational.Execute(self, proc) + +if __name__ == "__main__": + retcode = -1 + rational = Quantify() + if rational.Run(): + retcode = 0 + sys.exit(retcode) + diff --git a/tools/python/google/__init__.py b/tools/python/google/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tools/python/google/__init__.py @@ -0,0 +1 @@ + diff --git a/tools/python/google/gethash_timer.py b/tools/python/google/gethash_timer.py new file mode 100644 index 0000000..b36a5df --- /dev/null +++ b/tools/python/google/gethash_timer.py @@ -0,0 +1,168 @@ +#!/usr/bin/python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Issue a series of GetHash requests to the SafeBrowsing servers and measure the +# response times. +# +# Usage: +# +# $ ./gethash_timer.py --period=600 --samples=20 --output=resp.csv +# +# --period (or -p): The amount of time (in seconds) to wait between GetHash +# requests. Using a value of more than 300 (5 minutes) to +# include the effect of DNS. +# +# --samples (or -s): The number of requests to issue. If this parameter is not +# specified, the test will run indefinitely. +# +# --output (or -o): The path to a file where the output will be written in +# CSV format: sample_number,response_code,elapsed_time_ms + +import getopt +import httplib +import sys +import time + +_GETHASH_HOST = 'safebrowsing.clients.google.com' +_GETHASH_REQUEST = '/safebrowsing/gethash?client=googleclient&appver=1.0&pver=2.1' + +# Global logging file handle. +g_file_handle = None + + +def IssueGetHash(prefix): + '''Issue one GetHash request to the safebrowsing servers. + Args: + prefix: A 4 byte value to look up on the server. + Returns: + The HTTP response code for the GetHash request. + ''' + body = '4:4\n' + prefix + h = httplib.HTTPConnection(_GETHASH_HOST) + h.putrequest('POST', _GETHASH_REQUEST) + h.putheader('content-length', str(len(body))) + h.endheaders() + h.send(body) + response_code = h.getresponse().status + h.close() + return response_code + + +def TimedGetHash(prefix): + '''Measure the amount of time it takes to receive a GetHash response. + Args: + prefix: A 4 byte value to look up on the the server. + Returns: + A tuple of HTTP resonse code and the response time (in milliseconds). + ''' + start = time.time() + response_code = IssueGetHash(prefix) + return response_code, (time.time() - start) * 1000 + + +def RunTimedGetHash(period, samples=None): + '''Runs an experiment to measure the amount of time it takes to receive + multiple responses from the GetHash servers. + + Args: + period: A floating point value that indicates (in seconds) the delay + between requests. + samples: An integer value indicating the number of requests to make. + If 'None', the test continues indefinitely. + Returns: + None. + ''' + global g_file_handle + prefix = '\x50\x61\x75\x6c' + sample_count = 1 + while True: + response_code, elapsed_time = TimedGetHash(prefix) + LogResponse(sample_count, response_code, elapsed_time) + sample_count += 1 + if samples is not None and sample_count == samples: + break + time.sleep(period) + + +def LogResponse(sample_count, response_code, elapsed_time): + '''Output the response for one GetHash query. + Args: + sample_count: The current sample number. + response_code: The HTTP response code for the GetHash request. + elapsed_time: The round-trip time (in milliseconds) for the + GetHash request. + Returns: + None. + ''' + global g_file_handle + output_list = (sample_count, response_code, elapsed_time) + print 'Request: %d, status: %d, elapsed time: %f ms' % output_list + if g_file_handle is not None: + g_file_handle.write(('%d,%d,%f' % output_list) + '\n') + g_file_handle.flush() + + +def SetupOutputFile(file_name): + '''Open a file for logging results. + Args: + file_name: A path to a file to store the output. + Returns: + None. + ''' + global g_file_handle + g_file_handle = open(file_name, 'w') + + +if __name__ == '__main__': + period = 10 + samples = None + + options, args = getopt.getopt(sys.argv[1:], + 's:p:o:', + ['samples=', 'period=', 'output=']) + for option, value in options: + if option == '-s' or option == '--samples': + samples = int(value) + elif option == '-p' or option == '--period': + period = float(value) + elif option == '-o' or option == '--output': + file_name = value + else: + print 'Bad option: %s' % option + sys.exit(1) + try: + print 'Starting Timed GetHash ----------' + SetupOutputFile(file_name) + RunTimedGetHash(period, samples) + except KeyboardInterrupt: + pass + + print 'Timed GetHash complete ----------' + g_file_handle.close() diff --git a/tools/python/google/httpd_config/httpd.conf b/tools/python/google/httpd_config/httpd.conf new file mode 100644 index 0000000..fc7a498 --- /dev/null +++ b/tools/python/google/httpd_config/httpd.conf @@ -0,0 +1,734 @@ +## +## httpd.conf -- Apache HTTP server configuration file +## + +# +# Based upon the NCSA server configuration files originally by Rob McCool. +# +# This is the main Apache server configuration file. It contains the +# configuration directives that give the server its instructions. +# See <URL:http://httpd.apache.org/docs/> for detailed information about +# the directives. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# After this file is processed, the server will look for and process +# /private/etc/httpd/srm.conf and then /private/etc/httpd/access.conf +# unless you have overridden these with ResourceConfig and/or +# AccessConfig directives here. +# +# The configuration directives are grouped into three basic sections: +# 1. Directives that control the operation of the Apache server process as a +# whole (the 'global environment'). +# 2. Directives that define the parameters of the 'main' or 'default' server, +# which responds to requests that aren't handled by a virtual host. +# These directives also provide default values for the settings +# of all virtual hosts. +# 3. Settings for virtual hosts, which allow Web requests to be sent to +# different IP addresses or hostnames and have them handled by the +# same Apache server process. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so "logs/foo.log" +# with ServerRoot set to "/usr/local/apache" will be interpreted by the +# server as "/usr/local/apache/logs/foo.log". +# + +### Section 1: Global Environment +# +# The directives in this section affect the overall operation of Apache, +# such as the number of concurrent requests it can handle or where it +# can find its configuration files. +# + +# +# ServerType is either inetd, or standalone. Inetd mode is only supported on +# Unix platforms. +# +ServerType standalone + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# NOTE! If you intend to place this on an NFS (or otherwise network) +# mounted filesystem then please read the LockFile documentation +# (available at <URL:http://www.apache.org/docs/mod/core.html#lockfile>); +# you will save yourself a lot of trouble. +# +#ServerRoot "/usr" + +# +# The LockFile directive sets the path to the lockfile used when Apache +# is compiled with either USE_FCNTL_SERIALIZED_ACCEPT or +# USE_FLOCK_SERIALIZED_ACCEPT. This directive should normally be left at +# its default value. The main reason for changing it is if the logs +# directory is NFS mounted, since the lockfile MUST BE STORED ON A LOCAL +# DISK. The PID of the main server process is automatically appended to +# the filename. +# +#LockFile "/private/var/run/httpd.lock" + +# +# PidFile: The file in which the server should record its process +# identification number when it starts. +# +PidFile "/tmp/WebKit/httpd.pid" + +# +# ScoreBoardFile: File used to store internal server process information. +# Not all architectures require this. But if yours does (you'll know because +# this file will be created when you run Apache) then you *must* ensure that +# no two invocations of Apache share the same scoreboard file. +# +ScoreBoardFile "/tmp/WebKit/httpd.scoreboard" + +# +# In the standard configuration, the server will process httpd.conf (this +# file, specified by the -f command line option), srm.conf, and access.conf +# in that order. The latter two files are now distributed empty, as it is +# recommended that all directives be kept in a single file for simplicity. +# The commented-out values below are the built-in defaults. You can have the +# server ignore these files altogether by using "/dev/null" (for Unix) or +# "nul" (for Win32) for the arguments to the directives. +# +ResourceConfig /dev/null +AccessConfig /dev/null + +# +# Timeout: The number of seconds before receives and sends time out. +# +Timeout 300 + +# +# KeepAlive: Whether or not to allow persistent connections (more than +# one request per connection). Set to "Off" to deactivate. +# +KeepAlive On + +# +# MaxKeepAliveRequests: The maximum number of requests to allow +# during a persistent connection. Set to 0 to allow an unlimited amount. +# We recommend you leave this number high, for maximum performance. +# +MaxKeepAliveRequests 100 + +# +# KeepAliveTimeout: Number of seconds to wait for the next request from the +# same client on the same connection. +# +KeepAliveTimeout 15 + +# +# Server-pool size regulation. Rather than making you guess how many +# server processes you need, Apache dynamically adapts to the load it +# sees --- that is, it tries to maintain enough server processes to +# handle the current load, plus a few spare servers to handle transient +# load spikes (e.g., multiple simultaneous requests from a single +# Netscape browser). +# +# It does this by periodically checking how many servers are waiting +# for a request. If there are fewer than MinSpareServers, it creates +# a new spare. If there are more than MaxSpareServers, some of the +# spares die off. The default values are probably OK for most sites. +# +MinSpareServers 1 +MaxSpareServers 5 + +# +# Number of servers to start initially --- should be a reasonable ballpark +# figure. +# +StartServers 1 + +# +# Limit on total number of servers running, i.e., limit on the number +# of clients who can simultaneously connect --- if this limit is ever +# reached, clients will be LOCKED OUT, so it should NOT BE SET TOO LOW. +# It is intended mainly as a brake to keep a runaway server from taking +# the system with it as it spirals down... +# +MaxClients 150 + +# +# MaxRequestsPerChild: the number of requests each child process is +# allowed to process before the child dies. The child will exit so +# as to avoid problems after prolonged use when Apache (and maybe the +# libraries it uses) leak memory or other resources. On most systems, this +# isn't really needed, but a few (such as Solaris) do have notable leaks +# in the libraries. For these platforms, set to something like 10000 +# or so; a setting of 0 means unlimited. +# +# NOTE: This value does not include keepalive requests after the initial +# request per connection. For example, if a child process handles +# an initial request and 10 subsequent "keptalive" requests, it +# would only count as 1 request towards this limit. +# +MaxRequestsPerChild 100000 + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the <VirtualHost> +# directive. +# +# Configured from the httpd command line for WebKit layout tests. +# +Listen 127.0.0.1:8000 +Listen 127.0.0.1:8080 +Listen 127.0.0.1:8081 +Listen 127.0.0.1:9000 +Listen 127.0.0.1:9080 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Please read the file http://httpd.apache.org/docs/dso.html for more +# details about the DSO mechanism and run `httpd -l' for the list of already +# built-in (statically linked and thus always available) modules in your httpd +# binary. +# +# Note: The order in which modules are loaded is important. Don't change +# the order below without expert advice. +# +# Example: +# LoadModule foo_module lib/apache/mod_foo.dll +#LoadModule vhost_alias_module lib/apache/mod_vhost_alias.dll +#LoadModule env_module lib/apache/mod_env.dll +LoadModule config_log_module lib/apache/mod_log_config.dll +#LoadModule mime_magic_module lib/apache/mod_mime_magic.dll +LoadModule mime_module lib/apache/mod_mime.dll +LoadModule negotiation_module lib/apache/mod_negotiation.dll +#LoadModule status_module lib/apache/mod_status.dll +#LoadModule info_module lib/apache/mod_info.dll +LoadModule includes_module lib/apache/mod_include.dll +LoadModule autoindex_module lib/apache/mod_autoindex.dll +#LoadModule dir_module lib/apache/mod_dir.dll +LoadModule cgi_module lib/apache/mod_cgi.dll +LoadModule asis_module lib/apache/mod_asis.dll +LoadModule imap_module lib/apache/mod_imap.dll +LoadModule action_module lib/apache/mod_actions.dll +#LoadModule speling_module lib/apache/mod_speling.dll +#LoadModule userdir_module lib/apache/mod_userdir.dll +LoadModule alias_module lib/apache/mod_alias.dll +LoadModule rewrite_module lib/apache/mod_rewrite.dll +LoadModule access_module lib/apache/mod_access.dll +LoadModule auth_module lib/apache/mod_auth.dll +#LoadModule anon_auth_module lib/apache/mod_auth_anon.dll +#LoadModule dbm_auth_module lib/apache/mod_auth_dbm.dll +#LoadModule digest_module lib/apache/mod_digest.dll +#LoadModule proxy_module lib/apache/libproxy.dll +#LoadModule cern_meta_module lib/apache/mod_cern_meta.dll +#LoadModule expires_module lib/apache/mod_expires.dll +LoadModule headers_module lib/apache/mod_headers.dll +#LoadModule usertrack_module lib/apache/mod_usertrack.dll +#LoadModule log_forensic_module lib/apache/mod_log_forensic.dll +#LoadModule unique_id_module lib/apache/mod_unique_id.dll +#LoadModule setenvif_module lib/apache/mod_setenvif.dll +#LoadModule dav_module lib/apache/libdav.dll +#LoadModule ssl_module lib/apache/libssl.dll +#LoadModule perl_module lib/apache/libperl.dll +LoadModule php4_module lib/apache/libphp4.dll +#LoadModule hfs_apple_module lib/apache/mod_hfs_apple.dll +#LoadModule bonjour_module lib/apache/mod_bonjour.dll + +# Reconstruction of the complete module list from all available modules +# (static and shared ones) to achieve correct module execution order. +# [WHENEVER YOU CHANGE THE LOADMODULE SECTION ABOVE UPDATE THIS, TOO] +ClearModuleList +#AddModule mod_vhost_alias.c +#AddModule mod_env.c +AddModule mod_log_config.c +#AddModule mod_mime_magic.c +AddModule mod_mime.c +AddModule mod_negotiation.c +#AddModule mod_status.c +#AddModule mod_info.c +AddModule mod_include.c +AddModule mod_autoindex.c +#AddModule mod_dir.c +AddModule mod_cgi.c +AddModule mod_asis.c +AddModule mod_imap.c +AddModule mod_actions.c +#AddModule mod_speling.c +#AddModule mod_userdir.c +AddModule mod_alias.c +AddModule mod_rewrite.c +AddModule mod_access.c +AddModule mod_auth.c +#AddModule mod_auth_anon.c +#AddModule mod_auth_dbm.c +#AddModule mod_digest.c +#AddModule mod_proxy.c +#AddModule mod_cern_meta.c +#AddModule mod_expires.c +AddModule mod_headers.c +#AddModule mod_usertrack.c +#AddModule mod_log_forensic.c +#AddModule mod_unique_id.c +AddModule mod_so.c +#AddModule mod_setenvif.c +#AddModule mod_dav.c +#AddModule mod_ssl.c +#AddModule mod_perl.c +AddModule mod_php4.c +#AddModule mod_hfs_apple.c +#AddModule mod_bonjour.c + +### Section 2: 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# <VirtualHost> definition. These values also provide defaults for +# any <VirtualHost> containers you may define later in the file. +# +# All of these directives may appear inside <VirtualHost> containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerName allows you to set a host name which is sent back to clients for +# your server if it's different than the one the program would get (i.e., use +# "www" instead of the host's real name). +# +# Note: You cannot just invent host names and hope they work. The name you +# define here must be a valid DNS name for your host. If you don't understand +# this, ask your network administrator. +# If your host doesn't have a registered DNS name, enter its IP address here. +# You will have to access it by its address (e.g., http://123.45.67.89/) +# anyway, and this will make redirections work in a sensible way. +# +# 127.0.0.1 is the TCP/IP local loop-back address, often named localhost. Your +# machine always knows itself by this address. If you use Apache strictly for +# local testing and development, you may use 127.0.0.1 as the server name. +# +ServerName 127.0.0.1 + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +# Configured from the httpd command line for WebKit layout tests. +#DocumentRoot "/Library/WebServer/Documents" + +# +# Each directory to which Apache has access, can be configured with respect +# to which services and features are allowed and/or disabled in that +# directory (and its subdirectories). +# +<Directory /> +# +# This may also be "None", "All", or any combination of "Indexes", +# "Includes", "FollowSymLinks", "ExecCGI", or "MultiViews". +# +# Note that "MultiViews" must be named *explicitly* --- "Options All" +# doesn't give it to you. +# + Options Indexes FollowSymLinks MultiViews ExecCGI Includes + +# +# This controls which options the .htaccess files in directories can +# override. Can also be "All", or any combination of "Options", "FileInfo", +# "AuthConfig", and "Limit" +# + AllowOverride All + +# +# Controls who can get stuff from this server. +# + Order allow,deny + Allow from all +</Directory> + +# +# AccessFileName: The name of the file to look for in each directory +# for access control information. +# +AccessFileName .htaccess + +# +# The following lines prevent .htaccess files from being viewed by +# Web clients. Since .htaccess files often contain authorization +# information, access is disallowed for security reasons. Comment +# these lines out if you want Web visitors to see the contents of +# .htaccess files. If you change the AccessFileName directive above, +# be sure to make the corresponding changes here. +# +# Also, folks tend to use names such as .htpasswd for password +# files, so this will protect those as well. +# +<Files ~ "^\.([Hh][Tt]|[Dd][Ss]_[Ss])"> + Order allow,deny + Deny from all + Satisfy All +</Files> + +# +# Apple specific filesystem protection. +# + +<Files "rsrc"> + Order allow,deny + Deny from all + Satisfy All +</Files> + +<Directory ~ ".*\.\.namedfork"> + Order allow,deny + Deny from all + Satisfy All +</Directory> + +# +# CacheNegotiatedDocs: By default, Apache sends "Pragma: no-cache" with each +# document that was negotiated on the basis of content. This asks proxy +# servers not to cache the document. Uncommenting the following line disables +# this behavior, and proxies will be allowed to cache the documents. +# +#CacheNegotiatedDocs + +# +# UseCanonicalName: (new for 1.3) With this setting turned on, whenever +# Apache needs to construct a self-referencing URL (a URL that refers back +# to the server the response is coming from) it will use ServerName and +# Port to form a "canonical" name. With this setting off, Apache will +# use the hostname:port that the client supplied, when possible. This +# also affects SERVER_NAME and SERVER_PORT in CGI scripts. +# +UseCanonicalName On + +# +# TypesConfig describes where the mime.types file (or equivalent) is +# to be found. +# +# Configured from the httpd command line for WebKit layout tests. +# +#<IfModule mod_mime.c> +# TypesConfig /private/etc/httpd/mime.types +#</IfModule> + +# +# DefaultType is the default MIME type the server will use for a document +# if it cannot otherwise determine one, such as from filename extensions. +# If your server contains mostly text or HTML documents, "text/plain" is +# a good value. If most of your content is binary, such as applications +# or images, you may want to use "application/octet-stream" instead to +# keep browsers from trying to display binary files as though they are +# text. +# +DefaultType text/plain + +# +# HostnameLookups: Log the names of clients or just their IP addresses +# e.g., www.apache.org (on) or 204.62.129.132 (off). +# The default is off because it'd be overall better for the net if people +# had to knowingly turn this feature on, since enabling it means that +# each client request will result in AT LEAST one lookup request to the +# nameserver. +# +HostnameLookups Off + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a <VirtualHost> +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a <VirtualHost> +# container, that host's errors will be logged there and not here. +# +# Configured from the httpd command line for WebKit layout tests. +#ErrorLog "/tmp/layout-test-results/error_log" + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + +# +# The following directives define some format nicknames for use with +# a CustomLog directive (see below). +# +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%p %h %l %u %t \"%r\" %>s %b" common +LogFormat "%{Referer}i -> %U" referer +LogFormat "%{User-agent}i" agent + +# +# The location and format of the access logfile (Common Logfile Format). +# If you do not define any access logfiles within a <VirtualHost> +# container, they will be logged here. Contrariwise, if you *do* +# define per-<VirtualHost> access logfiles, transactions will be +# logged therein and *not* in this file. +# +# Configured from the httpd command line for WebKit layout tests. +#CustomLog "/tmp/layout-test-results/access_log" common + +# +# If you prefer a single logfile with access, agent, and referer information +# (Combined Logfile Format) you can use the following directive. +# +#CustomLog "/tmp/layout-test-results/access_log" combined + +# +# Optionally add a line containing the server version and virtual host +# name to server-generated pages (error documents, FTP directory listings, +# mod_status and mod_info output etc., but not CGI generated documents). +# Set to "EMail" to also include a mailto: link to the ServerAdmin. +# Set to one of: On | Off | EMail +# +ServerSignature On + +# +# Aliases: Add here as many aliases as you need (with no limit). The format is +# Alias fakename realname +# +<IfModule mod_alias.c> +</IfModule> +# End of aliases. + +# +# Redirect allows you to tell clients about documents which used to exist in +# your server's namespace, but do not anymore. This allows you to tell the +# clients where to look for the relocated document. +# Format: Redirect old-URI new-URL +# + +# +# Document types. +# +<IfModule mod_mime.c> + + # + # AddLanguage allows you to specify the language of a document. You can + # then use content negotiation to give a browser a file in a language + # it can understand. + # + # Note 1: The suffix does not have to be the same as the language + # keyword --- those with documents in Polish (whose net-standard + # language code is pl) may wish to use "AddLanguage pl .po" to + # avoid the ambiguity with the common suffix for perl scripts. + # + # Note 2: The example entries below illustrate that in quite + # some cases the two character 'Language' abbreviation is not + # identical to the two character 'Country' code for its country, + # E.g. 'Danmark/dk' versus 'Danish/da'. + # + # Note 3: In the case of 'ltz' we violate the RFC by using a three char + # specifier. But there is 'work in progress' to fix this and get + # the reference data for rfc1766 cleaned up. + # + # Danish (da) - Dutch (nl) - English (en) - Estonian (ee) + # French (fr) - German (de) - Greek-Modern (el) + # Italian (it) - Korean (kr) - Norwegian (no) - Norwegian Nynorsk (nn) + # Portugese (pt) - Luxembourgeois* (ltz) + # Spanish (es) - Swedish (sv) - Catalan (ca) - Czech(cs) + # Polish (pl) - Brazilian Portuguese (pt-br) - Japanese (ja) + # Russian (ru) + # + AddLanguage da .dk + AddLanguage nl .nl + AddLanguage en .en + AddLanguage et .ee + AddLanguage fr .fr + AddLanguage de .de + AddLanguage el .el + AddLanguage he .he + AddCharset ISO-8859-8 .iso8859-8 + AddLanguage it .it + AddLanguage ja .ja + AddCharset ISO-2022-JP .jis + AddLanguage kr .kr + AddCharset ISO-2022-KR .iso-kr + AddLanguage nn .nn + AddLanguage no .no + AddLanguage pl .po + AddCharset ISO-8859-2 .iso-pl + AddLanguage pt .pt + AddLanguage pt-br .pt-br + AddLanguage ltz .lu + AddLanguage ca .ca + AddLanguage es .es + AddLanguage sv .sv + AddLanguage cs .cz .cs + AddLanguage ru .ru + AddLanguage zh-TW .zh-tw + AddCharset Big5 .Big5 .big5 + AddCharset WINDOWS-1251 .cp-1251 + AddCharset CP866 .cp866 + AddCharset ISO-8859-5 .iso-ru + AddCharset KOI8-R .koi8-r + AddCharset UCS-2 .ucs2 + AddCharset UCS-4 .ucs4 + AddCharset UTF-8 .utf8 + + # LanguagePriority allows you to give precedence to some languages + # in case of a tie during content negotiation. + # + # Just list the languages in decreasing order of preference. We have + # more or less alphabetized them here. You probably want to change this. + # + <IfModule mod_negotiation.c> + LanguagePriority en da nl et fr de el it ja kr no pl pt pt-br ru ltz ca es sv tw + </IfModule> + + # + # AddType allows you to tweak mime.types without actually editing it, or to + # make certain files to be certain types. + # + AddType application/x-tar .tgz + + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # Despite the name similarity, the following Add* directives have nothing + # to do with the FancyIndexing customization directives above. + # + AddEncoding x-compress .Z + AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + #AddType application/x-compress .Z + #AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers", + # actions unrelated to filetype. These can be either built into the server + # or added with the Action command (see below) + # + # If you want to use server side includes, or CGI outside + # ScriptAliased directories, uncomment the following lines. + # + # To use CGI scripts: + # + AddHandler cgi-script .cgi .pl + + # + # To use server-parsed HTML files + # + AddType text/html .shtml + AddHandler server-parsed .shtml + + # + # Uncomment the following line to enable Apache's send-asis HTTP file + # feature + # + AddHandler send-as-is asis + + # + # If you wish to use server-parsed imagemap files, use + # + #AddHandler imap-file map + + # + # To enable type maps, you might want to use + # + #AddHandler type-map var + +</IfModule> +# End of document types. + +# +# Action lets you define media types that will execute a script whenever +# a matching file is called. This eliminates the need for repeated URL +# pathnames for oft-used CGI file processors. +# Format: Action media/type /cgi-script/location +# Format: Action handler-name /cgi-script/location +# + +# +# MetaDir: specifies the name of the directory in which Apache can find +# meta information files. These files contain additional HTTP headers +# to include when sending the document +# +#MetaDir .web + +# +# MetaSuffix: specifies the file name suffix for the file containing the +# meta information. +# +#MetaSuffix .meta + +# +# Customizable error response (Apache style) +# these come in three flavors +# +# 1) plain text +#ErrorDocument 500 "The server made a boo boo. +# n.b. the single leading (") marks it as text, it does not get output +# +# 2) local redirects +#ErrorDocument 404 /missing.html +# to redirect to local URL /missing.html +#ErrorDocument 404 /cgi-bin/missing_handler.pl +# N.B.: You can redirect to a script or a document using server-side-includes. +# +# 3) external redirects +#ErrorDocument 402 http://some.other-server.com/subscription_info.html +# N.B.: Many of the environment variables associated with the original +# request will *not* be available to such a script. + +# +# Proxy Server directives. Uncomment the following lines to +# enable the proxy server: +# +#<IfModule mod_proxy.c> +# ProxyRequests On + +# <Directory proxy:*> +# Order deny,allow +# Deny from all +# Allow from .your-domain.com +# </Directory> + + # + # Enable/disable the handling of HTTP/1.1 "Via:" headers. + # ("Full" adds the server version; "Block" removes all outgoing Via: headers) + # Set to one of: Off | On | Full | Block + # +# ProxyVia On + + # + # To enable the cache as well, edit and uncomment the following lines: + # (no cacheing without CacheRoot) + # +# CacheRoot "/private/var/run/proxy" +# CacheSize 5 +# CacheGcInterval 4 +# CacheMaxExpire 24 +# CacheLastModifiedFactor 0.1 +# CacheDefaultExpire 1 +# NoCache a-domain.com another-domain.edu joes.garage-sale.com + +#</IfModule> +# End of proxy directives. + + +<IfModule mod_php4.c> + # If php is turned on, we repsect .php and .phps files. + AddType application/x-httpd-php .php + AddType application/x-httpd-php-source .phps + + # Since most users will want index.php to work we + # also automatically enable index.php + <IfModule mod_dir.c> + DirectoryIndex index.html index.php + </IfModule> +</IfModule> + +<IfModule mod_rewrite.c> + RewriteEngine On + RewriteCond %{REQUEST_METHOD} ^TRACE + RewriteRule .* - [F] +</IfModule> diff --git a/tools/python/google/httpd_config/httpd2.conf b/tools/python/google/httpd_config/httpd2.conf new file mode 100644 index 0000000..8c96f75 --- /dev/null +++ b/tools/python/google/httpd_config/httpd2.conf @@ -0,0 +1,280 @@ +## httpd2.conf -- Apache 2.x HTTP server configuration file + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the <VirtualHost> +# directive. +# +Listen 127.0.0.1:8000 +Listen 127.0.0.1:8080 +Listen 127.0.0.1:8081 +Listen 127.0.0.1:8443 +Listen 127.0.0.1:9000 +Listen 127.0.0.1:9080 +Listen 127.0.0.1:9443 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Please read the file http://httpd.apache.org/docs/dso.html for more +# details about the DSO mechanism and run `httpd -l' for the list of already +# built-in (statically linked and thus always available) modules in your httpd +# binary. +# +# Note: The order in which modules are loaded is important. Don't change +# the order below without expert advice. +# +#LoadModule authn_file_module lib/apache2/mod_authn_file.so +#LoadModule authn_dbm_module lib/apache2/mod_authn_dbm.so +#LoadModule authn_anon_module lib/apache2/mod_authn_anon.so +#LoadModule authn_dbd_module lib/apache2/mod_authn_dbd.so +#LoadModule authn_default_module lib/apache2/mod_authn_default.so +LoadModule authz_host_module lib/apache2/mod_authz_host.so +#LoadModule authz_groupfile_module lib/apache2/mod_authz_groupfile.so +#LoadModule authz_user_module lib/apache2/mod_authz_user.so +#LoadModule authz_dbm_module lib/apache2/mod_authz_dbm.so +#LoadModule authz_owner_module lib/apache2/mod_authz_owner.so +#LoadModule authz_default_module lib/apache2/mod_authz_default.so +#LoadModule auth_basic_module lib/apache2/mod_auth_basic.so +#LoadModule auth_digest_module lib/apache2/mod_auth_digest.so +#LoadModule dbd_module lib/apache2/mod_dbd.so +#LoadModule dumpio_module lib/apache2/mod_dumpio.so +#LoadModule ext_filter_module lib/apache2/mod_ext_filter.so +LoadModule include_module lib/apache2/mod_include.so +#LoadModule filter_module lib/apache2/mod_filter.so +#LoadModule deflate_module lib/apache2/mod_deflate.so +LoadModule log_config_module lib/apache2/mod_log_config.so +#LoadModule log_forensic_module lib/apache2/mod_log_forensic.so +#LoadModule logio_module lib/apache2/mod_logio.so +#LoadModule env_module lib/apache2/mod_env.so +#LoadModule mime_magic_module lib/apache2/mod_mime_magic.so +#LoadModule cern_meta_module lib/apache2/mod_cern_meta.so +#LoadModule expires_module lib/apache2/mod_expires.so +LoadModule headers_module lib/apache2/mod_headers.so +#LoadModule ident_module lib/apache2/mod_ident.so +#LoadModule usertrack_module lib/apache2/mod_usertrack.so +#LoadModule unique_id_module lib/apache2/mod_unique_id.so +#LoadModule setenvif_module lib/apache2/mod_setenvif.so +#LoadModule version_module lib/apache2/mod_version.so +#LoadModule proxy_module lib/apache2/mod_proxy.so +#LoadModule proxy_connect_module lib/apache2/mod_proxy_connect.so +#LoadModule proxy_ftp_module lib/apache2/mod_proxy_ftp.so +#LoadModule proxy_http_module lib/apache2/mod_proxy_http.so +#LoadModule proxy_ajp_module lib/apache2/mod_proxy_ajp.so +#LoadModule proxy_balancer_module lib/apache2/mod_proxy_balancer.so +LoadModule ssl_module lib/apache2/mod_ssl.so +LoadModule mime_module lib/apache2/mod_mime.so +#LoadModule dav_module lib/apache2/mod_dav.so +#LoadModule status_module lib/apache2/mod_status.so +LoadModule autoindex_module lib/apache2/mod_autoindex.so +LoadModule asis_module lib/apache2/mod_asis.so +#LoadModule info_module lib/apache2/mod_info.so +LoadModule cgi_module lib/apache2/mod_cgi.so +#LoadModule dav_fs_module lib/apache2/mod_dav_fs.so +#LoadModule vhost_alias_module lib/apache2/mod_vhost_alias.so +LoadModule negotiation_module lib/apache2/mod_negotiation.so +#LoadModule dir_module lib/apache2/mod_dir.so +LoadModule imagemap_module lib/apache2/mod_imagemap.so +LoadModule actions_module lib/apache2/mod_actions.so +#LoadModule speling_module lib/apache2/mod_speling.so +#LoadModule userdir_module lib/apache2/mod_userdir.so +LoadModule alias_module lib/apache2/mod_alias.so +LoadModule rewrite_module lib/apache2/mod_rewrite.so +LoadModule php5_module lib/apache2/cygphp5.so + +#LoadModule imap_module lib/apache/mod_imap.dll +#LoadModule access_module lib/apache/mod_access.dll +#LoadModule auth_module lib/apache/mod_auth.dll + + +# +# Each directory to which Apache has access, can be configured with respect +# to which services and features are allowed and/or disabled in that +# directory (and its subdirectories). +# +<Directory /> + Options Indexes FollowSymLinks MultiViews ExecCGI Includes + AllowOverride All + Order allow,deny + Allow from all +</Directory> + + +# +# Apple specific filesystem protection. +# +<Files "rsrc"> + Order allow,deny + Deny from all + Satisfy All +</Files> +<Directory ~ ".*\.\.namedfork"> + Order allow,deny + Deny from all + Satisfy All +</Directory> + + +# +# UseCanonicalName: (new for 1.3) With this setting turned on, whenever +# Apache needs to construct a self-referencing URL (a URL that refers back +# to the server the response is coming from) it will use ServerName and +# Port to form a "canonical" name. With this setting off, Apache will +# use the hostname:port that the client supplied, when possible. This +# also affects SERVER_NAME and SERVER_PORT in CGI scripts. +# +UseCanonicalName On + + +# +# The following directives define some format nicknames for use with +# a CustomLog directive (see below). +# +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%h %l %u %t \"%r\" %>s %b" common +LogFormat "%{Referer}i -> %U" referer +LogFormat "%{User-agent}i" agent + + +# +# Optionally add a line containing the server version and virtual host +# name to server-generated pages (error documents, FTP directory listings, +# mod_status and mod_info output etc., but not CGI generated documents). +# Set to "EMail" to also include a mailto: link to the ServerAdmin. +# Set to one of: On | Off | EMail +# +ServerSignature On + + +# +# Document types. +# +<IfModule mime_module> + + # + # AddLanguage allows you to specify the language of a document. You can + # then use content negotiation to give a browser a file in a language + # it can understand. + # + # Note 1: The suffix does not have to be the same as the language + # keyword --- those with documents in Polish (whose net-standard + # language code is pl) may wish to use "AddLanguage pl .po" to + # avoid the ambiguity with the common suffix for perl scripts. + # + # Note 2: The example entries below illustrate that in quite + # some cases the two character 'Language' abbreviation is not + # identical to the two character 'Country' code for its country, + # E.g. 'Danmark/dk' versus 'Danish/da'. + # + # Note 3: In the case of 'ltz' we violate the RFC by using a three char + # specifier. But there is 'work in progress' to fix this and get + # the reference data for rfc1766 cleaned up. + # + # Danish (da) - Dutch (nl) - English (en) - Estonian (ee) + # French (fr) - German (de) - Greek-Modern (el) + # Italian (it) - Korean (kr) - Norwegian (no) - Norwegian Nynorsk (nn) + # Portugese (pt) - Luxembourgeois* (ltz) + # Spanish (es) - Swedish (sv) - Catalan (ca) - Czech(cs) + # Polish (pl) - Brazilian Portuguese (pt-br) - Japanese (ja) + # Russian (ru) + # + AddLanguage da .dk + AddLanguage nl .nl + AddLanguage en .en + AddLanguage et .ee + AddLanguage fr .fr + AddLanguage de .de + AddLanguage el .el + AddLanguage he .he + AddCharset ISO-8859-8 .iso8859-8 + AddLanguage it .it + AddLanguage ja .ja + AddCharset ISO-2022-JP .jis + AddLanguage kr .kr + AddCharset ISO-2022-KR .iso-kr + AddLanguage nn .nn + AddLanguage no .no + AddLanguage pl .po + AddCharset ISO-8859-2 .iso-pl + AddLanguage pt .pt + AddLanguage pt-br .pt-br + AddLanguage ltz .lu + AddLanguage ca .ca + AddLanguage es .es + AddLanguage sv .sv + AddLanguage cs .cz .cs + AddLanguage ru .ru + AddLanguage zh-TW .zh-tw + AddCharset Big5 .Big5 .big5 + AddCharset WINDOWS-1251 .cp-1251 + AddCharset CP866 .cp866 + AddCharset ISO-8859-5 .iso-ru + AddCharset KOI8-R .koi8-r + AddCharset UCS-2 .ucs2 + AddCharset UCS-4 .ucs4 + AddCharset UTF-8 .utf8 + + # LanguagePriority allows you to give precedence to some languages + # in case of a tie during content negotiation. + # + # Just list the languages in decreasing order of preference. We have + # more or less alphabetized them here. You probably want to change this. + # + <IfModule negotiation_module> + LanguagePriority en da nl et fr de el it ja kr no pl pt pt-br ru ltz ca es sv tw + </IfModule> + + # + # AddType allows you to tweak mime.types without actually editing it, or to + # make certain files to be certain types. + # + AddType application/x-tar .tgz + + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # Despite the name similarity, the following Add* directives have nothing + # to do with the FancyIndexing customization directives above. + # + AddEncoding x-compress .Z + AddEncoding x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers", + # actions unrelated to filetype. These can be either built into the server + # or added with the Action command (see below) + # + # If you want to use server side includes, or CGI outside + # ScriptAliased directories, uncomment the following lines. + # + # To use CGI scripts: + # + AddHandler cgi-script .cgi .pl + + # + # To use server-parsed HTML files + # + AddType text/html .shtml + AddHandler server-parsed .shtml + + # + # Uncomment the following line to enable Apache's send-asis HTTP file + # feature + # + AddHandler send-as-is asis +</IfModule> + + +<IfModule php5_module> + AddType application/x-httpd-php .php + AddType application/x-httpd-php-source .phps +</IfModule> + +<IfModule rewrite_module> + RewriteEngine On + RewriteCond %{REQUEST_METHOD} ^TRACE + RewriteRule .* - [F] +</IfModule> diff --git a/tools/python/google/httpd_config/httpd2.pem b/tools/python/google/httpd_config/httpd2.pem new file mode 100644 index 0000000..1ad5794 --- /dev/null +++ b/tools/python/google/httpd_config/httpd2.pem @@ -0,0 +1,36 @@ +-----BEGIN CERTIFICATE-----
+MIIDazCCAtSgAwIBAgIBEDANBgkqhkiG9w0BAQUFADByMQswCQYDVQQGEwJVUzET
+MBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEUMBIG
+A1UEChMLR29vZ2xlIEluYy4xIDAeBgNVBAMTF2F0dGljdXMuY29ycC5nb29nbGUu
+Y29tMB4XDTA2MDEwMTAwMDAwMFoXDTIwMDEwMTAwMDAwMFowTDELMAkGA1UEBhMC
+VVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFDASBgNVBAoTC0dvb2dsZSBJbmMuMRIw
+EAYDVQQDEwkxMjcuMC4wLjEwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK0I
+45wdTT2/fui9FfNOR17idQF0L/kpn88D1OzF0w0h3TMBN8szpOxAQYT2MTuB17Dt
+ttMiwzLuoByP7KipmeKy63GFlV7dHS2XEirzP23c/RxZ/7W9UQ/VmgSvXjNzIXyu
+t+Ylfeg6VfFANsPQmov3SIO1Zh3NQRS/d+M5ig7lAgMBAAGjggE1MIIBMTAJBgNV
+HRMEAjAAMCwGCWCGSAGG+EIBDQQfFh1PcGVuU1NMIEdlbmVyYXRlZCBDZXJ0aWZp
+Y2F0ZTAdBgNVHQ4EFgQU6n4XBAZRnkrXMlko23vCClLlztEwgY4GA1UdIwSBhjCB
+g6F2pHQwcjELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNV
+BAcTDU1vdW50YWluIFZpZXcxFDASBgNVBAoTC0dvb2dsZSBJbmMuMSAwHgYDVQQD
+ExdhdHRpY3VzLmNvcnAuZ29vZ2xlLmNvbYIJAKGvvYc2FdhzMEYGA1UdHwQ/MD0w
+O6A5oDeGNWh0dHA6Ly9jaHJvbWV0d3MuY29ycC5nb29nbGUuY29tOjgwODAvY3Js
+L2F0dGljdXMuY3JsMA0GCSqGSIb3DQEBBQUAA4GBAHt8Zxf4D86QBuWkP2STN4Pg
+aLzut1eaY0u5DOxr3fqrwbaDlPJrJJqQJmMPwUItVMjw+zWO/CmoBY7vsJMcX/57
++kIHoQ+XCQJdOnM7qa3Hslt4bLRx+d0zQqt6/+gBNrz5F36njgansvxjnnacbApv
+uNQFNT5wCAqRadGn4d63
+-----END CERTIFICATE-----
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQCtCOOcHU09v37ovRXzTkde4nUBdC/5KZ/PA9TsxdMNId0zATfL
+M6TsQEGE9jE7gdew7bbTIsMy7qAcj+yoqZnisutxhZVe3R0tlxIq8z9t3P0cWf+1
+vVEP1ZoEr14zcyF8rrfmJX3oOlXxQDbD0JqL90iDtWYdzUEUv3fjOYoO5QIDAQAB
+AoGALSNMfMUICTuTF5N7zmR1KcNZUm/Es/KvxYjyWnGghKR4131R2Kg4NYjtjonK
+fgN5CKZ6Msm2seqdydWhnr3fbl3KhPF7nTdFUR6OM4eUuUH3nV1DS1w9AW0Z/4nB
+3OipLkxvTtRomfEUoZdE3I4+2T9iBdfPGduZ3CNR1H9EL2ECQQDfyT3dOVg5Zr7Q
+gZ8rQhyxtA+OhWvqU9F5Ahii002fIMj//rMIowvGesW2rfMwYs/oFF36l2vXOhvh
+whL+hmJ/AkEAxfFnlp8hT3WeRqJ74b3QSkyItWz7XbSQRPFWurPEfNI8wz57Svvy
+8tv396X8yImPwxXhihjSn9bwSHoon0WUmwJBAMt2S1HUblsaCEjIYS/SVwZWIILC
+2Z/d/yiW+FEp7nvMMhNVve8PVohpXVM/CkkmvcJSqjNI8ppnUaxLaDuTcP8CQQCn
+40L7K+ky3g3q1zG/nCEog5WIW/Ev31BmSVRIcuT7Ac5rw3kLdnrvpcbCE9U9uf9n
+dg2f9cHftIyEzKfbv0Z1AkAlX4x7De9edylrZ14IA9BbEe0ztsCik3UGXYriYfwY
+Z9h+uyhqNJT+SaWjb1QcBQ8jbYC0d2zxQSMuf8NxYrd2
+-----END RSA PRIVATE KEY-----
diff --git a/tools/python/google/httpd_config/mime.types b/tools/python/google/httpd_config/mime.types new file mode 100644 index 0000000..6735f1b --- /dev/null +++ b/tools/python/google/httpd_config/mime.types @@ -0,0 +1,599 @@ +# This is a comment. I love comments. + +# This file controls what Internet media types are sent to the client for +# given file extension(s). Sending the correct media type to the client +# is important so they know how to handle the content of the file. +# Extra types can either be added here or by using an AddType directive +# in your config files. For more information about Internet media types, +# please read RFC 2045, 2046, 2047, 2048, and 2077. The Internet media type +# registry is at <http://www.iana.org/assignments/media-types/>. + +# MIME type Extensions +application/activemessage +application/andrew-inset ez +application/applefile +application/atom+xml atom +application/atomicmail +application/batch-smtp +application/beep+xml +application/cals-1840 +application/cnrp+xml +application/commonground +application/cpl+xml +application/cybercash +application/dca-rft +application/dec-dx +application/dvcs +application/edi-consent +application/edifact +application/edi-x12 +application/eshop +application/font-tdpfr +application/http +application/hyperstudio +application/iges +application/index +application/index.cmd +application/index.obj +application/index.response +application/index.vnd +application/iotp +application/ipp +application/isup +application/mac-binhex40 hqx +application/mac-compactpro cpt +application/macwriteii +application/marc +application/mathematica +application/mathml+xml mathml +application/msword doc +application/news-message-id +application/news-transmission +application/ocsp-request +application/ocsp-response +application/octet-stream bin dms lha lzh exe class so dll dmg +application/oda oda +application/ogg ogg +application/parityfec +application/pdf pdf +application/pgp-encrypted +application/pgp-keys +application/pgp-signature +application/pkcs10 +application/pkcs7-mime +application/pkcs7-signature +application/pkix-cert +application/pkix-crl +application/pkixcmp +application/postscript ai eps ps +application/prs.alvestrand.titrax-sheet +application/prs.cww +application/prs.nprend +application/prs.plucker +application/qsig +application/rdf+xml rdf +application/reginfo+xml +application/remote-printing +application/riscos +application/rtf +application/sdp +application/set-payment +application/set-payment-initiation +application/set-registration +application/set-registration-initiation +application/sgml +application/sgml-open-catalog +application/sieve +application/slate +application/smil smi smil +application/srgs gram +application/srgs+xml grxml +application/timestamp-query +application/timestamp-reply +application/tve-trigger +application/vemmi +application/vnd.3gpp.pic-bw-large +application/vnd.3gpp.pic-bw-small +application/vnd.3gpp.pic-bw-var +application/vnd.3gpp.sms +application/vnd.3m.post-it-notes +application/vnd.accpac.simply.aso +application/vnd.accpac.simply.imp +application/vnd.acucobol +application/vnd.acucorp +application/vnd.adobe.xfdf +application/vnd.aether.imp +application/vnd.amiga.ami +application/vnd.anser-web-certificate-issue-initiation +application/vnd.anser-web-funds-transfer-initiation +application/vnd.audiograph +application/vnd.blueice.multipass +application/vnd.bmi +application/vnd.businessobjects +application/vnd.canon-cpdl +application/vnd.canon-lips +application/vnd.cinderella +application/vnd.claymore +application/vnd.commerce-battelle +application/vnd.commonspace +application/vnd.contact.cmsg +application/vnd.cosmocaller +application/vnd.criticaltools.wbs+xml +application/vnd.ctc-posml +application/vnd.cups-postscript +application/vnd.cups-raster +application/vnd.cups-raw +application/vnd.curl +application/vnd.cybank +application/vnd.data-vision.rdz +application/vnd.dna +application/vnd.dpgraph +application/vnd.dreamfactory +application/vnd.dxr +application/vnd.ecdis-update +application/vnd.ecowin.chart +application/vnd.ecowin.filerequest +application/vnd.ecowin.fileupdate +application/vnd.ecowin.series +application/vnd.ecowin.seriesrequest +application/vnd.ecowin.seriesupdate +application/vnd.enliven +application/vnd.epson.esf +application/vnd.epson.msf +application/vnd.epson.quickanime +application/vnd.epson.salt +application/vnd.epson.ssf +application/vnd.ericsson.quickcall +application/vnd.eudora.data +application/vnd.fdf +application/vnd.ffsns +application/vnd.fints +application/vnd.flographit +application/vnd.framemaker +application/vnd.fsc.weblaunch +application/vnd.fujitsu.oasys +application/vnd.fujitsu.oasys2 +application/vnd.fujitsu.oasys3 +application/vnd.fujitsu.oasysgp +application/vnd.fujitsu.oasysprs +application/vnd.fujixerox.ddd +application/vnd.fujixerox.docuworks +application/vnd.fujixerox.docuworks.binder +application/vnd.fut-misnet +application/vnd.grafeq +application/vnd.groove-account +application/vnd.groove-help +application/vnd.groove-identity-message +application/vnd.groove-injector +application/vnd.groove-tool-message +application/vnd.groove-tool-template +application/vnd.groove-vcard +application/vnd.hbci +application/vnd.hhe.lesson-player +application/vnd.hp-hpgl +application/vnd.hp-hpid +application/vnd.hp-hps +application/vnd.hp-pcl +application/vnd.hp-pclxl +application/vnd.httphone +application/vnd.hzn-3d-crossword +application/vnd.ibm.afplinedata +application/vnd.ibm.electronic-media +application/vnd.ibm.minipay +application/vnd.ibm.modcap +application/vnd.ibm.rights-management +application/vnd.ibm.secure-container +application/vnd.informix-visionary +application/vnd.intercon.formnet +application/vnd.intertrust.digibox +application/vnd.intertrust.nncp +application/vnd.intu.qbo +application/vnd.intu.qfx +application/vnd.irepository.package+xml +application/vnd.is-xpr +application/vnd.japannet-directory-service +application/vnd.japannet-jpnstore-wakeup +application/vnd.japannet-payment-wakeup +application/vnd.japannet-registration +application/vnd.japannet-registration-wakeup +application/vnd.japannet-setstore-wakeup +application/vnd.japannet-verification +application/vnd.japannet-verification-wakeup +application/vnd.jisp +application/vnd.kde.karbon +application/vnd.kde.kchart +application/vnd.kde.kformula +application/vnd.kde.kivio +application/vnd.kde.kontour +application/vnd.kde.kpresenter +application/vnd.kde.kspread +application/vnd.kde.kword +application/vnd.kenameaapp +application/vnd.koan +application/vnd.liberty-request+xml +application/vnd.llamagraphics.life-balance.desktop +application/vnd.llamagraphics.life-balance.exchange+xml +application/vnd.lotus-1-2-3 +application/vnd.lotus-approach +application/vnd.lotus-freelance +application/vnd.lotus-notes +application/vnd.lotus-organizer +application/vnd.lotus-screencam +application/vnd.lotus-wordpro +application/vnd.mcd +application/vnd.mediastation.cdkey +application/vnd.meridian-slingshot +application/vnd.micrografx.flo +application/vnd.micrografx.igx +application/vnd.mif mif +application/vnd.minisoft-hp3000-save +application/vnd.mitsubishi.misty-guard.trustweb +application/vnd.mobius.daf +application/vnd.mobius.dis +application/vnd.mobius.mbk +application/vnd.mobius.mqy +application/vnd.mobius.msl +application/vnd.mobius.plc +application/vnd.mobius.txf +application/vnd.mophun.application +application/vnd.mophun.certificate +application/vnd.motorola.flexsuite +application/vnd.motorola.flexsuite.adsi +application/vnd.motorola.flexsuite.fis +application/vnd.motorola.flexsuite.gotap +application/vnd.motorola.flexsuite.kmr +application/vnd.motorola.flexsuite.ttc +application/vnd.motorola.flexsuite.wem +application/vnd.mozilla.xul+xml xul +application/vnd.ms-artgalry +application/vnd.ms-asf +application/vnd.ms-excel xls +application/vnd.ms-lrm +application/vnd.ms-powerpoint ppt +application/vnd.ms-project +application/vnd.ms-tnef +application/vnd.ms-works +application/vnd.ms-wpl +application/vnd.mseq +application/vnd.msign +application/vnd.music-niff +application/vnd.musician +application/vnd.netfpx +application/vnd.noblenet-directory +application/vnd.noblenet-sealer +application/vnd.noblenet-web +application/vnd.novadigm.edm +application/vnd.novadigm.edx +application/vnd.novadigm.ext +application/vnd.obn +application/vnd.osa.netdeploy +application/vnd.palm +application/vnd.pg.format +application/vnd.pg.osasli +application/vnd.powerbuilder6 +application/vnd.powerbuilder6-s +application/vnd.powerbuilder7 +application/vnd.powerbuilder7-s +application/vnd.powerbuilder75 +application/vnd.powerbuilder75-s +application/vnd.previewsystems.box +application/vnd.publishare-delta-tree +application/vnd.pvi.ptid1 +application/vnd.pwg-multiplexed +application/vnd.pwg-xhtml-print+xml +application/vnd.quark.quarkxpress +application/vnd.rapid +application/vnd.rn-realmedia rm +application/vnd.s3sms +application/vnd.sealed.net +application/vnd.seemail +application/vnd.shana.informed.formdata +application/vnd.shana.informed.formtemplate +application/vnd.shana.informed.interchange +application/vnd.shana.informed.package +application/vnd.smaf +application/vnd.sss-cod +application/vnd.sss-dtf +application/vnd.sss-ntf +application/vnd.street-stream +application/vnd.svd +application/vnd.swiftview-ics +application/vnd.triscape.mxs +application/vnd.trueapp +application/vnd.truedoc +application/vnd.ufdl +application/vnd.uplanet.alert +application/vnd.uplanet.alert-wbxml +application/vnd.uplanet.bearer-choice +application/vnd.uplanet.bearer-choice-wbxml +application/vnd.uplanet.cacheop +application/vnd.uplanet.cacheop-wbxml +application/vnd.uplanet.channel +application/vnd.uplanet.channel-wbxml +application/vnd.uplanet.list +application/vnd.uplanet.list-wbxml +application/vnd.uplanet.listcmd +application/vnd.uplanet.listcmd-wbxml +application/vnd.uplanet.signal +application/vnd.vcx +application/vnd.vectorworks +application/vnd.vidsoft.vidconference +application/vnd.visio +application/vnd.visionary +application/vnd.vividence.scriptfile +application/vnd.vsf +application/vnd.wap.sic +application/vnd.wap.slc +application/vnd.wap.wbxml wbxml +application/vnd.wap.wmlc wmlc +application/vnd.wap.wmlscriptc wmlsc +application/vnd.webturbo +application/vnd.wrq-hp3000-labelled +application/vnd.wt.stf +application/vnd.wv.csp+wbxml +application/vnd.xara +application/vnd.xfdl +application/vnd.yamaha.hv-dic +application/vnd.yamaha.hv-script +application/vnd.yamaha.hv-voice +application/vnd.yellowriver-custom-menu +application/voicexml+xml vxml +application/watcherinfo+xml +application/whoispp-query +application/whoispp-response +application/wita +application/wordperfect5.1 +application/x-bcpio bcpio +application/x-cdlink vcd +application/x-chess-pgn pgn +application/x-compress +application/x-cpio cpio +application/x-csh csh +application/x-director dcr dir dxr +application/x-dvi dvi +application/x-futuresplash spl +application/x-gtar gtar +application/x-gzip +application/x-hdf hdf +application/x-javascript js +application/x-java-jnlp-file jnlp +application/x-koan skp skd skt skm +application/x-latex latex +application/x-netcdf nc cdf +application/x-sh sh +application/x-shar shar +application/x-shockwave-flash swf +application/x-stuffit sit +application/x-sv4cpio sv4cpio +application/x-sv4crc sv4crc +application/x-tar tar +application/x-tcl tcl +application/x-tex tex +application/x-texinfo texinfo texi +application/x-troff t tr roff +application/x-troff-man man +application/x-troff-me me +application/x-troff-ms ms +application/x-ustar ustar +application/x-wais-source src +application/x400-bp +application/xhtml+xml xhtml xht +application/xslt+xml xslt +application/xml xml xsl +application/xml-dtd dtd +application/xml-external-parsed-entity +application/zip zip +audio/32kadpcm +audio/amr +audio/amr-wb +audio/basic au snd +audio/cn +audio/dat12 +audio/dsr-es201108 +audio/dvi4 +audio/evrc +audio/evrc0 +audio/g722 +audio/g.722.1 +audio/g723 +audio/g726-16 +audio/g726-24 +audio/g726-32 +audio/g726-40 +audio/g728 +audio/g729 +audio/g729D +audio/g729E +audio/gsm +audio/gsm-efr +audio/l8 +audio/l16 +audio/l20 +audio/l24 +audio/lpc +audio/midi mid midi kar +audio/mpa +audio/mpa-robust +audio/mp4a-latm m4a m4p +audio/mpeg mpga mp2 mp3 +audio/parityfec +audio/pcma +audio/pcmu +audio/prs.sid +audio/qcelp +audio/red +audio/smv +audio/smv0 +audio/telephone-event +audio/tone +audio/vdvi +audio/vnd.3gpp.iufp +audio/vnd.cisco.nse +audio/vnd.cns.anp1 +audio/vnd.cns.inf1 +audio/vnd.digital-winds +audio/vnd.everad.plj +audio/vnd.lucent.voice +audio/vnd.nortel.vbk +audio/vnd.nuera.ecelp4800 +audio/vnd.nuera.ecelp7470 +audio/vnd.nuera.ecelp9600 +audio/vnd.octel.sbc +audio/vnd.qcelp +audio/vnd.rhetorex.32kadpcm +audio/vnd.vmx.cvsd +audio/x-aiff aif aiff aifc +audio/x-alaw-basic +audio/x-mpegurl m3u +audio/x-pn-realaudio ram ra +audio/x-pn-realaudio-plugin +audio/x-wav wav +chemical/x-pdb pdb +chemical/x-xyz xyz +image/bmp bmp +image/cgm cgm +image/g3fax +image/gif gif +image/ief ief +image/jpeg jpeg jpg jpe +image/jp2 jp2 +image/naplps +image/pict pict pic pct +image/png png +image/prs.btif +image/prs.pti +image/svg+xml svg +image/t38 +image/tiff tiff tif +image/tiff-fx +image/vnd.cns.inf2 +image/vnd.djvu djvu djv +image/vnd.dwg +image/vnd.dxf +image/vnd.fastbidsheet +image/vnd.fpx +image/vnd.fst +image/vnd.fujixerox.edmics-mmr +image/vnd.fujixerox.edmics-rlc +image/vnd.globalgraphics.pgb +image/vnd.mix +image/vnd.ms-modi +image/vnd.net-fpx +image/vnd.svf +image/vnd.wap.wbmp wbmp +image/vnd.xiff +image/x-cmu-raster ras +image/x-macpaint pntg pnt mac +image/x-icon ico +image/x-portable-anymap pnm +image/x-portable-bitmap pbm +image/x-portable-graymap pgm +image/x-portable-pixmap ppm +image/x-quicktime qtif qti +image/x-rgb rgb +image/x-xbitmap xbm +image/x-xpixmap xpm +image/x-xwindowdump xwd +message/delivery-status +message/disposition-notification +message/external-body +message/http +message/news +message/partial +message/rfc822 +message/s-http +message/sip +message/sipfrag +model/iges igs iges +model/mesh msh mesh silo +model/vnd.dwf +model/vnd.flatland.3dml +model/vnd.gdl +model/vnd.gs-gdl +model/vnd.gtw +model/vnd.mts +model/vnd.parasolid.transmit.binary +model/vnd.parasolid.transmit.text +model/vnd.vtu +model/vrml wrl vrml +multipart/alternative +multipart/appledouble +multipart/byteranges +multipart/digest +multipart/encrypted +multipart/form-data +multipart/header-set +multipart/mixed +multipart/parallel +multipart/related +multipart/report +multipart/signed +multipart/voice-message +text/calendar ics ifb +text/css css +text/directory +text/enriched +text/html html htm +text/parityfec +text/plain asc txt +text/prs.lines.tag +text/rfc822-headers +text/richtext rtx +text/rtf rtf +text/sgml sgml sgm +text/t140 +text/tab-separated-values tsv +text/uri-list +text/vnd.abc +text/vnd.curl +text/vnd.dmclientscript +text/vnd.fly +text/vnd.fmi.flexstor +text/vnd.in3d.3dml +text/vnd.in3d.spot +text/vnd.iptc.nitf +text/vnd.iptc.newsml +text/vnd.latex-z +text/vnd.motorola.reflex +text/vnd.ms-mediapackage +text/vnd.net2phone.commcenter.command +text/vnd.sun.j2me.app-descriptor +text/vnd.wap.si +text/vnd.wap.sl +text/vnd.wap.wml wml +text/vnd.wap.wmlscript wmls +text/x-setext etx +text/xml +text/xml-external-parsed-entity +video/bmpeg +video/bt656 +video/celb +video/dv +video/h261 +video/h263 +video/h263-1998 +video/h263-2000 +video/jpeg +video/mp1s +video/mp2p +video/mp2t +video/mp4 mp4 +video/mp4v-es +video/mpv +video/mpeg mpeg mpg mpe +video/nv +video/parityfec +video/pointer +video/quicktime qt mov +video/smpte292m +video/vnd.fvt +video/vnd.motorola.video +video/vnd.motorola.videop +video/vnd.mpegurl mxu m4u +video/vnd.nokia.interleaved-multimedia +video/vnd.objectvideo +video/vnd.vivo +video/x-dv dv dif +video/x-msvideo avi +video/x-sgi-movie movie +x-conference/x-cooltalk ice diff --git a/tools/python/google/httpd_utils.py b/tools/python/google/httpd_utils.py new file mode 100644 index 0000000..7f76ca6 --- /dev/null +++ b/tools/python/google/httpd_utils.py @@ -0,0 +1,219 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""A class to help start/stop a local apache http server.""" + +import logging +import optparse +import os +import subprocess +import sys +import time +import urllib + +import google.path_utils +import google.platform_utils + +class HttpdNotStarted(Exception): pass + +def ApacheConfigDir(start_dir): + """Returns a path to the directory holding the Apache config files.""" + return google.path_utils.FindUpward(start_dir, 'tools', 'python', + 'google', 'httpd_config') + + +def GetCygserverPath(start_dir, apache2=False): + """Returns the path to the directory holding cygserver.exe file.""" + cygserver_path = None + if apache2: + cygserver_path = google.path_utils.FindUpward(start_dir, 'third_party', + 'cygwin', 'usr', 'sbin') + return cygserver_path + + +def StartServer(document_root=None, output_dir=None, apache2=False): + """Starts a local server on port 8000 using the basic configuration files. + + Args: + document_root: If present, specifies the document root for the server; + otherwise, the filesystem's root (e.g., C:/ or /) will be used. + output_dir: If present, specifies where to put server logs; otherwise, + they'll be placed in the system's temp dir (e.g., $TEMP or /tmp). + apache2: boolean if true will cause this function to configure + for Apache 2.x as opposed to Apache 1.3.x + + Returns: the ApacheHttpd object that was created + """ + script_dir = google.path_utils.ScriptDir() + platform_util = google.platform_utils.PlatformUtility(script_dir) + if not output_dir: + output_dir = platform_util.GetTempDirectory() + if not document_root: + document_root = platform_util.GetFilesystemRoot() + apache_config_dir = ApacheConfigDir(script_dir) + if apache2: + httpd_conf_path = os.path.join(apache_config_dir, 'httpd2.conf') + else: + httpd_conf_path = os.path.join(apache_config_dir, 'httpd.conf') + mime_types_path = os.path.join(apache_config_dir, 'mime.types') + start_cmd = platform_util.GetStartHttpdCommand(output_dir, + httpd_conf_path, + mime_types_path, + document_root, + apache2=apache2) + stop_cmd = platform_util.GetStopHttpdCommand() + httpd = ApacheHttpd(start_cmd, stop_cmd, [8000], + cygserver_path=GetCygserverPath(script_dir, apache2)) + httpd.StartServer() + return httpd + + +def StopServers(apache2=False): + """Calls the platform's stop command on a newly created server, forcing it + to stop. + + The details depend on the behavior of the platform stop command. For example, + it's often implemented to kill all running httpd processes, as implied by + the name of this function. + + Args: + apache2: boolean if true will cause this function to configure + for Apache 2.x as opposed to Apache 1.3.x + """ + script_dir = google.path_utils.ScriptDir() + platform_util = google.platform_utils.PlatformUtility(script_dir) + httpd = ApacheHttpd('', platform_util.GetStopHttpdCommand(), [], + cygserver_path=GetCygserverPath(script_dir, apache2)) + httpd.StopServer(force=True) + + +class ApacheHttpd(object): + def __init__(self, start_command, stop_command, port_list, + cygserver_path=None): + """Args: + start_command: command list to call to start the httpd + stop_command: command list to call to stop the httpd if one has been + started. May kill all httpd processes running on the machine. + port_list: list of ports expected to respond on the local machine when + the server has been successfully started. + cygserver_path: Path to cygserver.exe. If specified, exe will be started + with server as well as stopped when server is stopped. + """ + self._http_server_proc = None + self._start_command = start_command + self._stop_command = stop_command + self._port_list = port_list + self._cygserver_path = cygserver_path + + def StartServer(self): + if self._http_server_proc: + return + if self._cygserver_path: + cygserver_exe = os.path.join(self._cygserver_path, "cygserver.exe") + cygbin = google.path_utils.FindUpward(cygserver_exe, 'third_party', + 'cygwin', 'bin') + env = os.environ + env['PATH'] += ";" + cygbin + subprocess.Popen(cygserver_exe, env=env) + logging.info('Starting http server') + self._http_server_proc = subprocess.Popen(self._start_command) + + # Ensure that the server is running on all the desired ports. + for port in self._port_list: + if not self._UrlIsAlive('http://127.0.0.1:%s/' % str(port)): + raise HttpdNotStarted('Failed to start httpd on port %s' % str(port)) + + def _UrlIsAlive(self, url): + """Checks to see if we get an http response from |url|. + We poll the url 5 times with a 1 second delay. If we don't + get a reply in that time, we give up and assume the httpd + didn't start properly. + + Args: + url: The URL to check. + Return: + True if the url is alive. + """ + wait_time = 5 + while wait_time > 0: + try: + response = urllib.urlopen(url) + # Server is up and responding. + return True + except IOError: + pass + wait_time -= 1 + # Wait a second and try again. + time.sleep(1) + + return False + + def StopServer(self, force=False): + """If we started an httpd.exe process, or if force is True, call + self._stop_command (passed in on init so it can be platform-dependent). + This will presumably kill it, and may also kill any other httpd.exe + processes that are running. + """ + if force or self._http_server_proc: + logging.info('Stopping http server') + kill_proc = subprocess.Popen(self._stop_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + logging.info('%s\n%s' % (kill_proc.stdout.read(), + kill_proc.stderr.read())) + self._http_server_proc = None + if self._cygserver_path: + subprocess.Popen(["taskkill.exe", "/f", "/im", "cygserver.exe"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + +if '__main__' == __name__: + # Provide some command line params for starting/stopping the http server + # manually. + option_parser = optparse.OptionParser() + option_parser.add_option('-k', '--server', help='Server action (start|stop)') + option_parser.add_option('-r', '--root', help='Document root (optional)') + option_parser.add_option('-a', '--apache2', action='store_true', + default=False, help='Starts Apache 2 instead of Apache 1.3 (default).') + options, args = option_parser.parse_args() + + if not options.server: + print ("Usage: %s -k {start|stop} [-r document_root] [--apache2]" % + sys.argv[0]) + sys.exit(0) + + document_root = None + if options.root: + document_root = options.root + + if 'start' == options.server: + StartServer(document_root, apache2=options.apache2) + else: + StopServers(apache2=options.apache2) diff --git a/tools/python/google/logging_utils.py b/tools/python/google/logging_utils.py new file mode 100644 index 0000000..cf9f3fa --- /dev/null +++ b/tools/python/google/logging_utils.py @@ -0,0 +1,110 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# logging_utils.py + +''' Utility functions and objects for logging. +''' + +import logging +import sys + +class StdoutStderrHandler(logging.Handler): + ''' Subclass of logging.Handler which outputs to either stdout or stderr + based on a threshold level. + ''' + + def __init__(self, threshold=logging.WARNING, err=sys.stderr, out=sys.stdout): + ''' Args: + threshold: below this logging level messages are sent to stdout, + otherwise they are sent to stderr + err: a stream object that error messages are sent to, defaults to + sys.stderr + out: a stream object that non-error messages are sent to, defaults to + sys.stdout + ''' + logging.Handler.__init__(self) + self._err = logging.StreamHandler(err) + self._out = logging.StreamHandler(out) + self._threshold = threshold + self._last_was_err = False + + def setLevel(self, lvl): + logging.Handler.setLevel(self, lvl) + self._err.setLevel(lvl) + self._out.setLevel(lvl) + + def setFormatter(self, formatter): + logging.Handler.setFormatter(self, formatter) + self._err.setFormatter(formatter) + self._out.setFormatter(formatter) + + def emit(self, record): + if record.levelno < self._threshold: + self._out.emit(record) + self._last_was_err = False + else: + self._err.emit(record) + self._last_was_err = False + + def flush(self): + # preserve order on the flushing, the stalest stream gets flushed first + if self._last_was_err: + self._out.flush() + self._err.flush() + else: + self._err.flush() + self._out.flush() + + +FORMAT = "%(asctime)s %(filename)s [%(levelname)s] %(message)s" +DATEFMT = "%H:%M:%S" + +def config_root(level=logging.INFO, threshold=logging.WARNING, format=FORMAT, + datefmt=DATEFMT): + ''' Configure the root logger to use a StdoutStderrHandler and some default + formatting. + Args: + level: messages below this level are ignored + threshold: below this logging level messages are sent to stdout, + otherwise they are sent to stderr + format: format for log messages, see logger.Format + datefmt: format for date in log messages + + ''' + # to set the handler of the root logging object, we need to do setup + # manually rather than using basicConfig + root = logging.getLogger() + root.setLevel(level) + formatter = logging.Formatter(format, datefmt) + handler = StdoutStderrHandler(threshold=threshold) + handler.setLevel(level) + handler.setFormatter(formatter) + root.addHandler(handler) diff --git a/tools/python/google/path_utils.py b/tools/python/google/path_utils.py new file mode 100644 index 0000000..0585614 --- /dev/null +++ b/tools/python/google/path_utils.py @@ -0,0 +1,109 @@ +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Some utility methods for getting and manipulating paths.""" + +# TODO(pamg): Have the buildbot use these, too. + + +import errno +import os +import sys + +class PathNotFound(Exception): pass + +def ScriptDir(): + """Get the full path to the directory containing the current script.""" + script_filename = os.path.abspath(sys.argv[0]) + return os.path.dirname(script_filename) + +def FindAncestor(start_dir, ancestor): + """Finds an ancestor dir in a path. + + For example, FindAncestor('c:\foo\bar\baz', 'bar') would return + 'c:\foo\bar'. Unlike FindUpward*, this only looks at direct path ancestors. + """ + start_dir = os.path.abspath(start_dir) + path = start_dir + while True: + (parent, tail) = os.path.split(path) + if tail == ancestor: + return path + if not tail: + break + path = parent + raise PathNotFound("Unable to find ancestor %s in %s" % (ancestor, start_dir)) + +def FindUpwardParent(start_dir, *desired_list): + """Finds the desired object's parent, searching upward from the start_dir. + + Searches start_dir and all its parents looking for the desired directory + or file, which may be given in one or more path components. Returns the + first directory in which the top desired path component was found, or raises + PathNotFound if it wasn't. + """ + desired_path = os.path.join(*desired_list) + last_dir = '' + cur_dir = start_dir + found_path = os.path.join(cur_dir, desired_path) + while not os.path.exists(found_path): + last_dir = cur_dir + cur_dir = os.path.dirname(cur_dir) + if last_dir == cur_dir: + raise PathNotFound('Unable to find %s above %s' % + (desired_path, start_dir)) + found_path = os.path.join(cur_dir, desired_path) + # Strip the entire original desired path from the end of the one found + # and remove a trailing path separator, if present. + found_path = found_path[:len(found_path) - len(desired_path)] + if found_path.endswith(os.sep): + found_path = found_path[:len(found_path) - 1] + return found_path + + +def FindUpward(start_dir, *desired_list): + """Returns a path to the desired directory or file, searching upward. + + Searches start_dir and all its parents looking for the desired directory + or file, which may be given in one or more path components. Returns the full + path to the desired object, or raises PathNotFound if it wasn't found. + """ + parent = FindUpwardParent(start_dir, *desired_list) + return os.path.join(parent, *desired_list) + + +def MaybeMakeDirectory(*path): + """Creates an entire path, if it doesn't already exist.""" + file_path = os.path.join(*path) + try: + os.makedirs(file_path) + except OSError, e: + # errno.EEXIST is "File exists". If we see another error, re-raise. + if e.errno != errno.EEXIST: + raise diff --git a/tools/python/google/platform_utils.py b/tools/python/google/platform_utils.py new file mode 100644 index 0000000..1fcb182 --- /dev/null +++ b/tools/python/google/platform_utils.py @@ -0,0 +1,46 @@ +#!/bin/env python +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Platform-specific utilities and pseudo-constants + +Any functions whose implementations or values differ from one platform to +another should be defined in their respective platform_utils_<platform>.py +modules. The appropriate one of those will be imported into this module to +provide callers with a common, platform-independent interface. +""" + +import sys + +# We may not support the version of Python that a user has installed (Cygwin +# especially has had problems), but we'll allow the platform utils to be +# included in any case so we don't get an import error. +if sys.platform in ('cygwin', 'win32'): + from platform_utils_win import * + diff --git a/tools/python/google/platform_utils_win.py b/tools/python/google/platform_utils_win.py new file mode 100644 index 0000000..df72e3d --- /dev/null +++ b/tools/python/google/platform_utils_win.py @@ -0,0 +1,219 @@ +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Platform-specific utility methods shared by several scripts.""" + +import os +import re +import subprocess +import sys + +import google.path_utils + +# Cache a single cygpath process for use throughout, even across instances of +# the PlatformUtility class. +_cygpath_proc = None + +class PlatformUtility(object): + def __init__(self, base_dir): + """Args: + base_dir: a directory above which third_party/cygwin can be found, + used to locate the cygpath executable for path conversions. + """ + self._cygwin_root = None + self._base_dir = base_dir + + def _CygwinRoot(self): + """Returns the full path to third_party/cygwin/.""" + if not self._cygwin_root: + self._cygwin_root = google.path_utils.FindUpward(self._base_dir, + 'third_party', 'cygwin') + return self._cygwin_root + + def _PathToExecutable(self, executable): + """Returns the full path to an executable in Cygwin's bin dir.""" + return os.path.join(self._CygwinRoot(), 'bin', executable) + + def GetAbsolutePath(self, path, force=False): + """Returns an absolute windows path. If platform is cygwin, converts it to + windows style using cygpath. + + For performance reasons, we use a single cygpath process, shared among all + instances of this class. Otherwise Python can run out of file handles. + """ + if not force and sys.platform != "cygwin": + return os.path.abspath(path) + global _cygpath_proc + if not _cygpath_proc: + cygpath_command = [self._PathToExecutable("cygpath.exe"), + "-a", "-m", "-f", "-"] + _cygpath_proc = subprocess.Popen(cygpath_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + _cygpath_proc.stdin.write(path + "\n") + return _cygpath_proc.stdout.readline().rstrip() + + def GetFilesystemRoot(self): + """Returns the root directory of the file system.""" + return os.environ['SYSTEMDRIVE'] + '\\' + + def GetTempDirectory(self): + """Returns the file system's base temp directory, or the filesystem root + if the standard temp directory can't be determined. + + Note that this does not use a random subdirectory, so it's not + intrinsically secure. If you need a secure subdir, use the tempfile + package. + """ + return os.environ.get('TEMP', self.GetFilesystemRoot()) + + def FilenameToUri(self, path, use_http=False, use_ssl=False, port=8000): + """Convert a Windows style path to a URI. + + Args: + path: For an http URI, the path relative to the httpd server's + DocumentRoot; for a file URI, the full path to the file. + use_http: if True, returns a URI of the form http://127.0.0.1:8000/. + If False, returns a file:/// URI. + use_ssl: if True, returns HTTPS URL (https://127.0.0.1:8000/). + This parameter is ignored if use_http=False. + port: The port number to append when returning an HTTP URI + """ + if use_http: + protocol = 'http' + if use_ssl: + protocol = 'https' + path = path.replace("\\", "/") + return "%s://127.0.0.1:%s/%s" % (protocol, str(port), path) + return "file:///" + self.GetAbsolutePath(path) + + def GetStartHttpdCommand(self, output_dir, + httpd_conf_path, mime_types_path, + document_root=None, apache2=False): + """Prepares the config file and output directory to start an httpd server. + Returns a list of strings containing the server's command line+args. + + Args: + output_dir: the path to the server's output directory, for log files. + It will be created if necessary. + httpd_conf_path: full path to the httpd.conf file to be used. + mime_types_path: full path to the mime.types file to be used. + document_root: full path to the DocumentRoot. If None, the DocumentRoot + from the httpd.conf file will be used. Note that the httpd.conf + file alongside this script does not specify any DocumentRoot, so if + you're using that one, be sure to specify a document_root here. + apache2: boolean if true will cause this function to return start + command for Apache 2.x as opposed to Apache 1.3.x + """ + + if document_root: + document_root = GetCygwinPath(document_root) + exe_name = "httpd" + cert_file = "" + if apache2: + exe_name = "httpd2" + cert_file = google.path_utils.FindUpward(self._base_dir, 'tools', + 'python', 'google', + 'httpd_config', 'httpd2.pem') + httpd_vars = { + "httpd_executable_path": GetCygwinPath( + os.path.join(self._CygwinRoot(), "usr", "sbin", exe_name)), + "httpd_conf_path": GetCygwinPath(httpd_conf_path), + "ssl_certificate_file": GetCygwinPath(cert_file), + "document_root" : document_root, + "server_root": GetCygwinPath(os.path.join(self._CygwinRoot(), "usr")), + "mime_types_path": GetCygwinPath(mime_types_path), + "output_dir": GetCygwinPath(output_dir), + "bindir": GetCygwinPath(os.path.join(self._CygwinRoot(), "bin")), + "user": os.environ.get("USERNAME", os.environ.get("USER", "")), + } + if not httpd_vars["user"]: + # Failed to get the username from the environment; use whoami.exe + # instead. + proc = subprocess.Popen(self._PathToExecutable("whoami.exe"), + stdout=subprocess.PIPE) + httpd_vars["user"] = proc.stdout.read().strip() + + if not httpd_vars["user"]: + raise Exception("Failed to get username.") + + google.path_utils.MaybeMakeDirectory(output_dir) + + # We have to wrap the command in bash because the cygwin environment + # is required for httpd to run. + # -C: process directive before reading config files + # -c: process directive after reading config files + # Apache wouldn't run CGIs with permissions==700 unless we add + # -c User "<username>" + bash = self._PathToExecutable("bash.exe") + httpd_cmd_string = ( + ' PATH=%(bindir)s %(httpd_executable_path)s' + ' -f %(httpd_conf_path)s' + ' -c \'TypesConfig "%(mime_types_path)s"\'' + ' -c \'CustomLog "%(output_dir)s/access_log.txt" common\'' + ' -c \'ErrorLog "%(output_dir)s/error_log.txt"\'' + ' -c \'PidFile "%(output_dir)s/httpd.pid"\'' + ' -C \'User "%(user)s"\'' + ' -C \'ServerRoot "%(server_root)s"\'' + ) + if apache2: + httpd_cmd_string = ('export CYGWIN=server;' + httpd_cmd_string + + ' -c \'SSLCertificateFile "%(ssl_certificate_file)s"\'') + if document_root: + httpd_cmd_string += ' -C \'DocumentRoot "%(document_root)s"\'' + + httpd_cmd = [bash, "-c", httpd_cmd_string % httpd_vars] + return httpd_cmd + + def GetStopHttpdCommand(self): + """Returns a list of strings that contains the command line+args needed to + stop the http server used in the http tests. + """ + # Force kill (/f) *all* httpd processes. This has the side effect of + # killing httpd processes that we didn't start. + return ["taskkill.exe", "/f", "/im", "httpd*"] + +########################################################################### +# This method is specific to windows, expected to be used only by *_win.py +# files. + +def GetCygwinPath(path): + """Convert a Windows path to a cygwin path. + + The cygpath utility insists on converting paths that it thinks are Cygwin + root paths to what it thinks the correct roots are. So paths such as + "C:\b\slave\webkit-release-kjs\build\third_party\cygwin\bin" are converted to + plain "/usr/bin". To avoid this, we do the conversion manually. + + The path is expected to be an absolute path, on any drive. + """ + drive_regexp = re.compile(r'([a-z]):[/\\]', re.IGNORECASE) + def LowerDrive(matchobj): + return '/cygdrive/%s/' % matchobj.group(1).lower() + path = drive_regexp.sub(LowerDrive, path) + return path.replace('\\', '/') diff --git a/tools/python/google/process_utils.py b/tools/python/google/process_utils.py new file mode 100644 index 0000000..966fdf6 --- /dev/null +++ b/tools/python/google/process_utils.py @@ -0,0 +1,156 @@ +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Shared process-related utility functions.""" + +import errno +import os +import subprocess +import sys + +class CommandNotFound(Exception): pass + + +TASKKILL = os.path.join(os.environ['WINDIR'], 'system32', 'taskkill.exe') +TASKKILL_PROCESS_NOT_FOUND_ERR = 128 +# On windows 2000 there is no taskkill.exe, we need to have pskill somewhere +# in the path. +PSKILL = 'pskill.exe' +PSKILL_PROCESS_NOT_FOUND_ERR = -1 + +def KillAll(executables): + """Tries to kill all copies of each process in the processes list. Returns + an error if any running processes couldn't be killed. + """ + result = 0 + if os.path.exists(TASKKILL): + command = [TASKKILL, '/f', '/im'] + process_not_found_err = TASKKILL_PROCESS_NOT_FOUND_ERR + else: + command = [PSKILL, '/t'] + process_not_found_err = PSKILL_PROCESS_NOT_FOUND_ERR + + for name in executables: + new_error = RunCommand(command + [name]) + # Ignore "process not found" error. + if new_error != 0 and new_error != process_not_found_err: + result = new_error + return result + +def RunCommandFull(command, verbose=True, collect_output=False, + print_output=True): + """Runs the command list. + + Prints the given command (which should be a list of one or more strings). + If specified, prints its stderr (and optionally stdout) to stdout, + line-buffered, converting line endings to CRLF (see note below). If + specified, collects the output as a list of lines and returns it. Waits + for the command to terminate and returns its status. + + Args: + command: the full command to run, as a list of one or more strings + verbose: if True, combines all output (stdout and stderr) into stdout. + Otherwise, prints only the command's stderr to stdout. + collect_output: if True, collects the output of the command as a list of + lines and returns it + print_output: if True, prints the output of the command + + Returns: + A tuple consisting of the process's exit status and output. If + collect_output is False, the output will be []. + + Raises: + CommandNotFound if the command executable could not be found. + """ + print '\n' + subprocess.list2cmdline(command).replace('\\', '/') + '\n', ### + + if verbose: + out = subprocess.PIPE + err = subprocess.STDOUT + else: + out = file(os.devnull, 'w') + err = subprocess.PIPE + try: + proc = subprocess.Popen(command, stdout=out, stderr=err, bufsize=1) + except OSError, e: + if e.errno == errno.ENOENT: + raise CommandNotFound('Unable to find "%s"' % command[0]) + raise + + output = [] + + if verbose: + read_from = proc.stdout + else: + read_from = proc.stderr + line = read_from.readline() + while line: + line = line.rstrip() + + if collect_output: + output.append(line) + + if print_output: + # Windows Python converts \n to \r\n automatically whenever it + # encounters it written to a text file (including stdout). The only + # way around it is to write to a binary file, which isn't feasible for + # stdout. So we end up with \r\n here even though we explicitly write + # \n. (We could write \r instead, which doesn't get converted to \r\n, + # but that's probably more troublesome for people trying to read the + # files.) + print line + '\n', + + # Python on windows writes the buffer only when it reaches 4k. This is + # not fast enough for all purposes. + sys.stdout.flush() + line = read_from.readline() + + if not verbose: + out.close() + return (proc.returncode, output) + +def RunCommand(command, verbose=True): + """Runs the command list, printing its output and returning its exit status. + + Prints the given command (which should be a list of one or more strings), + then runs it and prints its stderr (and optionally stdout) to stdout, + line-buffered, converting line endings to CRLF. Waits for the command to + terminate and returns its status. + + Args: + command: the full command to run, as a list of one or more strings + verbose: if True, combines all output (stdout and stderr) into stdout. + Otherwise, prints only the command's stderr to stdout. + + Returns: + The process's exit status. + + Raises: + CommandNotFound if the command executable could not be found. + """ + return RunCommandFull(command, verbose)[0] diff --git a/tools/site_compare/command_line.py b/tools/site_compare/command_line.py new file mode 100644 index 0000000..b99a1c9 --- /dev/null +++ b/tools/site_compare/command_line.py @@ -0,0 +1,823 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Parse a command line, retrieving a command and its arguments. + +Supports the concept of command line commands, each with its own set +of arguments. Supports dependent arguments and mutually exclusive arguments. +Basically, a better optparse. I took heed of epg's WHINE() in gvn.cmdline +and dumped optparse in favor of something better. +""" + +import os.path +import re +import string +import sys +import textwrap +import types + + +def IsString(var): + """Little helper function to see if a variable is a string.""" + return type(var) in types.StringTypes + + +class ParseError(Exception): + """Encapsulates errors from parsing, string arg is description.""" + pass + + +class Command(object): + """Implements a single command.""" + + def __init__(self, names, helptext, validator=None, impl=None): + """Initializes Command from names and helptext, plus optional callables. + + Args: + names: command name, or list of synonyms + helptext: brief string description of the command + validator: callable for custom argument validation + Should raise ParseError if it wants + impl: callable to be invoked when command is called + """ + self.names = names + self.validator = validator + self.helptext = helptext + self.impl = impl + self.args = [] + self.required_groups = [] + self.arg_dict = {} + self.positional_args = [] + self.cmdline = None + + class Argument(object): + """Encapsulates an argument to a command.""" + VALID_TYPES = ['string', 'readfile', 'int', 'flag', 'coords'] + TYPES_WITH_VALUES = ['string', 'readfile', 'int', 'coords'] + + def __init__(self, names, helptext, type, metaname, + required, default, positional): + """Command-line argument to a command. + + Args: + names: argument name, or list of synonyms + helptext: brief description of the argument + type: type of the argument. Valid values include: + string - a string + readfile - a file which must exist and be available + for reading + int - an integer + flag - an optional flag (bool) + coords - (x,y) where x and y are ints + metaname: Name to display for value in help, inferred if not + specified + required: True if argument must be specified + default: Default value if not specified + positional: Argument specified by location, not name + + Raises: + ValueError: the argument name is invalid for some reason + """ + if type not in Command.Argument.VALID_TYPES: + raise ValueError("Invalid type: %r" % type) + + if required and default is not None: + raise ValueError("required and default are mutually exclusive") + + if required and type == 'flag': + raise ValueError("A required flag? Give me a break.") + + if metaname and type not in Command.Argument.TYPES_WITH_VALUES: + raise ValueError("Type %r can't have a metaname" % type) + + # If no metaname is provided, infer it: use the alphabetical characters + # of the last provided name + if not metaname and type in Command.Argument.TYPES_WITH_VALUES: + metaname = ( + names[-1].lstrip(string.punctuation + string.whitespace).upper()) + + self.names = names + self.helptext = helptext + self.type = type + self.required = required + self.default = default + self.positional = positional + self.metaname = metaname + + self.mutex = [] # arguments that are mutually exclusive with + # this one + self.depends = [] # arguments that must be present for this + # one to be valid + self.present = False # has this argument been specified? + + def AddDependency(self, arg): + """Makes this argument dependent on another argument. + + Args: + arg: name of the argument this one depends on + """ + if arg not in self.depends: + self.depends.append(arg) + + def AddMutualExclusion(self, arg): + """Makes this argument invalid if another is specified. + + Args: + arg: name of the mutually exclusive argument. + """ + if arg not in self.mutex: + self.mutex.append(arg) + + def GetUsageString(self): + """Returns a brief string describing the argument's usage.""" + if not self.positional: + string = self.names[0] + if self.type in Command.Argument.TYPES_WITH_VALUES: + string += "="+self.metaname + else: + string = self.metaname + + if not self.required: + string = "["+string+"]" + + return string + + def GetNames(self): + """Returns a string containing a list of the arg's names.""" + if self.positional: + return self.metaname + else: + return ", ".join(self.names) + + def GetHelpString(self, width=80, indent=5, names_width=20, gutter=2): + """Returns a help string including help for all the arguments.""" + names = [" "*indent + line +" "*(names_width-len(line)) for line in + textwrap.wrap(self.GetNames(), names_width)] + + helpstring = textwrap.wrap(self.helptext, width-indent-names_width-gutter) + + if len(names) < len(helpstring): + names += [" "*(indent+names_width)]*(len(helpstring)-len(names)) + + if len(helpstring) < len(names): + helpstring += [""]*(len(names)-len(helpstring)) + + return "\n".join([name_line + " "*gutter + help_line for + name_line, help_line in zip(names, helpstring)]) + + def __repr__(self): + if self.present: + string = '= %r' % self.value + else: + string = "(absent)" + + return "Argument %s '%s'%s" % (self.type, self.names[0], string) + + # end of nested class Argument + + def AddArgument(self, names, helptext, type="string", metaname=None, + required=False, default=None, positional=False): + """Command-line argument to a command. + + Args: + names: argument name, or list of synonyms + helptext: brief description of the argument + type: type of the argument + metaname: Name to display for value in help, inferred if not + required: True if argument must be specified + default: Default value if not specified + positional: Argument specified by location, not name + + Raises: + ValueError: the argument already exists or is invalid + + Returns: + The newly-created argument + """ + if IsString(names): names = [names] + + names = [name.lower() for name in names] + + for name in names: + if name in self.arg_dict: + raise ValueError("%s is already an argument"%name) + + if (positional and required and + [arg for arg in self.args if arg.positional] and + not [arg for arg in self.args if arg.positional][-1].required): + raise ValueError( + "A required positional argument may not follow an optional one.") + + arg = Command.Argument(names, helptext, type, metaname, + required, default, positional) + + self.args.append(arg) + + for name in names: + self.arg_dict[name] = arg + + return arg + + def GetArgument(self, name): + """Return an argument from a name.""" + return self.arg_dict[name.lower()] + + def AddMutualExclusion(self, args): + """Specifies that a list of arguments are mutually exclusive.""" + if len(args) < 2: + raise ValueError("At least two arguments must be specified.") + + args = [arg.lower() for arg in args] + + for index in xrange(len(args)-1): + for index2 in xrange(index+1, len(args)): + self.arg_dict[args[index]].AddMutualExclusion(self.arg_dict[args[index2]]) + + def AddDependency(self, dependent, depends_on): + """Specifies that one argument may only be present if another is. + + Args: + dependent: the name of the dependent argument + depends_on: the name of the argument on which it depends + """ + self.arg_dict[dependent.lower()].AddDependency( + self.arg_dict[depends_on.lower()]) + + def AddMutualDependency(self, args): + """Specifies that a list of arguments are all mutually dependent.""" + if len(args) < 2: + raise ValueError("At least two arguments must be specified.") + + args = [arg.lower() for arg in args] + + for (arg1, arg2) in [(arg1, arg2) for arg1 in args for arg2 in args]: + if arg1 == arg2: continue + self.arg_dict[arg1].AddDependency(self.arg_dict[arg2]) + + def AddRequiredGroup(self, args): + """Specifies that at least one of the named arguments must be present.""" + if len(args) < 2: + raise ValueError("At least two arguments must be in a required group.") + + args = [self.arg_dict[arg.lower()] for arg in args] + + self.required_groups.append(args) + + def ParseArguments(self): + """Given a command line, parse and validate the arguments.""" + + # reset all the arguments before we parse + for arg in self.args: + arg.present = False + arg.value = None + + self.parse_errors = [] + + # look for arguments remaining on the command line + while len(self.cmdline.rargs): + try: + self.ParseNextArgument() + except ParseError, e: + self.parse_errors.append(e.args[0]) + + # after all the arguments are parsed, check for problems + for arg in self.args: + if not arg.present and arg.required: + self.parse_errors.append("'%s': required parameter was missing" + % arg.names[0]) + + if not arg.present and arg.default: + arg.present = True + arg.value = arg.default + + if arg.present: + for mutex in arg.mutex: + if mutex.present: + self.parse_errors.append( + "'%s', '%s': arguments are mutually exclusive" % + (arg.argstr, mutex.argstr)) + + for depend in arg.depends: + if not depend.present: + self.parse_errors.append("'%s': '%s' must be specified as well" % + (arg.argstr, depend.names[0])) + + # check for required groups + for group in self.required_groups: + if not [arg for arg in group if arg.present]: + self.parse_errors.append("%s: at least one must be present" % + (", ".join(["'%s'" % arg.names[-1] for arg in group]))) + + # if we have any validators, invoke them + if not self.parse_errors and self.validator: + try: + self.validator(self) + except ParseError, e: + self.parse_errors.append(e.args[0]) + + # Helper methods so you can treat the command like a dict + def __getitem__(self, key): + arg = self.arg_dict[key.lower()] + + if arg.type == 'flag': + return arg.present + else: + return arg.value + + def __iter__(self): + return [arg for arg in self.args if arg.present].__iter__() + + def ArgumentPresent(self, key): + """Tests if an argument exists and has been specified.""" + return key.lower() in self.arg_dict and self.arg_dict[key.lower()].present + + def __contains__(self, key): + return self.ArgumentPresent(key) + + def ParseNextArgument(self): + """Find the next argument in the command line and parse it.""" + arg = None + value = None + argstr = self.cmdline.rargs.pop(0) + + # First check: is this a literal argument? + if argstr.lower() in self.arg_dict: + arg = self.arg_dict[argstr.lower()] + if arg.type in Command.Argument.TYPES_WITH_VALUES: + if len(self.cmdline.rargs): + value = self.cmdline.rargs.pop(0) + + # Second check: is this of the form "arg=val" or "arg:val"? + if arg is None: + delimiter_pos = -1 + + for delimiter in [':', '=']: + pos = argstr.find(delimiter) + if pos >= 0: + if delimiter_pos < 0 or pos < delimiter_pos: + delimiter_pos = pos + + if delimiter_pos >= 0: + testarg = argstr[:delimiter_pos] + testval = argstr[delimiter_pos+1:] + + if testarg.lower() in self.arg_dict: + arg = self.arg_dict[testarg.lower()] + argstr = testarg + value = testval + + # Third check: does this begin an argument? + if arg is None: + for key in self.arg_dict.iterkeys(): + if (len(key) < len(argstr) and + self.arg_dict[key].type in Command.Argument.TYPES_WITH_VALUES and + argstr[:len(key)].lower() == key): + value = argstr[len(key):] + argstr = argstr[:len(key)] + arg = self.arg_dict[argstr] + + # Fourth check: do we have any positional arguments available? + if arg is None: + for positional_arg in [ + testarg for testarg in self.args if testarg.positional]: + if not positional_arg.present: + arg = positional_arg + value = argstr + argstr = positional_arg.names[0] + break + + # Push the retrieved argument/value onto the largs stack + if argstr: self.cmdline.largs.append(argstr) + if value: self.cmdline.largs.append(value) + + # If we've made it this far and haven't found an arg, give up + if arg is None: + raise ParseError("Unknown argument: '%s'" % argstr) + + # Convert the value, if necessary + if arg.type in Command.Argument.TYPES_WITH_VALUES and value is None: + raise ParseError("Argument '%s' requires a value" % argstr) + + if value is not None: + value = self.StringToValue(value, arg.type, argstr) + + arg.argstr = argstr + arg.value = value + arg.present = True + + # end method ParseNextArgument + + def StringToValue(self, value, type, argstr): + """Convert a string from the command line to a value type.""" + try: + if type == 'string': + pass # leave it be + + elif type == 'int': + try: + value = int(value) + except ValueError: + raise ParseError + + elif type == 'readfile': + if not os.path.isfile(value): + raise ParseError("'%s': '%s' does not exist" % (argstr, value)) + + elif type == 'coords': + try: + value = [int(val) for val in + re.match("\(\s*(\d+)\s*\,\s*(\d+)\s*\)\s*\Z", value). + groups()] + except AttributeError: + raise ParseError + + else: + raise ValueError("Unknown type: '%s'" % type) + + except ParseError, e: + # The bare exception is raised in the generic case; more specific errors + # will arrive with arguments and should just be reraised + if not e.args: + e = ParseError("'%s': unable to convert '%s' to type '%s'" % + (argstr, value, type)) + raise e + + return value + + def SortArgs(self): + """Returns a method that can be passed to sort() to sort arguments.""" + + def ArgSorter(arg1, arg2): + """Helper for sorting arguments in the usage string. + + Positional arguments come first, then required arguments, + then optional arguments. Pylint demands this trivial function + have both Args: and Returns: sections, sigh. + + Args: + arg1: the first argument to compare + arg2: the second argument to compare + + Returns: + -1 if arg1 should be sorted first, +1 if it should be sorted second, + and 0 if arg1 and arg2 have the same sort level. + """ + return ((arg2.positional-arg1.positional)*2 + + (arg2.required-arg1.required)) + return ArgSorter + + def GetUsageString(self, width=80, name=None): + """Gets a string describing how the command is used.""" + if name is None: name = self.names[0] + + initial_indent = "Usage: %s %s " % (self.cmdline.prog, name) + subsequent_indent = " " * len(initial_indent) + + sorted_args = self.args[:] + sorted_args.sort(self.SortArgs()) + + return textwrap.fill( + " ".join([arg.GetUsageString() for arg in sorted_args]), width, + initial_indent=initial_indent, + subsequent_indent=subsequent_indent) + + def GetHelpString(self, width=80): + """Returns a list of help strings for all this command's arguments.""" + sorted_args = self.args[:] + sorted_args.sort(self.SortArgs()) + + return "\n".join([arg.GetHelpString(width) for arg in sorted_args]) + + # end class Command + + +class CommandLine(object): + """Parse a command line, extracting a command and its arguments.""" + + def __init__(self): + self.commands = [] + self.cmd_dict = {} + + # Add the help command to the parser + help_cmd = self.AddCommand(["help", "--help", "-?", "-h"], + "Displays help text for a command", + ValidateHelpCommand, + DoHelpCommand) + + help_cmd.AddArgument( + "command", "Command to retrieve help for", positional=True) + help_cmd.AddArgument( + "--width", "Width of the output", type='int', default=80) + + self.Exit = sys.exit # override this if you don't want the script to halt + # on error or on display of help + + self.out = sys.stdout # override these if you want to redirect + self.err = sys.stderr # output or error messages + + def AddCommand(self, names, helptext, validator=None, impl=None): + """Add a new command to the parser. + + Args: + names: command name, or list of synonyms + helptext: brief string description of the command + validator: method to validate a command's arguments + impl: callable to be invoked when command is called + + Raises: + ValueError: raised if command already added + + Returns: + The new command + """ + if IsString(names): names = [names] + + for name in names: + if name in self.cmd_dict: + raise ValueError("%s is already a command"%name) + + cmd = Command(names, helptext, validator, impl) + cmd.cmdline = self + + self.commands.append(cmd) + for name in names: + self.cmd_dict[name.lower()] = cmd + + return cmd + + def GetUsageString(self): + """Returns simple usage instructions.""" + return "Type '%s help' for usage." % self.prog + + def ParseCommandLine(self, argv=None, prog=None, execute=True): + """Does the work of parsing a command line. + + Args: + argv: list of arguments, defaults to sys.args[1:] + prog: name of the command, defaults to the base name of the script + execute: if false, just parse, don't invoke the 'impl' member + + Returns: + The command that was executed + """ + if argv is None: argv = sys.argv[1:] + if prog is None: prog = os.path.basename(sys.argv[0]).split('.')[0] + + # Store off our parameters, we may need them someday + self.argv = argv + self.prog = prog + + # We shouldn't be invoked without arguments, that's just lame + if not len(argv): + self.out.writelines(self.GetUsageString()) + self.Exit() + return None # in case the client overrides Exit + + # Is it a valid command? + self.command_string = argv[0].lower() + if not self.command_string in self.cmd_dict: + self.err.write("Unknown command: '%s'\n\n" % self.command_string) + self.out.write(self.GetUsageString()) + self.Exit() + return None # in case the client overrides Exit + + self.command = self.cmd_dict[self.command_string] + + # "rargs" = remaining (unparsed) arguments + # "largs" = already parsed, "left" of the read head + self.rargs = argv[1:] + self.largs = [] + + # let the command object do the parsing + self.command.ParseArguments() + + if self.command.parse_errors: + # there were errors, output the usage string and exit + self.err.write(self.command.GetUsageString()+"\n\n") + self.err.write("\n".join(self.command.parse_errors)) + self.err.write("\n\n") + + self.Exit() + + elif execute and self.command.impl: + self.command.impl(self.command) + + return self.command + + def __getitem__(self, key): + return self.cmd_dict[key] + + def __iter__(self): + return self.cmd_dict.__iter__() + + +def ValidateHelpCommand(command): + """Checks to make sure an argument to 'help' is a valid command.""" + if 'command' in command and command['command'] not in command.cmdline: + raise ParseError("'%s': unknown command" % command['command']) + + +def DoHelpCommand(command): + """Executed when the command is 'help'.""" + out = command.cmdline.out + width = command['--width'] + + if 'command' not in command: + out.write(command.GetUsageString()) + out.write("\n\n") + + indent = 5 + gutter = 2 + + command_width = ( + max([len(cmd.names[0]) for cmd in command.cmdline.commands]) + gutter) + + for cmd in command.cmdline.commands: + cmd_name = cmd.names[0] + + initial_indent = (" "*indent + cmd_name + " "* + (command_width+gutter-len(cmd_name))) + subsequent_indent = " "*(indent+command_width+gutter) + + out.write(textwrap.fill(cmd.helptext, width, + initial_indent=initial_indent, + subsequent_indent=subsequent_indent)) + out.write("\n") + + out.write("\n") + + else: + help_cmd = command.cmdline[command['command']] + + out.write(textwrap.fill(help_cmd.helptext, width)) + out.write("\n\n") + out.write(help_cmd.GetUsageString(width=width)) + out.write("\n\n") + out.write(help_cmd.GetHelpString(width=width)) + out.write("\n") + + command.cmdline.Exit() + +if __name__ == "__main__": + # If we're invoked rather than imported, run some tests + cmdline = CommandLine() + + # Since we're testing, override Exit() + def TestExit(): + pass + cmdline.Exit = TestExit + + # Actually, while we're at it, let's override error output too + cmdline.err = open(os.path.devnull, "w") + + test = cmdline.AddCommand(["test", "testa", "testb"], "test command") + test.AddArgument(["-i", "--int", "--integer", "--optint", "--optionalint"], + "optional integer parameter", type='int') + test.AddArgument("--reqint", "required integer parameter", type='int', + required=True) + test.AddArgument("pos1", "required positional argument", positional=True, + required=True) + test.AddArgument("pos2", "optional positional argument", positional=True) + test.AddArgument("pos3", "another optional positional arg", + positional=True) + + # mutually dependent arguments + test.AddArgument("--mutdep1", "mutually dependent parameter 1") + test.AddArgument("--mutdep2", "mutually dependent parameter 2") + test.AddArgument("--mutdep3", "mutually dependent parameter 3") + test.AddMutualDependency(["--mutdep1", "--mutdep2", "--mutdep3"]) + + # mutually exclusive arguments + test.AddArgument("--mutex1", "mutually exclusive parameter 1") + test.AddArgument("--mutex2", "mutually exclusive parameter 2") + test.AddArgument("--mutex3", "mutually exclusive parameter 3") + test.AddMutualExclusion(["--mutex1", "--mutex2", "--mutex3"]) + + # dependent argument + test.AddArgument("--dependent", "dependent argument") + test.AddDependency("--dependent", "--int") + + # other argument types + test.AddArgument("--file", "filename argument", type='readfile') + test.AddArgument("--coords", "coordinate argument", type='coords') + test.AddArgument("--flag", "flag argument", type='flag') + + test.AddArgument("--req1", "part of a required group", type='flag') + test.AddArgument("--req2", "part 2 of a required group", type='flag') + + test.AddRequiredGroup(["--req1", "--req2"]) + + # a few failure cases + exception_cases = """ + test.AddArgument("failpos", "can't have req'd pos arg after opt", + positional=True, required=True) ++++ + test.AddArgument("--int", "this argument already exists") ++++ + test.AddDependency("--int", "--doesntexist") ++++ + test.AddMutualDependency(["--doesntexist", "--mutdep2"]) ++++ + test.AddMutualExclusion(["--doesntexist", "--mutex2"]) ++++ + test.AddArgument("--reqflag", "required flag", required=True, type='flag') ++++ + test.AddRequiredGroup(["--req1", "--doesntexist"]) +""" + for exception_case in exception_cases.split("+++"): + try: + exception_case = exception_case.strip() + exec exception_case # yes, I'm using exec, it's just for a test. + except ValueError: + # this is expected + pass + except KeyError: + # ...and so is this + pass + else: + print ("FAILURE: expected an exception for '%s'" + " and didn't get it" % exception_case) + + # Let's do some parsing! first, the minimal success line: + MIN = "test --reqint 123 param1 --req1 " + + # tuples of (command line, expected error count) + test_lines = [ + ("test --int 3 foo --req1", 1), # missing required named parameter + ("test --reqint 3 --req1", 1), # missing required positional parameter + (MIN, 0), # success! + ("test param1 --reqint 123 --req1", 0), # success, order shouldn't matter + ("test param1 --reqint 123 --req2", 0), # success, any of required group ok + (MIN+"param2", 0), # another positional parameter is okay + (MIN+"param2 param3", 0), # and so are three + (MIN+"param2 param3 param4", 1), # but four are just too many + (MIN+"--int", 1), # where's the value? + (MIN+"--int 456", 0), # this is fine + (MIN+"--int456", 0), # as is this + (MIN+"--int:456", 0), # and this + (MIN+"--int=456", 0), # and this + (MIN+"--file c:\\windows\\system32\\kernel32.dll", 0), # yup + (MIN+"--file c:\\thisdoesntexist", 1), # nope + (MIN+"--mutdep1 a", 2), # no! + (MIN+"--mutdep2 b", 2), # also no! + (MIN+"--mutdep3 c", 2), # dream on! + (MIN+"--mutdep1 a --mutdep2 b", 2), # almost! + (MIN+"--mutdep1 a --mutdep2 b --mutdep3 c", 0), # yes + (MIN+"--mutex1 a", 0), # yes + (MIN+"--mutex2 b", 0), # yes + (MIN+"--mutex3 c", 0), # fine + (MIN+"--mutex1 a --mutex2 b", 1), # not fine + (MIN+"--mutex1 a --mutex2 b --mutex3 c", 3), # even worse + (MIN+"--dependent 1", 1), # no + (MIN+"--dependent 1 --int 2", 0), # ok + (MIN+"--int abc", 1), # bad type + (MIN+"--coords abc", 1), # also bad + (MIN+"--coords (abc)", 1), # getting warmer + (MIN+"--coords (abc,def)", 1), # missing something + (MIN+"--coords (123)", 1), # ooh, so close + (MIN+"--coords (123,def)", 1), # just a little farther + (MIN+"--coords (123,456)", 0), # finally! + ("test --int 123 --reqint=456 foo bar --coords(42,88) baz --req1", 0) + ] + + badtests = 0 + + for (test, expected_failures) in test_lines: + cmdline.ParseCommandLine([x.strip() for x in test.strip().split(" ")]) + + if not len(cmdline.command.parse_errors) == expected_failures: + print "FAILED:\n issued: '%s'\n expected: %d\n received: %d\n\n" % ( + test, expected_failures, len(cmdline.command.parse_errors)) + badtests += 1 + + print "%d failed out of %d tests" % (badtests, len(test_lines)) + + cmdline.ParseCommandLine(["help", "test"]) + diff --git a/tools/site_compare/commands/__init__.py b/tools/site_compare/commands/__init__.py new file mode 100644 index 0000000..a699508 --- /dev/null +++ b/tools/site_compare/commands/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/Python2.4 + diff --git a/tools/site_compare/commands/compare2.py b/tools/site_compare/commands/compare2.py new file mode 100644 index 0000000..6dc00c7 --- /dev/null +++ b/tools/site_compare/commands/compare2.py @@ -0,0 +1,196 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""SiteCompare command to invoke the same page in two versions of a browser. + +Does the easiest compatibility test: equality comparison between two different +versions of the same browser. Invoked with a series of command line options +that specify which URLs to check, which browser to use, where to store results, +etc. +""" + +import os # Functions for walking the directory tree +import tempfile # Get a temporary directory to hold intermediates + +import command_line +import drivers # Functions for driving keyboard/mouse/windows, OS-specific +import operators # Functions that, given two bitmaps as input, produce + # output depending on the performance of an operation +import scrapers # Functions that know how to capture a render from + # particular browsers + + +def CreateCommand(cmdline): + """Inserts the command and arguments into a command line for parsing.""" + cmd = cmdline.AddCommand( + ["compare2"], + "Compares the output of two browsers on the same URL or list of URLs", + ValidateCompare2, + ExecuteCompare2) + + cmd.AddArgument( + ["-b1", "--browser1"], "Full path to first browser's executable", + type="readfile", metaname="PATH", required=True) + cmd.AddArgument( + ["-b2", "--browser2"], "Full path to second browser's executable", + type="readfile", metaname="PATH", required=True) + cmd.AddArgument( + ["-b", "--browser"], "Which browser to use", type="string", + default="chrome") + cmd.AddArgument( + ["-b1v", "--browser1ver"], "Version of first browser", metaname="VERSION") + cmd.AddArgument( + ["-b2v", "--browser2ver"], "Version of second browser", metaname="VERSION") + cmd.AddArgument( + ["-b1n", "--browser1name"], "Optional name for first browser (used in " + "directory to hold intermediate files)", metaname="NAME") + cmd.AddArgument( + ["-b2n", "--browser2name"], "Optional name for second browser (used in " + "directory to hold intermediate files)", metaname="NAME") + cmd.AddArgument( + ["-o", "--outdir"], "Directory to store scrape files", metaname="DIR") + cmd.AddArgument( + ["-u", "--url"], "URL to compare") + cmd.AddArgument( + ["-l", "--list"], "List of URLs to compare", type="readfile") + cmd.AddMutualExclusion(["--url", "--list"]) + cmd.AddArgument( + ["-s", "--startline"], "First line of URL list", type="int") + cmd.AddArgument( + ["-e", "--endline"], "Last line of URL list (exclusive)", type="int") + cmd.AddArgument( + ["-c", "--count"], "Number of lines of URL file to use", type="int") + cmd.AddDependency("--startline", "--list") + cmd.AddRequiredGroup(["--url", "--list"]) + cmd.AddDependency("--endline", "--list") + cmd.AddDependency("--count", "--list") + cmd.AddMutualExclusion(["--count", "--endline"]) + cmd.AddDependency("--count", "--startline") + cmd.AddArgument( + ["-t", "--timeout"], "Amount of time (seconds) to wait for browser to " + "finish loading", + type="int", default=60) + cmd.AddArgument( + ["-log", "--logfile"], "File to write output", type="string", required=True) + cmd.AddArgument( + ["-sz", "--size"], "Browser window size", default=(800, 600), type="coords") + cmd.AddArgument( + ["-m", "--maskdir"], "Path that holds masks to use for comparison") + cmd.AddArgument( + ["-d", "--diffdir"], "Path to hold the difference of comparisons that fail") + + +def ValidateCompare2(command): + """Validate the arguments to compare2. Raises ParseError if failed.""" + executables = [".exe", ".com", ".bat"] + if (os.path.splitext(command["--browser1"])[1].lower() not in executables or + os.path.splitext(command["--browser2"])[1].lower() not in executables): + raise command_line.ParseError("Browser filename must be an executable") + + +def ExecuteCompare2(command): + """Executes the Compare2 command.""" + if command["--url"]: + url_list = [command["--url"]] + else: + startline = command["--startline"] + if command["--count"]: + endline = startline+command["--count"] + else: + endline = command["--endline"] + url_list = [url.strip() for url in + open(command["--list"], "r").readlines()[startline:endline]] + + log_file = open(command["--logfile"], "w") + + outdir = command["--outdir"] + if not outdir: outdir = tempfile.gettempdir() + + scrape_info_list = [] + + class ScrapeInfo(object): + """Helper class to hold information about a scrape.""" + __slots__ = ["browser_path", "scraper", "outdir", "result"] + + for index in xrange(1, 3): + scrape_info = ScrapeInfo() + scrape_info.browser_path = command["--browser%d" % index] + scrape_info.scraper = scrapers.GetScraper( + (command["--browser"], command["--browser%dver" % index])) + + if command["--browser%dname" % index]: + scrape_info.outdir = os.path.join(outdir, + command["--browser%dname" % index]) + else: + scrape_info.outdir = os.path.join(outdir, str(index)) + + drivers.windowing.PreparePath(scrape_info.outdir) + scrape_info_list.append(scrape_info) + + compare = operators.GetOperator("equals_with_mask") + + for url in url_list: + success = True + + for scrape_info in scrape_info_list: + scrape_info.result = scrape_info.scraper.Scrape( + [url], scrape_info.outdir, command["--size"], (0, 0), + command["--timeout"], path=scrape_info.browser_path) + + if not scrape_info.result: + scrape_info.result = "success" + else: + success = False + + result = "unknown" + + if success: + result = "equal" + + file1 = drivers.windowing.URLtoFilename( + url, scrape_info_list[0].outdir, ".bmp") + file2 = drivers.windowing.URLtoFilename( + url, scrape_info_list[1].outdir, ".bmp") + + comparison_result = compare.Compare(file1, file2, + maskdir=command["--maskdir"]) + + if comparison_result is not None: + result = "not-equal" + + if command["--diffdir"]: + comparison_result[1].save( + drivers.windowing.URLtoFilename(url, command["--diffdir"], ".bmp")) + + # TODO(jhaas): maybe use the logging module rather than raw file writes + log_file.write("%s %s %s %s\n" % (url, + scrape_info_list[0].result, + scrape_info_list[1].result, + result)) diff --git a/tools/site_compare/commands/maskmaker.py b/tools/site_compare/commands/maskmaker.py new file mode 100644 index 0000000..95bdeb45 --- /dev/null +++ b/tools/site_compare/commands/maskmaker.py @@ -0,0 +1,298 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Component for automatically creating masks of changing areas of a website. + +Works by repeated invokation of a browser and scraping of the resulting page. +Areas that differ will be added to the auto-generated mask. The mask generator +considers the mask complete when further scrapes fail to produce any differences +in the mask. +""" + +import os # Functions for walking the directory tree +import tempfile # Get a temporary directory to hold intermediates +import time # Used for sleep() and naming masks by time + +import command_line +import drivers +from PIL import Image +from PIL import ImageChops +import scrapers + + +def CreateCommand(cmdline): + """Inserts the command and arguments into a command line for parsing.""" + cmd = cmdline.AddCommand( + ["maskmaker"], + "Automatically generates a mask from a list of URLs", + ValidateMaskmaker, + ExecuteMaskmaker) + + cmd.AddArgument( + ["-bp", "--browserpath"], "Full path to browser's executable", + type="readfile", metaname="PATH") + cmd.AddArgument( + ["-b", "--browser"], "Which browser to use", type="string", + default="chrome") + cmd.AddArgument( + ["-bv", "--browserver"], "Version of the browser", metaname="VERSION") + cmd.AddArgument( + ["-o", "--outdir"], "Directory to store generated masks", metaname="DIR", + required=True) + cmd.AddArgument( + ["-u", "--url"], "URL to compare") + cmd.AddArgument( + ["-l", "--list"], "List of URLs to compare", type="readfile") + cmd.AddMutualExclusion(["--url", "--list"]) + cmd.AddArgument( + ["-s", "--startline"], "First line of URL list", type="int") + cmd.AddArgument( + ["-e", "--endline"], "Last line of URL list (exclusive)", type="int") + cmd.AddArgument( + ["-c", "--count"], "Number of lines of URL file to use", type="int") + cmd.AddDependency("--startline", "--list") + cmd.AddRequiredGroup(["--url", "--list"]) + cmd.AddDependency("--endline", "--list") + cmd.AddDependency("--count", "--list") + cmd.AddMutualExclusion(["--count", "--endline"]) + cmd.AddDependency("--count", "--startline") + cmd.AddArgument( + ["-t", "--timeout"], "Amount of time (seconds) to wait for browser to " + "finish loading", + type="int", default=60) + cmd.AddArgument( + ["-w", "--wait"], + "Amount of time (in seconds) to wait between successive scrapes", + type="int", default=60) + cmd.AddArgument( + ["-sc", "--scrapes"], + "Number of successive scrapes which must result in no change to a mask " + "before mask creation is considered complete", type="int", default=10) + cmd.AddArgument( + ["-sz", "--size"], "Browser window size", default=(800, 600), type="coords") + cmd.AddArgument(["-sd", "--scrapedir"], "Directory to store scrapes") + cmd.AddArgument( + ["-gu", "--giveup"], + "Number of times to scrape before giving up", type="int", default=50) + cmd.AddArgument( + ["-th", "--threshhold"], + "Percentage of different pixels (0-100) above which the scrape will be" + "discarded and the mask not updated.", type="int", default=100) + cmd.AddArgument( + ["--er", "--errors"], + "Number of times a scrape can fail before giving up on the URL.", + type="int", default=1) + + +def ValidateMaskmaker(command): + """Validate the arguments to maskmaker. Raises ParseError if failed.""" + executables = [".exe", ".com", ".bat"] + if command["--browserpath"]: + if os.path.splitext(command["--browserpath"])[1].lower() not in executables: + raise command_line.ParseError("Browser filename must be an executable") + + +def ExecuteMaskmaker(command): + """Performs automatic mask generation.""" + + # Get the list of URLs to generate masks for + class MaskmakerURL(object): + """Helper class for holding information about a URL passed to maskmaker.""" + __slots__ = ['url', 'consecutive_successes', 'errors'] + def __init__(self, url): + self.url = url + self.consecutive_successes = 0 + self.errors = 0 + + if command["--url"]: + url_list = [MaskmakerURL(command["--url"])] + else: + startline = command["--startline"] + if command["--count"]: + endline = startline+command["--count"] + else: + endline = command["--endline"] + url_list = [MaskmakerURL(url.strip()) for url in + open(command["--list"], "r").readlines()[startline:endline]] + + complete_list = [] + error_list = [] + + outdir = command["--outdir"] + scrapes = command["--scrapes"] + errors = command["--errors"] + size = command["--size"] + scrape_pass = 0 + + scrapedir = command["--scrapedir"] + if not scrapedir: scrapedir = tempfile.gettempdir() + + # Get the scraper + scraper = scrapers.GetScraper((command["--browser"], command["--browserver"])) + + # Repeatedly iterate through the list of URLs until either every URL has + # a successful mask or too many errors, or we've exceeded the giveup limit + while url_list and scrape_pass < command["--giveup"]: + # Scrape each URL + for url in url_list: + print "Processing %r..." % url.url + mask_filename = drivers.windowing.URLtoFilename(url.url, outdir, ".bmp") + + # Load the existing mask. This is in a loop so we can try to recover + # from error conditions + while True: + try: + mask = Image.open(mask_filename) + if mask.size != size: + print " %r already exists and is the wrong size! (%r vs %r)" % ( + mask_filename, mask.size, size) + mask_filename = "%s_%r%s" % ( + mask_filename[:-4], size, mask_filename[-4:]) + print " Trying again as %r..." % mask_filename + continue + break + except IOError: + print " %r does not exist, creating" % mask_filename + mask = Image.new("1", size, 1) + mask.save(mask_filename) + + # Find the stored scrape path + mask_scrape_dir = os.path.join( + scrapedir, os.path.splitext(os.path.basename(mask_filename))[0]) + drivers.windowing.PreparePath(mask_scrape_dir) + + # Find the baseline image + mask_scrapes = os.listdir(mask_scrape_dir) + mask_scrapes.sort() + + if not mask_scrapes: + print " No baseline image found, mask will not be updated" + baseline = None + else: + baseline = Image.open(os.path.join(mask_scrape_dir, mask_scrapes[0])) + + mask_scrape_filename = os.path.join(mask_scrape_dir, + time.strftime("%y%m%d-%H%M%S.bmp")) + + # Do the scrape + result = scraper.Scrape( + [url.url], mask_scrape_dir, size, (0, 0), + command["--timeout"], path=command["--browserpath"], + filename=mask_scrape_filename) + + if result: + # Return value other than None means an error + print " Scrape failed with error '%r'" % result + url.errors += 1 + if url.errors >= errors: + print " ** Exceeded maximum error count for this URL, giving up" + continue + + # Load the new scrape + scrape = Image.open(mask_scrape_filename) + + # Calculate the difference between the new scrape and the baseline, + # subject to the current mask + if baseline: + diff = ImageChops.multiply(ImageChops.difference(scrape, baseline), + mask.convert(scrape.mode)) + + # If the difference is none, there's nothing to update + if max(diff.getextrema()) == (0, 0): + print " Scrape identical to baseline, no change in mask" + url.consecutive_successes += 1 + if url.consecutive_successes >= scrapes: + print " ** No change for %r scrapes, done!" % scrapes + else: + # convert the difference to black and white, then change all + # black pixels (where the scrape and the baseline were identical) + # to white, all others (where the scrape and the baseline differed) + # to black. + # + # Since the below command is a little unclear, here's how it works. + # 1. convert("L") converts the RGB image to grayscale + # 2. point() maps grayscale values (or the individual channels) + # of an RGB image) to different ones. Because it operates on + # individual channels, the grayscale conversion from step 1 + # is necessary. + # 3. The "1" second parameter to point() outputs the result as + # a monochrome bitmap. If the original RGB image were converted + # directly to monochrome, PIL would dither it. + diff = diff.convert("L").point([255]+[0]*255, "1") + + # count the number of different pixels + diff_pixels = diff.getcolors()[0][0] + + # is this too much? + diff_pixel_percent = diff_pixels * 100.0 / (mask.size[0]*mask.size[1]) + if diff_pixel_percent > command["--threshhold"]: + print (" Scrape differed from baseline by %.2f percent, ignoring" + % diff_pixel_percent) + else: + print " Scrape differed in %d pixels, updating mask" % diff_pixels + mask = ImageChops.multiply(mask, diff) + mask.save(mask_filename) + + # reset the number of consecutive "good" scrapes + url.consecutive_successes = 0 + + # Remove URLs whose mask is deemed done + complete_list.extend( + [url for url in url_list if url.consecutive_successes >= scrapes]) + error_list.extend( + [url for url in url_list if url.errors >= errors]) + url_list = [ + url for url in url_list if + url.consecutive_successes < scrapes and + url.errors < errors] + + scrape_pass += 1 + print "**Done with scrape pass %d\n" % scrape_pass + + if scrape_pass >= command["--giveup"]: + print "**Exceeded giveup threshhold. Giving up." + else: + print "Waiting %d seconds..." % command["--wait"] + time.sleep(command["--wait"]) + + print + print "*** MASKMAKER COMPLETE ***" + print "Summary report:" + print " %d masks successfully generated" % len(complete_list) + for url in complete_list: + print " ", url.url + print " %d masks failed with too many errors" % len(error_list) + for url in error_list: + print " ", url.url + if scrape_pass >= command["--giveup"]: + print (" %d masks were not completed before " + "reaching the giveup threshhold" % len(url_list)) + for url in url_list: + print " ", url.url diff --git a/tools/site_compare/commands/measure.py b/tools/site_compare/commands/measure.py new file mode 100644 index 0000000..477db57 --- /dev/null +++ b/tools/site_compare/commands/measure.py @@ -0,0 +1,78 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Command for measuring how long pages take to load in a browser. + +Prerequisites: + 1. The command_line package from tools/site_compare + 2. Either the IE BHO or Firefox extension (or both) + +Installation: + 1. Build the IE BHO, or call regsvr32 on a prebuilt binary + 2. Add a file called "measurepageloadtimeextension@google.com" to + the default Firefox profile directory under extensions, containing + the path to the Firefox extension root + +Invoke with the command line arguments as documented within +the command line. +""" + +import command_line +import win32process + +from drivers import windowing +from utils import browser_iterate + +def CreateCommand(cmdline): + """Inserts the command and arguments into a command line for parsing.""" + cmd = cmdline.AddCommand( + ["measure"], + "Measures how long a series of URLs takes to load in one or more browsers.", + None, + ExecuteMeasure) + + browser_iterate.SetupIterationCommandLine(cmd) + cmd.AddArgument( + ["-log", "--logfile"], "File to write output", type="string", required=True) + + +def ExecuteMeasure(command): + """Executes the Measure command.""" + + def LogResult(url, proc, wnd, result): + """Write the result of the browse to the log file.""" + log_file.write(result) + + log_file = open(command["--logfile"], "w") + + browser_iterate.Iterate(command, LogResult) + + # Close the log file and return. We're done. + log_file.close() diff --git a/tools/site_compare/commands/scrape.py b/tools/site_compare/commands/scrape.py new file mode 100644 index 0000000..a9b3398 --- /dev/null +++ b/tools/site_compare/commands/scrape.py @@ -0,0 +1,85 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Command for scraping images from a URL or list of URLs. + +Prerequisites: + 1. The command_line package from tools/site_compare + 2. Either the IE BHO or Firefox extension (or both) + +Installation: + 1. Build the IE BHO, or call regsvr32 on a prebuilt binary + 2. Add a file called "measurepageloadtimeextension@google.com" to + the default Firefox profile directory under extensions, containing + the path to the Firefox extension root + +Invoke with the command line arguments as documented within +the command line. +""" + +import command_line + +from drivers import windowing +from utils import browser_iterate + +def CreateCommand(cmdline): + """Inserts the command and arguments into a command line for parsing.""" + cmd = cmdline.AddCommand( + ["scrape"], + "Scrapes an image from a URL or series of URLs.", + None, + ExecuteScrape) + + browser_iterate.SetupIterationCommandLine(cmd) + cmd.AddArgument( + ["-log", "--logfile"], "File to write text output", type="string") + cmd.AddArgument( + ["-out", "--outdir"], "Directory to store scrapes", type="string", required=True) + + +def ExecuteScrape(command): + """Executes the Scrape command.""" + + def ScrapeResult(url, proc, wnd, result): + """Capture and save the scrape.""" + if log_file: log_file.write(result) + + # Scrape the page + image = windowing.ScrapeWindow(wnd) + filename = windowing.URLtoFilename(url, command["--outdir"], ".bmp") + image.save(filename) + + if command["--logfile"]: log_file = open(command["--logfile"], "w") + else: log_file = None + + browser_iterate.Iterate(command, ScrapeResult) + + # Close the log file and return. We're done. + if log_file: log_file.close() diff --git a/tools/site_compare/commands/timeload.py b/tools/site_compare/commands/timeload.py new file mode 100644 index 0000000..a983173 --- /dev/null +++ b/tools/site_compare/commands/timeload.py @@ -0,0 +1,170 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""SiteCompare command to time page loads + +Loads a series of URLs in a series of browsers (and browser versions) +and measures how long the page takes to load in each. Outputs a +comma-delimited file. The first line is "URL,[browser names", each +additional line is a URL follored by comma-delimited times (in seconds), +or the string "timeout" or "crashed". + +""" + +import os # Functions for walking the directory tree +import tempfile # Get a temporary directory to hold intermediates + +import command_line +import drivers # Functions for driving keyboard/mouse/windows, OS-specific +import operators # Functions that, given two bitmaps as input, produce + # output depending on the performance of an operation +import scrapers # Functions that know how to capture a render from + # particular browsers + + +def CreateCommand(cmdline): + """Inserts the command and arguments into a command line for parsing.""" + cmd = cmdline.AddCommand( + ["timeload"], + "Measures how long a series of URLs takes to load in one or more browsers.", + None, + ExecuteTimeLoad) + + cmd.AddArgument( + ["-b", "--browsers"], "List of browsers to use. Comma-separated", + type="string", required=True) + cmd.AddArgument( + ["-bp", "--browserpaths"], "List of paths to browsers. Comma-separated", + type="string", required=False) + cmd.AddArgument( + ["-bv", "--browserversions"], "List of versions of browsers. Comma-separated", + type="string", required=False) + cmd.AddArgument( + ["-u", "--url"], "URL to time") + cmd.AddArgument( + ["-l", "--list"], "List of URLs to time", type="readfile") + cmd.AddMutualExclusion(["--url", "--list"]) + cmd.AddArgument( + ["-s", "--startline"], "First line of URL list", type="int") + cmd.AddArgument( + ["-e", "--endline"], "Last line of URL list (exclusive)", type="int") + cmd.AddArgument( + ["-c", "--count"], "Number of lines of URL file to use", type="int") + cmd.AddDependency("--startline", "--list") + cmd.AddRequiredGroup(["--url", "--list"]) + cmd.AddDependency("--endline", "--list") + cmd.AddDependency("--count", "--list") + cmd.AddMutualExclusion(["--count", "--endline"]) + cmd.AddDependency("--count", "--startline") + cmd.AddArgument( + ["-t", "--timeout"], "Amount of time (seconds) to wait for browser to " + "finish loading", + type="int", default=60) + cmd.AddArgument( + ["-log", "--logfile"], "File to write output", type="string", required=True) + cmd.AddArgument( + ["-sz", "--size"], "Browser window size", default=(800, 600), type="coords") + + +def ExecuteTimeLoad(command): + """Executes the TimeLoad command.""" + browsers = command["--browsers"].split(",") + num_browsers = len(browsers) + + if command["--browserversions"]: + browser_versions = command["--browserversions"].split(",") + else: + browser_versions = [None] * num_browsers + + if command["--browserpaths"]: + browser_paths = command["--browserpaths"].split(",") + else: + browser_paths = [None] * num_browsers + + if len(browser_versions) != num_browsers: + raise ValueError( + "--browserversions must be same length as --browser_paths") + if len(browser_paths) != num_browsers: + raise ValueError( + "--browserversions must be same length as --browser_paths") + + if [b for b in browsers if b not in ["chrome", "ie", "firefox"]]: + raise ValueError("unknown browsers: %r" % b) + + scraper_list = [] + + for b in xrange(num_browsers): + version = browser_versions[b] + if not version: version = None + + scraper = scrapers.GetScraper( (browsers[b], version) ) + if not scraper: + raise ValueError("could not find scraper for (%r, %r)" % + (browsers[b], version)) + scraper_list.append(scraper) + + if command["--url"]: + url_list = [command["--url"]] + else: + startline = command["--startline"] + if command["--count"]: + endline = startline+command["--count"] + else: + endline = command["--endline"] + url_list = [url.strip() for url in + open(command["--list"], "r").readlines()[startline:endline]] + + log_file = open(command["--logfile"], "w") + + log_file.write("URL") + for b in xrange(num_browsers): + log_file.write(",%s" % browsers[b]) + + if browser_versions[b]: log_file.write(" %s" % browser_versions[b]) + log_file.write("\n") + + results = {} + for url in url_list: + results[url] = [None] * num_browsers + + for b in xrange(num_browsers): + result = scraper_list[b].Time(url_list, command["--size"], + command["--timeout"], + path=browser_paths[b]) + + for (url, time) in result: + results[url][b] = time + + # output the results + for url in url_list: + log_file.write(url) + for b in xrange(num_browsers): + log_file.write(",%r" % results[url][b]) + diff --git a/tools/site_compare/drivers/__init__.py b/tools/site_compare/drivers/__init__.py new file mode 100644 index 0000000..befc1353 --- /dev/null +++ b/tools/site_compare/drivers/__init__.py @@ -0,0 +1,15 @@ +#!/usr/bin/python2.4 +# +# Copyright 2007 Google Inc. All Rights Reserved. + +"""Imports a set of drivers appropriate to the current OS.""" + +__author__ = 'jhaas@google.com (Jonathan Haas)' + +import sys + +platform_dir = sys.platform + +keyboard = __import__(platform_dir+".keyboard", globals(), locals(), ['']) +mouse = __import__(platform_dir+".mouse", globals(), locals(), ['']) +windowing = __import__(platform_dir+".windowing", globals(), locals(), ['']) diff --git a/tools/site_compare/drivers/win32/__init__.py b/tools/site_compare/drivers/win32/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/site_compare/drivers/win32/__init__.py diff --git a/tools/site_compare/drivers/win32/keyboard.py b/tools/site_compare/drivers/win32/keyboard.py new file mode 100644 index 0000000..5888318 --- /dev/null +++ b/tools/site_compare/drivers/win32/keyboard.py @@ -0,0 +1,223 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""SiteCompare module for simulating keyboard input. + +This module contains functions that can be used to simulate a user +pressing keys on a keyboard. Support is provided for formatted strings +including special characters to represent modifier keys like CTRL and ALT +""" + +import time # for sleep +import win32api # for keybd_event and VkKeyCode +import win32con # Windows constants + +# TODO(jhaas): Ask the readability guys if this would be acceptable: +# +# from win32con import VK_SHIFT, VK_CONTROL, VK_MENU, VK_LWIN, KEYEVENTF_KEYUP +# +# This is a violation of the style guide but having win32con. everywhere +# is just plain ugly, and win32con is a huge import for just a handful of +# constants + + +def PressKey(down, key): + """Presses or unpresses a key. + + Uses keybd_event to simulate either depressing or releasing + a key + + Args: + down: Whether the key is to be pressed or released + key: Virtual key code of key to press or release + """ + + # keybd_event injects key events at a very low level (it's the + # Windows API keyboard device drivers call) so this is a very + # reliable way of simulating user input + win32api.keybd_event(key, 0, (not down) * win32con.KEYEVENTF_KEYUP) + + +def TypeKey(key, keystroke_time=0): + """Simulate a keypress of a virtual key. + + Args: + key: which key to press + keystroke_time: length of time (in seconds) to "hold down" the key + Note that zero works just fine + + Returns: + None + """ + + # This just wraps a pair of PressKey calls with an intervening delay + PressKey(True, key) + time.sleep(keystroke_time) + PressKey(False, key) + + +def TypeString(string_to_type, + use_modifiers=False, + keystroke_time=0, + time_between_keystrokes=0): + """Simulate typing a string on the keyboard. + + Args: + string_to_type: the string to print + use_modifiers: specifies whether the following modifier characters + should be active: + {abc}: type characters with ALT held down + [abc]: type characters with CTRL held down + \ escapes {}[] and treats these values as literal + standard escape sequences are valid even if use_modifiers is false + \p is "pause" for one second, useful when driving menus + \1-\9 is F-key, \0 is F10 + + TODO(jhaas): support for explicit control of SHIFT, support for + nonprintable keys (F-keys, ESC, arrow keys, etc), + support for explicit control of left vs. right ALT or SHIFT, + support for Windows key + + keystroke_time: length of time (in secondes) to "hold down" the key + time_between_keystrokes: length of time (seconds) to pause between keys + + Returns: + None + """ + + shift_held = win32api.GetAsyncKeyState(win32con.VK_SHIFT ) < 0 + ctrl_held = win32api.GetAsyncKeyState(win32con.VK_CONTROL) < 0 + alt_held = win32api.GetAsyncKeyState(win32con.VK_MENU ) < 0 + + next_escaped = False + escape_chars = { + 'a': '\a', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', 'v': '\v'} + + for char in string_to_type: + vk = None + handled = False + + # Check to see if this is the start or end of a modified block (that is, + # {abc} for ALT-modified keys or [abc] for CTRL-modified keys + if use_modifiers and not next_escaped: + handled = True + if char == "{" and not alt_held: + alt_held = True + PressKey(True, win32con.VK_MENU) + elif char == "}" and alt_held: + alt_held = False + PressKey(False, win32con.VK_MENU) + elif char == "[" and not ctrl_held: + ctrl_held = True + PressKey(True, win32con.VK_CONTROL) + elif char == "]" and ctrl_held: + ctrl_held = False + PressKey(False, win32con.VK_CONTROL) + else: + handled = False + + # If this is an explicitly-escaped character, replace it with the + # appropriate code + if next_escaped and char in escape_chars: char = escape_chars[char] + + # If this is \p, pause for one second. + if next_escaped and char == 'p': + time.sleep(1) + next_escaped = False + handled = True + + # If this is \(d), press F key + if next_escaped and char.isdigit(): + fkey = int(char) + if not fkey: fkey = 10 + next_escaped = False + vk = win32con.VK_F1 + fkey - 1 + + # If this is the backslash, the next character is escaped + if not next_escaped and char == "\\": + next_escaped = True + handled = True + + # If we make it here, it's not a special character, or it's an + # escaped special character which should be treated as a literal + if not handled: + next_escaped = False + if not vk: vk = win32api.VkKeyScan(char) + + # VkKeyScan() returns the scan code in the low byte. The upper + # byte specifies modifiers necessary to produce the given character + # from the given scan code. The only one we're concerned with at the + # moment is Shift. Determine the shift state and compare it to the + # current state... if it differs, press or release the shift key. + new_shift_held = bool(vk & (1<<8)) + + if new_shift_held != shift_held: + PressKey(new_shift_held, win32con.VK_SHIFT) + shift_held = new_shift_held + + # Type the key with the specified length, then wait the specified delay + TypeKey(vk & 0xFF, keystroke_time) + time.sleep(time_between_keystrokes) + + # Release the modifier keys, if held + if shift_held: PressKey(False, win32con.VK_SHIFT) + if ctrl_held: PressKey(False, win32con.VK_CONTROL) + if alt_held: PressKey(False, win32con.VK_MENU) + +if __name__ == "__main__": + # We're being invoked rather than imported. Let's do some tests + + # Press command-R to bring up the Run dialog + PressKey(True, win32con.VK_LWIN) + TypeKey(ord('R')) + PressKey(False, win32con.VK_LWIN) + + # Wait a sec to make sure it comes up + time.sleep(1) + + # Invoke Notepad through the Run dialog + TypeString("wordpad\n") + + # Wait another sec, then start typing + time.sleep(1) + TypeString("This is a test of SiteCompare's Keyboard.py module.\n\n") + TypeString("There should be a blank line above and below this one.\n\n") + TypeString("This line has control characters to make " + "[b]boldface text[b] and [i]italic text[i] and normal text.\n\n", + use_modifiers=True) + TypeString(r"This line should be typed with a visible delay between " + "characters. When it ends, there should be a 3-second pause, " + "then the menu will select File/Exit, then another 3-second " + "pause, then No to exit without saving. Ready?\p\p\p{f}x\p\p\pn", + use_modifiers=True, + keystroke_time=0.05, + time_between_keystrokes=0.05) + +
\ No newline at end of file diff --git a/tools/site_compare/drivers/win32/mouse.py b/tools/site_compare/drivers/win32/mouse.py new file mode 100644 index 0000000..9475f2d --- /dev/null +++ b/tools/site_compare/drivers/win32/mouse.py @@ -0,0 +1,243 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""SiteCompare module for simulating mouse input. + +This module contains functions that can be used to simulate a user +navigating using a pointing device. This includes mouse movement, +clicking with any button, and dragging. +""" + +import time # for sleep + +import win32api # for mouse_event +import win32con # Windows constants +import win32gui # for window functions + + +def ScreenToMouse(pt): + """Convert a value in screen coordinates to mouse coordinates. + + Mouse coordinates are specified as a percentage of screen dimensions, + normalized to 16 bits. 0 represents the far left/top of the screen, + 65535 represents the far right/bottom. This function assumes that + the size of the screen is fixed at module load time and does not change + + Args: + pt: the point of the coords to convert + + Returns: + the converted point + """ + + # Initialize the screen dimensions on first execution. Note that this + # function assumes that the screen dimensions do not change during run. + if not ScreenToMouse._SCREEN_DIMENSIONS: + desktop = win32gui.GetClientRect(win32gui.GetDesktopWindow()) + ScreenToMouse._SCREEN_DIMENSIONS = (desktop[2], desktop[3]) + + return ((65535 * pt[0]) / ScreenToMouse._SCREEN_DIMENSIONS[0], + (65535 * pt[1]) / ScreenToMouse._SCREEN_DIMENSIONS[1]) + +ScreenToMouse._SCREEN_DIMENSIONS = None + + +def PressButton(down, button='left'): + """Simulate a mouse button press or release at the current mouse location. + + Args: + down: whether the button is pressed or released + button: which button is pressed + + Returns: + None + """ + + # Put the mouse_event flags in a convenient dictionary by button + flags = { + 'left': (win32con.MOUSEEVENTF_LEFTUP, win32con.MOUSEEVENTF_LEFTDOWN), + 'middle': (win32con.MOUSEEVENTF_MIDDLEUP, win32con.MOUSEEVENTF_MIDDLEDOWN), + 'right': (win32con.MOUSEEVENTF_RIGHTUP, win32con.MOUSEEVENTF_RIGHTDOWN) + } + + # hit the button + win32api.mouse_event(flags[button][down], 0, 0) + + +def ClickButton(button='left', click_time=0): + """Press and release a mouse button at the current mouse location. + + Args: + button: which button to click + click_time: duration between press and release + + Returns: + None + """ + PressButton(True, button) + time.sleep(click_time) + PressButton(False, button) + + +def DoubleClickButton(button='left', click_time=0, time_between_clicks=0): + """Double-click a mouse button at the current mouse location. + + Args: + button: which button to click + click_time: duration between press and release + time_between_clicks: time to pause between clicks + + Returns: + None + """ + ClickButton(button, click_time) + time.sleep(time_between_clicks) + ClickButton(button, click_time) + + +def MoveToLocation(pos, duration=0, tick=0.01): + """Move the mouse cursor to a specified location, taking the specified time. + + Args: + pos: position (in screen coordinates) to move to + duration: amount of time the move should take + tick: amount of time between successive moves of the mouse + + Returns: + None + """ + # calculate the number of moves to reach the destination + num_steps = (duration/tick)+1 + + # get the current and final mouse position in mouse coords + current_location = ScreenToMouse(win32gui.GetCursorPos()) + end_location = ScreenToMouse(pos) + + # Calculate the step size + step_size = ((end_location[0]-current_location[0])/num_steps, + (end_location[1]-current_location[1])/num_steps) + step = 0 + + while step < num_steps: + # Move the mouse one step + current_location = (current_location[0]+step_size[0], + current_location[1]+step_size[1]) + + # Coerce the coords to int to avoid a warning from pywin32 + win32api.mouse_event( + win32con.MOUSEEVENTF_MOVE|win32con.MOUSEEVENTF_ABSOLUTE, + int(current_location[0]), int(current_location[1])) + + step += 1 + time.sleep(tick) + + +def ClickAtLocation(pos, button='left', click_time=0): + """Simulate a mouse click in a particular location, in screen coordinates. + + Args: + pos: position in screen coordinates (x,y) + button: which button to click + click_time: duration of the click + + Returns: + None + """ + MoveToLocation(pos) + ClickButton(button, click_time) + + +def ClickInWindow(hwnd, offset=None, button='left', click_time=0): + """Simulate a user mouse click in the center of a window. + + Args: + hwnd: handle of the window to click in + offset: where to click, defaults to dead center + button: which button to click + click_time: duration of the click + + Returns: + Nothing + """ + + rect = win32gui.GetClientRect(hwnd) + if offset is None: offset = (rect[2]/2, rect[3]/2) + + # get the screen coordinates of the window's center + pos = win32gui.ClientToScreen(hwnd, offset) + + ClickAtLocation(pos, button, click_time) + + +def DoubleClickInWindow( + hwnd, offset=None, button='left', click_time=0, time_between_clicks=0.1): + """Simulate a user mouse double click in the center of a window. + + Args: + hwnd: handle of the window to click in + offset: where to click, defaults to dead center + button: which button to click + click_time: duration of the clicks + time_between_clicks: length of time to pause between clicks + + Returns: + Nothing + """ + ClickInWindow(hwnd, offset, button, click_time) + time.sleep(time_between_clicks) + ClickInWindow(hwnd, offset, button, click_time) + +if __name__ == "__main__": + # We're being invoked rather than imported. Let's do some tests + + screen_size = win32gui.GetClientRect(win32gui.GetDesktopWindow()) + screen_size = (screen_size[2], screen_size[3]) + + # move the mouse (instantly) to the upper right corner + MoveToLocation((screen_size[0], 0)) + + # move the mouse (over five seconds) to the lower left corner + MoveToLocation((0, screen_size[1]), 5) + + # click the left mouse button. This will open up the Start menu + # if the taskbar is at the bottom + + ClickButton() + + # wait a bit, then click the right button to open the context menu + time.sleep(3) + ClickButton('right') + + # move the mouse away and then click the left button to dismiss the + # context menu + MoveToLocation((screen_size[0]/2, screen_size[1]/2), 3) + MoveToLocation((0, 0), 3) + ClickButton() +
\ No newline at end of file diff --git a/tools/site_compare/drivers/win32/windowing.py b/tools/site_compare/drivers/win32/windowing.py new file mode 100644 index 0000000..94ec511 --- /dev/null +++ b/tools/site_compare/drivers/win32/windowing.py @@ -0,0 +1,386 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""SiteCompare module for invoking, locating, and manipulating windows. + +This module is a catch-all wrapper for operating system UI functionality +that doesn't belong in other modules. It contains functions for finding +particular windows, scraping their contents, and invoking processes to +create them. +""" + +import os +import string +import time + +import PIL.ImageGrab +import pywintypes +import win32event +import win32gui +import win32process + + +def FindChildWindows(hwnd, path): + """Find a set of windows through a path specification. + + Args: + hwnd: Handle of the parent window + path: Path to the window to find. Has the following form: + "foo/bar/baz|foobar/|foobarbaz" + The slashes specify the "path" to the child window. + The text is the window class, a pipe (if present) is a title. + * is a wildcard and will find all child windows at that level + + Returns: + A list of the windows that were found + """ + windows_to_check = [hwnd] + + # The strategy will be to take windows_to_check and use it + # to find a list of windows that match the next specification + # in the path, then repeat with the list of found windows as the + # new list of windows to check + for segment in path.split("/"): + windows_found = [] + check_values = segment.split("|") + + # check_values is now a list with the first element being + # the window class, the second being the window caption. + # If the class is absent (or wildcarded) set it to None + if check_values[0] == "*" or not check_values[0]: check_values[0] = None + + # If the window caption is also absent, force it to None as well + if len(check_values) == 1: check_values.append(None) + + # Loop through the list of windows to check + for window_check in windows_to_check: + window_found = None + while window_found != 0: # lint complains, but 0 != None + if window_found is None: window_found = 0 + try: + # Look for the next sibling (or first sibling if window_found is 0) + # of window_check with the specified caption and/or class + window_found = win32gui.FindWindowEx( + window_check, window_found, check_values[0], check_values[1]) + except pywintypes.error, e: + # FindWindowEx() raises error 2 if not found + if e[0] == 2: + window_found = 0 + else: + raise e + + # If FindWindowEx struck gold, add to our list of windows found + if window_found: windows_found.append(window_found) + + # The windows we found become the windows to check for the next segment + windows_to_check = windows_found + + return windows_found + + +def FindChildWindow(hwnd, path): + """Find a window through a path specification. + + This method is a simple wrapper for FindChildWindows() for the + case (the majority case) where you expect to find a single window + + Args: + hwnd: Handle of the parent window + path: Path to the window to find. See FindChildWindows() + + Returns: + The window that was found + """ + return FindChildWindows(hwnd, path)[0] + + +def ScrapeWindow(hwnd, rect=None): + """Scrape a visible window and return its contents as a bitmap. + + Args: + hwnd: handle of the window to scrape + rect: rectangle to scrape in client coords, defaults to the whole thing + If specified, it's a 4-tuple of (left, top, right, bottom) + + Returns: + An Image containing the scraped data + """ + # Activate the window + SetForegroundWindow(hwnd) + + # If no rectangle was specified, use the fill client rectangle + if not rect: rect = win32gui.GetClientRect(hwnd) + + upper_left = win32gui.ClientToScreen(hwnd, (rect[0], rect[1])) + lower_right = win32gui.ClientToScreen(hwnd, (rect[2], rect[3])) + rect = upper_left+lower_right + + return PIL.ImageGrab.grab(rect) + + +def SetForegroundWindow(hwnd): + """Bring a window to the foreground.""" + win32gui.SetForegroundWindow(hwnd) + + +def InvokeAndWait(path, cmdline="", timeout=10, tick=1.): + """Invoke an application and wait for it to bring up a window. + + Args: + path: full path to the executable to invoke + cmdline: command line to pass to executable + timeout: how long (in seconds) to wait before giving up + tick: length of time to wait between checks + + Returns: + A tuple of handles to the process and the application's window, + or (None, None) if it timed out waiting for the process + """ + + def EnumWindowProc(hwnd, ret): + """Internal enumeration func, checks for visibility and proper PID.""" + if win32gui.IsWindowVisible(hwnd): # don't bother even checking hidden wnds + pid = win32process.GetWindowThreadProcessId(hwnd)[1] + if pid == ret[0]: + ret[1] = hwnd + return 0 # 0 means stop enumeration + return 1 # 1 means continue enumeration + + # We don't need to change anything about the startupinfo structure + # (the default is quite sufficient) but we need to create it just the + # same. + sinfo = win32process.STARTUPINFO() + + proc = win32process.CreateProcess( + path, # path to new process's executable + cmdline, # application's command line + None, # process security attributes (default) + None, # thread security attributes (default) + False, # inherit parent's handles + 0, # creation flags + None, # environment variables + None, # directory + sinfo) # default startup info + + # Create process returns (prochandle, pid, threadhandle, tid). At + # some point we may care about the other members, but for now, all + # we're after is the pid + pid = proc[2] + + # Enumeration APIs can take an arbitrary integer, usually a pointer, + # to be passed to the enumeration function. We'll pass a pointer to + # a structure containing the PID we're looking for, and an empty out + # parameter to hold the found window ID + ret = [pid, None] + + tries_until_timeout = timeout/tick + num_tries = 0 + + # Enumerate top-level windows, look for one with our PID + while num_tries < tries_until_timeout and ret[1] is None: + try: + win32gui.EnumWindows(EnumWindowProc, ret) + except pywintypes.error, e: + # error 0 isn't an error, it just meant the enumeration was + # terminated early + if e[0]: raise e + + time.sleep(tick) + num_tries += 1 + + # TODO(jhaas): Should we throw an exception if we timeout? Or is returning + # a window ID of None sufficient? + return (proc[0], ret[1]) + + +def WaitForProcessExit(proc, timeout=None): + """Waits for a given process to terminate. + + Args: + proc: handle to process + timeout: timeout (in seconds). None = wait indefinitely + + Returns: + True if process ended, False if timed out + """ + if timeout is None: + timeout = win32event.INFINITE + else: + # convert sec to msec + timeout *= 1000 + + return (win32event.WaitForSingleObject(proc, timeout) == + win32event.WAIT_OBJECT_0) + + +def WaitForThrobber(hwnd, rect=None, timeout=20, tick=0.1, done=10): + """Wait for a browser's "throbber" (loading animation) to complete. + + Args: + hwnd: window containing the throbber + rect: rectangle of the throbber, in client coords. If None, whole window + timeout: if the throbber is still throbbing after this long, give up + tick: how often to check the throbber + done: how long the throbber must be unmoving to be considered done + + Returns: + Number of seconds waited, -1 if timed out + """ + if not rect: rect = win32gui.GetClientRect(hwnd) + + # last_throbber will hold the results of the preceding scrape; + # we'll compare it against the current scrape to see if we're throbbing + last_throbber = ScrapeWindow(hwnd, rect) + start_clock = time.clock() + timeout_clock = start_clock + timeout + last_changed_clock = start_clock; + + while time.clock() < timeout_clock: + time.sleep(tick) + + current_throbber = ScrapeWindow(hwnd, rect) + if current_throbber.tostring() != last_throbber.tostring(): + last_throbber = current_throbber + last_changed_clock = time.clock() + else: + if time.clock() - last_changed_clock > done: + return last_changed_clock - start_clock + + return -1 + + +def MoveAndSizeWindow(wnd, position=None, size=None, child=None): + """Moves and/or resizes a window. + + Repositions and resizes a window. If a child window is provided, + the parent window is resized so the child window has the given size + + Args: + wnd: handle of the frame window + position: new location for the frame window + size: new size for the frame window (or the child window) + child: handle of the child window + + Returns: + None + """ + rect = win32gui.GetWindowRect(wnd) + + if position is None: position = (rect[0], rect[1]) + if size is None: + size = (rect[2]-rect[0], rect[3]-rect[1]) + elif child is not None: + child_rect = win32gui.GetWindowRect(child) + slop = (rect[2]-rect[0]-child_rect[2]+child_rect[0], + rect[3]-rect[1]-child_rect[3]+child_rect[1]) + size = (size[0]+slop[0], size[1]+slop[1]) + + win32gui.MoveWindow(wnd, # window to move + position[0], # new x coord + position[1], # new y coord + size[0], # new width + size[1], # new height + True) # repaint? + + +def EndProcess(proc, code=0): + """Ends a process. + + Wraps the OS TerminateProcess call for platform-independence + + Args: + proc: process ID + code: process exit code + + Returns: + None + """ + win32process.TerminateProcess(proc, code) + + +def URLtoFilename(url, path=None, extension=None): + """Converts a URL to a filename, given a path. + + This in theory could cause collisions if two URLs differ only + in unprintable characters (eg. http://www.foo.com/?bar and + http://www.foo.com/:bar. In practice this shouldn't be a problem. + + Args: + url: The URL to convert + path: path to the directory to store the file + extension: string to append to filename + + Returns: + filename + """ + trans = string.maketrans(r'\/:*?"<>|', '_________') + + if path is None: path = "" + if extension is None: extension = "" + if len(path) > 0 and path[-1] != '\\': path += '\\' + url = url.translate(trans) + return "%s%s%s" % (path, url, extension) + + +def PreparePath(path): + """Ensures that a given path exists, making subdirectories if necessary. + + Args: + path: fully-qualified path of directory to ensure exists + + Returns: + None + """ + try: + os.makedirs(path) + except OSError, e: + if e[0] != 17: raise e # error 17: path already exists + +if __name__ == "__main__": + PreparePath(r"c:\sitecompare\scrapes\ie7") + # We're being invoked rather than imported. Let's do some tests + + # Hardcode IE's location for the purpose of this test + (proc, wnd) = InvokeAndWait( + r"c:\program files\internet explorer\iexplore.exe") + + # Find the browser pane in the IE window + browser = FindChildWindow( + wnd, "TabWindowClass/Shell DocObject View/Internet Explorer_Server") + + # Move and size the window + MoveAndSizeWindow(wnd, (0, 0), (1024, 768), browser) + + # Take a screenshot + i = ScrapeWindow(browser) + + i.show() + + EndProcess(proc, 0) diff --git a/tools/site_compare/operators/__init__.py b/tools/site_compare/operators/__init__.py new file mode 100644 index 0000000..02eac07 --- /dev/null +++ b/tools/site_compare/operators/__init__.py @@ -0,0 +1,26 @@ +#!/usr/bin/python2.4 +# +# Copyright 2007 Google Inc. All Rights Reserved. + +"""Selects the appropriate operator.""" + +__author__ = 'jhaas@google.com (Jonathan Haas)' + + +def GetOperator(operator): + """Given an operator by name, returns its module. + + Args: + operator: string describing the comparison + + Returns: + module + """ + + # TODO(jhaas): come up with a happy way of integrating multiple operators + # with different, possibly divergent and possibly convergent, operators. + + module = __import__(operator, globals(), locals(), ['']) + + return module + diff --git a/tools/site_compare/operators/equals.py b/tools/site_compare/operators/equals.py new file mode 100644 index 0000000..2771401 --- /dev/null +++ b/tools/site_compare/operators/equals.py @@ -0,0 +1,66 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Compare two images for equality.""" + +from PIL import Image +from PIL import ImageChops + + +def Compare(file1, file2, **kwargs): + """Compares two images to see if they're identical. + + Args: + file1: path to first image to compare + file2: path to second image to compare + kwargs: unused for this operator + + Returns: + None if the images are identical + A tuple of (errorstring, image) if they're not + """ + kwargs = kwargs # unused parameter + + im1 = Image.open(file1) + im2 = Image.open(file2) + + if im1.size != im2.size: + return ("The images are of different size (%s vs %s)" % + (im1.size, im2.size), im1) + + diff = ImageChops.difference(im1, im2) + + if max(diff.getextrema()) != (0, 0): + return ("The images differ", diff) + else: + return None + + +
\ No newline at end of file diff --git a/tools/site_compare/operators/equals_with_mask.py b/tools/site_compare/operators/equals_with_mask.py new file mode 100644 index 0000000..574457a --- /dev/null +++ b/tools/site_compare/operators/equals_with_mask.py @@ -0,0 +1,86 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Compare two images for equality, subject to a mask.""" + +from PIL import Image +from PIL import ImageChops + +import os.path + + +def Compare(file1, file2, **kwargs): + """Compares two images to see if they're identical subject to a mask. + + An optional directory containing masks is supplied. If a mask exists + which matches file1's name, areas under the mask where it's black + are ignored. + + Args: + file1: path to first image to compare + file2: path to second image to compare + kwargs: ["maskdir"] contains the directory holding the masks + + Returns: + None if the images are identical + A tuple of (errorstring, image) if they're not + """ + + maskdir = None + if "maskdir" in kwargs: + maskdir = kwargs["maskdir"] + + im1 = Image.open(file1) + im2 = Image.open(file2) + + if im1.size != im2.size: + return ("The images are of different size (%r vs %r)" % + (im1.size, im2.size), im1) + + diff = ImageChops.difference(im1, im2) + + if maskdir: + maskfile = os.path.join(maskdir, os.path.basename(file1)) + if os.path.exists(maskfile): + mask = Image.open(maskfile) + + if mask.size != im1.size: + return ("The mask is of a different size than the images (%r vs %r)" % + (mask.size, im1.size), mask) + + diff = ImageChops.multiply(diff, mask.convert(diff.mode)) + + if max(diff.getextrema()) != (0, 0): + return ("The images differ", diff) + else: + return None + + +
\ No newline at end of file diff --git a/tools/site_compare/scrapers/__init__.py b/tools/site_compare/scrapers/__init__.py new file mode 100644 index 0000000..08790aa --- /dev/null +++ b/tools/site_compare/scrapers/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# +# Copyright 2007 Google Inc. All Rights Reserved. + +"""Selects the appropriate scraper for a given browser and version.""" + +__author__ = 'jhaas@google.com (Jonathan Haas)' + +import types + +# TODO(jhaas): unify all optional scraper parameters into kwargs + +def GetScraper(browser): + """Given a browser and an optional version, returns the scraper module. + + Args: + browser: either a string (browser name) or a tuple (name, version) + + Returns: + module + """ + + if type(browser) == types.StringType: browser = (browser, None) + + package = __import__(browser[0], globals(), locals(), ['']) + module = package.GetScraper(browser[1]) + if browser[1] is not None: module.version = browser[1] + + return module + +# if invoked rather than imported, do some tests +if __name__ == "__main__": + print GetScraper("IE") +
\ No newline at end of file diff --git a/tools/site_compare/scrapers/chrome/__init__.py b/tools/site_compare/scrapers/chrome/__init__.py new file mode 100644 index 0000000..2ba76c4 --- /dev/null +++ b/tools/site_compare/scrapers/chrome/__init__.py @@ -0,0 +1,38 @@ +#!/usr/bin/python2.4 +# +# Copyright 2007 Google Inc. All Rights Reserved. + +"""Selects the appropriate scraper for Chrome.""" + +__author__ = 'jhaas@google.com (Jonathan Haas)' + +def GetScraper(version): + """Returns the scraper module for the given version. + + Args: + version: version string of Chrome, or None for most recent + + Returns: + scrape module for given version + """ + if version is None: + version = "0.1.101.0" + + parsed_version = [int(x) for x in version.split(".")] + + if (parsed_version[0] > 0 or + parsed_version[1] > 1 or + parsed_version[2] > 97 or + parsed_version[3] > 0): + scraper_version = "chrome011010" + else: + scraper_version = "chrome01970" + + return __import__(scraper_version, globals(), locals(), ['']) + +# if invoked rather than imported, test +if __name__ == "__main__": + version = "0.1.101.0" + + print GetScraper(version).version +
\ No newline at end of file diff --git a/tools/site_compare/scrapers/chrome/chrome011010.py b/tools/site_compare/scrapers/chrome/chrome011010.py new file mode 100644 index 0000000..0b75ff8 --- /dev/null +++ b/tools/site_compare/scrapers/chrome/chrome011010.py @@ -0,0 +1,68 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Does scraping for versions of Chrome from 0.1.101.0 up.""" + +from drivers import windowing + +import chromebase + +# Default version +version = "0.1.101.0" + + +def GetChromeRenderPane(wnd): + return windowing.FindChildWindow(wnd, "Chrome_TabContents") + + +def Scrape(urls, outdir, size, pos, timeout=20, **kwargs): + """Invoke a browser, send it to a series of URLs, and save its output. + + Args: + urls: list of URLs to scrape + outdir: directory to place output + size: size of browser window to use + pos: position of browser window + timeout: amount of time to wait for page to load + kwargs: miscellaneous keyword args + + Returns: + None if succeeded, else an error code + """ + chromebase.GetChromeRenderPane = GetChromeRenderPane + + return chromebase.Scrape(urls, outdir, size, pos, timeout, kwargs) + + +def Time(urls, size, timeout, **kwargs): + """Forwards the Time command to chromebase.""" + chromebase.GetChromeRenderPane = GetChromeRenderPane + + return chromebase.Time(urls, size, timeout, kwargs) diff --git a/tools/site_compare/scrapers/chrome/chrome01970.py b/tools/site_compare/scrapers/chrome/chrome01970.py new file mode 100644 index 0000000..bf43095 --- /dev/null +++ b/tools/site_compare/scrapers/chrome/chrome01970.py @@ -0,0 +1,69 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Does scraping for versions of Chrome up to 0.1.97.0.""" + +from drivers import windowing + +import chromebase + +# Default version +version = "0.1.97.0" + + +def GetChromeRenderPane(wnd): + return windowing.FindChildWindow(wnd, "Chrome_BrowserWindow") + + +def Scrape(urls, outdir, size, pos, timeout=20, **kwargs): + """Invoke a browser, send it to a series of URLs, and save its output. + + Args: + urls: list of URLs to scrape + outdir: directory to place output + size: size of browser window to use + pos: position of browser window + timeout: amount of time to wait for page to load + kwargs: miscellaneous keyword args + + Returns: + None if succeeded, else an error code + """ + chromebase.GetChromeRenderPane = GetChromeRenderPane + + return chromebase.Scrape(urls, outdir, size, pos, timeout, kwargs) + + +def Time(urls, size, timeout, **kwargs): + """Forwards the Time command to chromebase.""" + chromebase.GetChromeRenderPane = GetChromeRenderPane + + return chromebase.Time(urls, size, timeout, kwargs) +
\ No newline at end of file diff --git a/tools/site_compare/scrapers/chrome/chromebase.py b/tools/site_compare/scrapers/chrome/chromebase.py new file mode 100644 index 0000000..4825049 --- /dev/null +++ b/tools/site_compare/scrapers/chrome/chromebase.py @@ -0,0 +1,217 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Does scraping for all currently-known versions of Chrome""" + +import pywintypes +import types + +from drivers import keyboard +from drivers import mouse +from drivers import windowing + +# TODO: this has moved, use some logic to find it. For now, +# expects a subst k:. +DEFAULT_PATH = r"k:\chrome.exe" + +def InvokeBrowser(path): + """Invoke the Chrome browser. + + Args: + path: full path to browser + + Returns: + A tuple of (main window, process handle, address bar, render pane) + """ + + # Reuse an existing instance of the browser if we can find one. This + # may not work correctly, especially if the window is behind other windows. + + # TODO(jhaas): make this work with Vista + wnds = windowing.FindChildWindows(0, "Chrome_XPFrame") + if len(wnds): + wnd = wnds[0] + proc = None + else: + # Invoke Chrome + (proc, wnd) = windowing.InvokeAndWait(path) + + # Get windows we'll need + address_bar = windowing.FindChildWindow(wnd, "Chrome_AutocompleteEdit") + render_pane = GetChromeRenderPane(wnd) + + return (wnd, proc, address_bar, render_pane) + + +def Scrape(urls, outdir, size, pos, timeout, kwargs): + """Invoke a browser, send it to a series of URLs, and save its output. + + Args: + urls: list of URLs to scrape + outdir: directory to place output + size: size of browser window to use + pos: position of browser window + timeout: amount of time to wait for page to load + kwargs: miscellaneous keyword args + + Returns: + None if success, else an error string + """ + if "path" in kwargs and kwargs["path"]: path = kwargs["path"] + else: path = DEFAULT_PATH + + (wnd, proc, address_bar, render_pane) = InvokeBrowser(path) + + # Resize and reposition the frame + windowing.MoveAndSizeWindow(wnd, pos, size, render_pane) + + # Visit each URL we're given + if type(urls) in types.StringTypes: urls = [urls] + + timedout = False + + for url in urls: + # Double-click in the address bar, type the name, and press Enter + mouse.ClickInWindow(address_bar) + keyboard.TypeString(url, 0.1) + keyboard.TypeString("\n") + + # Wait for the page to finish loading + load_time = windowing.WaitForThrobber(wnd, (20, 16, 36, 32), timeout) + timedout = load_time < 0 + + if timedout: + break + + # Scrape the page + image = windowing.ScrapeWindow(render_pane) + + # Save to disk + if "filename" in kwargs: + if callable(kwargs["filename"]): + filename = kwargs["filename"](url) + else: + filename = kwargs["filename"] + else: + filename = windowing.URLtoFilename(url, outdir, ".bmp") + image.save(filename) + + if proc: + windowing.SetForegroundWindow(wnd) + + # Send Alt-F4, then wait for process to end + keyboard.TypeString(r"{\4}", use_modifiers=True) + if not windowing.WaitForProcessExit(proc, timeout): + windowing.EndProcess(proc) + return "crashed" + + if timedout: + return "timeout" + + return None + + +def Time(urls, size, timeout, kwargs): + """Measure how long it takes to load each of a series of URLs + + Args: + urls: list of URLs to time + size: size of browser window to use + timeout: amount of time to wait for page to load + kwargs: miscellaneous keyword args + + Returns: + A list of tuples (url, time). "time" can be "crashed" or "timeout" + """ + if "path" in kwargs and kwargs["path"]: path = kwargs["path"] + else: path = DEFAULT_PATH + proc = None + + # Visit each URL we're given + if type(urls) in types.StringTypes: urls = [urls] + + ret = [] + for url in urls: + try: + # Invoke the browser if necessary + if not proc: + (wnd, proc, address_bar, render_pane) = InvokeBrowser(path) + + # Resize and reposition the frame + windowing.MoveAndSizeWindow(wnd, (0,0), size, render_pane) + + # Double-click in the address bar, type the name, and press Enter + mouse.ClickInWindow(address_bar) + keyboard.TypeString(url, 0.1) + keyboard.TypeString("\n") + + # Wait for the page to finish loading + load_time = windowing.WaitForThrobber(wnd, (20, 16, 36, 32), timeout) + + timedout = load_time < 0 + + if timedout: + load_time = "timeout" + + # Send an alt-F4 to make the browser close; if this times out, + # we've probably got a crash + windowing.SetForegroundWindow(wnd) + + keyboard.TypeString(r"{\4}", use_modifiers=True) + if not windowing.WaitForProcessExit(proc, timeout): + windowing.EndProcess(proc) + load_time = "crashed" + proc = None + except pywintypes.error: + proc = None + load_time = "crashed" + + ret.append( (url, load_time) ) + + if proc: + windowing.SetForegroundWindow(wnd) + keyboard.TypeString(r"{\4}", use_modifiers=True) + if not windowing.WaitForProcessExit(proc, timeout): + windowing.EndProcess(proc) + + return ret + + +if __name__ == "__main__": + # We're being invoked rather than imported, so run some tests + path = r"c:\sitecompare\scrapes\chrome\0.1.97.0" + windowing.PreparePath(path) + + # Scrape three sites and save the results + Scrape([ + "http://www.microsoft.com", + "http://www.google.com", + "http://www.sun.com"], + path, (1024, 768), (0, 0)) diff --git a/tools/site_compare/scrapers/firefox/__init__.py b/tools/site_compare/scrapers/firefox/__init__.py new file mode 100644 index 0000000..255dc4b --- /dev/null +++ b/tools/site_compare/scrapers/firefox/__init__.py @@ -0,0 +1,31 @@ +#!/usr/bin/python2.4 +# +# Copyright 2007 Google Inc. All Rights Reserved. + +"""Selects the appropriate scraper for Firefox.""" + +__author__ = 'jhaas@google.com (Jonathan Haas)' + + +def GetScraper(version): + """Returns the scraper module for the given version. + + Args: + version: version string of IE, or None for most recent + + Returns: + scrape module for given version + """ + + # Pychecker will warn that the parameter is unused; we only + # support one version of Firefox at this time + + # We only have one version of the Firefox scraper for now + return __import__("firefox2", globals(), locals(), ['']) + +# if invoked rather than imported, test +if __name__ == "__main__": + version = "2.0.0.6" + + print GetScraper("2.0.0.6").version +
\ No newline at end of file diff --git a/tools/site_compare/scrapers/firefox/firefox2.py b/tools/site_compare/scrapers/firefox/firefox2.py new file mode 100644 index 0000000..d91534e --- /dev/null +++ b/tools/site_compare/scrapers/firefox/firefox2.py @@ -0,0 +1,269 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Does scraping for Firefox 2.0.""" + +import pywintypes +import time +import types + +from drivers import keyboard +from drivers import mouse +from drivers import windowing + +# Default version +version = "2.0.0.6" + +DEFAULT_PATH = r"c:\program files\mozilla firefox\firefox.exe" + +# TODO(jhaas): the Firefox scraper is a bit rickety at the moment. Known +# issues: 1) won't work if the default profile puts toolbars in different +# locations, 2) uses sleep() statements rather than more robust checks, +# 3) fails badly if an existing Firefox window is open when the scrape +# is invoked. This needs to be fortified at some point. + +def GetBrowser(path): + """Invoke the Firefox browser and return the process and window. + + Args: + path: full path to browser + + Returns: + A tuple of (process handle, render pane) + """ + if not path: path = DEFAULT_PATH + + # Invoke Firefox + (proc, wnd) = windowing.InvokeAndWait(path) + + # Get the content pane + render_pane = windowing.FindChildWindow( + wnd, + "MozillaWindowClass/MozillaWindowClass/MozillaWindowClass") + + return (proc, wnd, render_pane) + + +def InvokeBrowser(path): + """Invoke the Firefox browser. + + Args: + path: full path to browser + + Returns: + A tuple of (main window, process handle, render pane) + """ + # Reuse an existing instance of the browser if we can find one. This + # may not work correctly, especially if the window is behind other windows. + wnds = windowing.FindChildWindows(0, "MozillaUIWindowClass") + if len(wnds): + wnd = wnds[0] + proc = None + else: + # Invoke Firefox + (proc, wnd) = windowing.InvokeAndWait(path) + + # Get the content pane + render_pane = windowing.FindChildWindow( + wnd, + "MozillaWindowClass/MozillaWindowClass/MozillaWindowClass") + + return (wnd, proc, render_pane) + + +def Scrape(urls, outdir, size, pos, timeout=20, **kwargs): + """Invoke a browser, send it to a series of URLs, and save its output. + + Args: + urls: list of URLs to scrape + outdir: directory to place output + size: size of browser window to use + pos: position of browser window + timeout: amount of time to wait for page to load + kwargs: miscellaneous keyword args + + Returns: + None if success, else an error string + """ + if "path" in kwargs and kwargs["path"]: path = kwargs["path"] + else: path = DEFAULT_PATH + + (wnd, proc, render_pane) = InvokeBrowser(path) + + # Resize and reposition the frame + windowing.MoveAndSizeWindow(wnd, pos, size, render_pane) + + time.sleep(3) + + # Firefox is a bit of a pain: it doesn't use standard edit controls, + # and it doesn't display a throbber when there's no tab. Let's make + # sure there's at least one tab, then select the first one + + mouse.ClickInWindow(wnd) + keyboard.TypeString("[t]", True) + mouse.ClickInWindow(wnd, (30, 115)) + time.sleep(2) + + timedout = False + + # Visit each URL we're given + if type(urls) in types.StringTypes: urls = [urls] + + for url in urls: + + # Use keyboard shortcuts + keyboard.TypeString("{d}", True) + keyboard.TypeString(url) + keyboard.TypeString("\n") + + # Wait for the page to finish loading + load_time = windowing.WaitForThrobber(wnd, (10, 96, 26, 112), timeout) + timedout = load_time < 0 + + if timedout: + break + + # Scrape the page + image = windowing.ScrapeWindow(render_pane) + + # Save to disk + if "filename" in kwargs: + if callable(kwargs["filename"]): + filename = kwargs["filename"](url) + else: + filename = kwargs["filename"] + else: + filename = windowing.URLtoFilename(url, outdir, ".bmp") + image.save(filename) + + # Close all the tabs, cheesily + mouse.ClickInWindow(wnd) + + while len(windowing.FindChildWindows(0, "MozillaUIWindowClass")): + keyboard.TypeString("[w]", True) + time.sleep(1) + + if timedout: + return "timeout" + + +def Time(urls, size, timeout, **kwargs): + """Measure how long it takes to load each of a series of URLs + + Args: + urls: list of URLs to time + size: size of browser window to use + timeout: amount of time to wait for page to load + kwargs: miscellaneous keyword args + + Returns: + A list of tuples (url, time). "time" can be "crashed" or "timeout" + """ + if "path" in kwargs and kwargs["path"]: path = kwargs["path"] + else: path = DEFAULT_PATH + proc = None + + # Visit each URL we're given + if type(urls) in types.StringTypes: urls = [urls] + + ret = [] + for url in urls: + try: + # Invoke the browser if necessary + if not proc: + (wnd, proc, render_pane) = InvokeBrowser(path) + + # Resize and reposition the frame + windowing.MoveAndSizeWindow(wnd, (0,0), size, render_pane) + + time.sleep(3) + + # Firefox is a bit of a pain: it doesn't use standard edit controls, + # and it doesn't display a throbber when there's no tab. Let's make + # sure there's at least one tab, then select the first one + + mouse.ClickInWindow(wnd) + keyboard.TypeString("[t]", True) + mouse.ClickInWindow(wnd, (30, 115)) + time.sleep(2) + + # Use keyboard shortcuts + keyboard.TypeString("{d}", True) + keyboard.TypeString(url) + keyboard.TypeString("\n") + + # Wait for the page to finish loading + load_time = windowing.WaitForThrobber(wnd, (10, 96, 26, 112), timeout) + timedout = load_time < 0 + + if timedout: + load_time = "timeout" + + # Try to close the browser; if this fails it's probably a crash + mouse.ClickInWindow(wnd) + + count = 0 + while (len(windowing.FindChildWindows(0, "MozillaUIWindowClass")) + and count < 5): + keyboard.TypeString("[w]", True) + time.sleep(1) + count = count + 1 + + if len(windowing.FindChildWindows(0, "MozillaUIWindowClass")): + windowing.EndProcess(proc) + load_time = "crashed" + + proc = None + except pywintypes.error: + proc = None + load_time = "crashed" + + ret.append( (url, load_time) ) + + if proc: + count = 0 + while (len(windowing.FindChildWindows(0, "MozillaUIWindowClass")) + and count < 5): + keyboard.TypeString("[w]", True) + time.sleep(1) + count = count + 1 + return ret + + +if __name__ == "__main__": + # We're being invoked rather than imported, so run some tests + path = r"c:\sitecompare\scrapes\Firefox\2.0.0.6" + windowing.PreparePath(path) + + # Scrape three sites and save the results + Scrape( + ["http://www.microsoft.com", "http://www.google.com", + "http://www.sun.com"], + path, (1024, 768), (0, 0)) diff --git a/tools/site_compare/scrapers/ie/__init__.py b/tools/site_compare/scrapers/ie/__init__.py new file mode 100644 index 0000000..4b8949b --- /dev/null +++ b/tools/site_compare/scrapers/ie/__init__.py @@ -0,0 +1,31 @@ +#!/usr/bin/python2.4 +# +# Copyright 2007 Google Inc. All Rights Reserved. + +"""Selects the appropriate scraper for Internet Explorer.""" + +__author__ = 'jhaas@google.com (Jonathan Haas)' + + +def GetScraper(version): + """Returns the scraper module for the given version. + + Args: + version: version string of IE, or None for most recent + + Returns: + scrape module for given version + """ + + # Pychecker will warn that the parameter is unused; we only + # support one version of IE at this time + + # We only have one version of the IE scraper for now + return __import__("ie7", globals(), locals(), ['']) + +# if invoked rather than imported, test +if __name__ == "__main__": + version = "7.0.5370.1" + + print GetScraper(version).version +
\ No newline at end of file diff --git a/tools/site_compare/scrapers/ie/ie7.py b/tools/site_compare/scrapers/ie/ie7.py new file mode 100644 index 0000000..a0475e0 --- /dev/null +++ b/tools/site_compare/scrapers/ie/ie7.py @@ -0,0 +1,230 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Does scraping for all known versions of IE.""" + +import pywintypes +import time +import types + +from drivers import keyboard +from drivers import mouse +from drivers import windowing + +# Default version +version = "7.0.5730.1" + +DEFAULT_PATH = r"c:\program files\internet explorer\iexplore.exe" + +def GetBrowser(path): + """Invoke the IE browser and return the process, frame, and content window. + + Args: + path: full path to browser + + Returns: + A tuple of (process handle, render pane) + """ + if not path: path = DEFAULT_PATH + + (iewnd, ieproc, address_bar, render_pane, tab_window) = InvokeBrowser(path) + return (ieproc, iewnd, render_pane) + + +def InvokeBrowser(path): + """Invoke the IE browser. + + Args: + path: full path to browser + + Returns: + A tuple of (main window, process handle, address bar, + render_pane, tab_window) + """ + # Invoke IE + (ieproc, iewnd) = windowing.InvokeAndWait(path) + + # Get windows we'll need + for tries in xrange(10): + try: + address_bar = windowing.FindChildWindow( + iewnd, "WorkerW|Navigation Bar/ReBarWindow32/" + "Address Band Root/ComboBoxEx32/ComboBox/Edit") + render_pane = windowing.FindChildWindow( + iewnd, "TabWindowClass/Shell DocObject View") + tab_window = windowing.FindChildWindow( + iewnd, "CommandBarClass/ReBarWindow32/TabBandClass/DirectUIHWND") + except IndexError: + time.sleep(1) + continue + break + + return (iewnd, ieproc, address_bar, render_pane, tab_window) + + +def Scrape(urls, outdir, size, pos, timeout=20, **kwargs): + """Invoke a browser, send it to a series of URLs, and save its output. + + Args: + urls: list of URLs to scrape + outdir: directory to place output + size: size of browser window to use + pos: position of browser window + timeout: amount of time to wait for page to load + kwargs: miscellaneous keyword args + + Returns: + None if success, else an error string + """ + path = r"c:\program files\internet explorer\iexplore.exe" + + if "path" in kwargs and kwargs["path"]: path = kwargs["path"] + + (iewnd, ieproc, address_bar, render_pane, tab_window) = ( + InvokeBrowser(path) ) + + # Resize and reposition the frame + windowing.MoveAndSizeWindow(iewnd, pos, size, render_pane) + + # Visit each URL we're given + if type(urls) in types.StringTypes: urls = [urls] + + timedout = False + + for url in urls: + + # Double-click in the address bar, type the name, and press Enter + mouse.DoubleClickInWindow(address_bar) + keyboard.TypeString(url) + keyboard.TypeString("\n") + + # Wait for the page to finish loading + load_time = windowing.WaitForThrobber( + tab_window, (6, 8, 22, 24), timeout) + timedout = load_time < 0 + + if timedout: + break + + # Scrape the page + image = windowing.ScrapeWindow(render_pane) + + # Save to disk + if "filename" in kwargs: + if callable(kwargs["filename"]): + filename = kwargs["filename"](url) + else: + filename = kwargs["filename"] + else: + filename = windowing.URLtoFilename(url, outdir, ".bmp") + image.save(filename) + + windowing.EndProcess(ieproc) + + if timedout: + return "timeout" + + +def Time(urls, size, timeout, **kwargs): + """Measure how long it takes to load each of a series of URLs + + Args: + urls: list of URLs to time + size: size of browser window to use + timeout: amount of time to wait for page to load + kwargs: miscellaneous keyword args + + Returns: + A list of tuples (url, time). "time" can be "crashed" or "timeout" + """ + if "path" in kwargs and kwargs["path"]: path = kwargs["path"] + else: path = DEFAULT_PATH + proc = None + + # Visit each URL we're given + if type(urls) in types.StringTypes: urls = [urls] + + ret = [] + for url in urls: + try: + # Invoke the browser if necessary + if not proc: + (wnd, proc, address_bar, render_pane, tab_window) = InvokeBrowser(path) + + # Resize and reposition the frame + windowing.MoveAndSizeWindow(wnd, (0,0), size, render_pane) + + # Double-click in the address bar, type the name, and press Enter + mouse.DoubleClickInWindow(address_bar) + keyboard.TypeString(url) + keyboard.TypeString("\n") + + # Wait for the page to finish loading + load_time = windowing.WaitForThrobber( + tab_window, (6, 8, 22, 24), timeout) + timedout = load_time < 0 + + if timedout: + load_time = "timeout" + + # Send an alt-F4 to make the browser close; if this times out, + # we've probably got a crash + keyboard.TypeString(r"{\4}", use_modifiers=True) + if not windowing.WaitForProcessExit(proc, timeout): + windowing.EndProcess(proc) + load_time = "crashed" + proc = None + except pywintypes.error: + load_time = "crashed" + proc = None + + ret.append( (url, load_time) ) + + # Send an alt-F4 to make the browser close; if this times out, + # we've probably got a crash + if proc: + keyboard.TypeString(r"{\4}", use_modifiers=True) + if not windowing.WaitForProcessExit(proc, timeout): + windowing.EndProcess(proc) + + return ret + + +if __name__ == "__main__": + # We're being invoked rather than imported, so run some tests + path = r"c:\sitecompare\scrapes\ie7\7.0.5380.11" + windowing.PreparePath(path) + + # Scrape three sites and save the results + Scrape( + ["http://www.microsoft.com", + "http://www.google.com", + "http://www.sun.com"], + path, (1024, 768), (0, 0)) diff --git a/tools/site_compare/site_compare.py b/tools/site_compare/site_compare.py new file mode 100644 index 0000000..8acfdcf --- /dev/null +++ b/tools/site_compare/site_compare.py @@ -0,0 +1,202 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""SiteCompare component to handle bulk scrapes. + +Invokes a list of browsers and sends them to a list of URLs, +saving the rendered results to a specified directory, then +performs comparison operations on the resulting bitmaps and +saves the results +""" + + +# This line is necessary to work around a QEMU bug +import _imaging + +import os # Functions for walking the directory tree +import types # Runtime type-checking + +import command_line # command-line parsing +import drivers # Functions for driving keyboard/mouse/windows, OS-specific +import operators # Functions that, given two bitmaps as input, produce + # output depending on the performance of an operation +import scrapers # Functions that know how to capture a render from + # particular browsers + +import commands.compare2 # compare one page in two versions of same browser +import commands.maskmaker # generate a mask based on repeated scrapes +import commands.measure # measure length of time a page takes to load +import commands.scrape # scrape a URL or series of URLs to a bitmap + +# The timeload command is obsolete (too flaky); it may be reinstated +# later but for now it's been superceded by "measure" +# import commands.timeload # measure length of time a page takes to load + +def Scrape(browsers, urls, window_size=(1024, 768), + window_pos=(0, 0), timeout=20, save_path=None, **kwargs): + """Invoke one or more browsers over one or more URLs, scraping renders. + + Args: + browsers: browsers to invoke with optional version strings + urls: URLs to visit + window_size: size of the browser window to display + window_pos: location of browser window + timeout: time (in seconds) to wait for page to load + save_path: root of save path, automatically appended with browser and + version + kwargs: miscellaneous keyword args, passed to scraper + Returns: + None + + @TODO(jhaas): more parameters, or perhaps an indefinite dictionary + parameter, for things like length of time to wait for timeout, speed + of mouse clicks, etc. Possibly on a per-browser, per-URL, or + per-browser-per-URL basis + """ + + if type(browsers) in types.StringTypes: browsers = [browsers] + + if save_path is None: + # default save path is "scrapes" off the current root + save_path = os.path.join(os.path.split(__file__)[0], "Scrapes") + + for browser in browsers: + # Browsers should be tuples of (browser, version) + if type(browser) in types.StringTypes: browser = (browser, None) + scraper = scrapers.GetScraper(browser) + + full_path = os.path.join(save_path, browser[0], scraper.version) + drivers.windowing.PreparePath(full_path) + + scraper.Scrape(urls, full_path, window_size, window_pos, timeout, kwargs) + + +def Compare(base, compare, ops, root_path=None, out_path=None): + """Compares a series of scrapes using a series of operators. + + Args: + base: (browser, version) tuple of version to consider the baseline + compare: (browser, version) tuple of version to compare to + ops: list of operators plus operator arguments + root_path: root of the scrapes + out_path: place to put any output from the operators + + Returns: + None + + @TODO(jhaas): this method will likely change, to provide a robust and + well-defined way of chaining operators, applying operators conditionally, + and full-featured scripting of the operator chain. There also needs + to be better definition of the output; right now it's to stdout and + a log.txt file, with operator-dependent images saved for error output + """ + if root_path is None: + # default save path is "scrapes" off the current root + root_path = os.path.join(os.path.split(__file__)[0], "Scrapes") + + if out_path is None: + out_path = os.path.join(os.path.split(__file__)[0], "Compares") + + if type(base) in types.StringTypes: base = (base, None) + if type(compare) in types.StringTypes: compare = (compare, None) + if type(ops) in types.StringTypes: ops = [ops] + + base_dir = os.path.join(root_path, base[0]) + compare_dir = os.path.join(root_path, compare[0]) + + if base[1] is None: + # base defaults to earliest capture + base = (base[0], max(os.listdir(base_dir))) + + if compare[1] is None: + # compare defaults to latest capture + compare = (compare[0], min(os.listdir(compare_dir))) + + out_path = os.path.join(out_path, base[0], base[1], compare[0], compare[1]) + drivers.windowing.PreparePath(out_path) + + # TODO(jhaas): right now we're just dumping output to a log file + # (and the console), which works as far as it goes but isn't nearly + # robust enough. Change this after deciding exactly what we want to + # change it to. + out_file = open(os.path.join(out_path, "log.txt"), "w") + description_string = ("Comparing %s %s to %s %s" % + (base[0], base[1], compare[0], compare[1])) + out_file.write(description_string) + print description_string + + base_dir = os.path.join(base_dir, base[1]) + compare_dir = os.path.join(compare_dir, compare[1]) + + for filename in os.listdir(base_dir): + out_file.write("%s: " % filename) + + if not os.path.isfile(os.path.join(compare_dir, filename)): + out_file.write("Does not exist in target directory\n") + print "File %s does not exist in target directory" % filename + continue + + base_filename = os.path.join(base_dir, filename) + compare_filename = os.path.join(compare_dir, filename) + + for op in ops: + if type(op) in types.StringTypes: op = (op, None) + + module = operators.GetOperator(op[0]) + + ret = module.Compare(base_filename, compare_filename) + if ret is None: + print "%s: OK" % (filename,) + out_file.write("OK\n") + else: + print "%s: %s" % (filename, ret[0]) + out_file.write("%s\n" % (ret[0])) + ret[1].save(os.path.join(out_path, filename)) + + out_file.close() + + +def main(): + """Main executable. Parse the command line and invoke the command.""" + cmdline = command_line.CommandLine() + + # The below two commands are currently unstable so have been disabled + # commands.compare2.CreateCommand(cmdline) + # commands.maskmaker.CreateCommand(cmdline) + commands.measure.CreateCommand(cmdline) + commands.scrape.CreateCommand(cmdline) + + cmdline.ParseCommandLine() + + + +if __name__ == "__main__": + main() +
\ No newline at end of file diff --git a/tools/site_compare/utils/__init__.py b/tools/site_compare/utils/__init__.py new file mode 100644 index 0000000..69f2237 --- /dev/null +++ b/tools/site_compare/utils/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/python2.4 +# +# Copyright 2007 Google Inc. All Rights Reserved. + +"""Utilities for site_compare.""" + +__author__ = 'jhaas@google.com (Jonathan Haas)' diff --git a/tools/site_compare/utils/browser_iterate.py b/tools/site_compare/utils/browser_iterate.py new file mode 100644 index 0000000..65ba24f --- /dev/null +++ b/tools/site_compare/utils/browser_iterate.py @@ -0,0 +1,225 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Utility to use a browser to visit multiple URLs. + +Prerequisites: + 1. The command_line package from tools/site_compare + 2. Either the IE BHO or Firefox extension (or both) + +Installation: + 1. Build the IE BHO, or call regsvr32 on a prebuilt binary + 2. Add a file called "measurepageloadtimeextension@google.com" to + the default Firefox profile directory under extensions, containing + the path to the Firefox extension root + +Invoke with the command line arguments as documented within +the command line. +""" + +import command_line +import scrapers +import socket +import time + +from drivers import windowing + +# Constants +MAX_URL = 1024 +PORT = 42492 + +def SetupIterationCommandLine(cmd): + """Adds the necessary flags for iteration to a command. + + Args: + cmd: an object created by cmdline.AddCommand + """ + cmd.AddArgument( + ["-b", "--browser"], "Browser to use (ie, firefox, chrome)", + type="string", required=True) + cmd.AddArgument( + ["-b1v", "--browserver"], "Version of browser", metaname="VERSION") + cmd.AddArgument( + ["-p", "--browserpath"], "Path to browser.", + type="string", required=False) + cmd.AddArgument( + ["-u", "--url"], "URL to visit") + cmd.AddArgument( + ["-l", "--list"], "File containing list of URLs to visit", type="readfile") + cmd.AddMutualExclusion(["--url", "--list"]) + cmd.AddArgument( + ["-s", "--startline"], "First line of URL list", type="int") + cmd.AddArgument( + ["-e", "--endline"], "Last line of URL list (exclusive)", type="int") + cmd.AddArgument( + ["-c", "--count"], "Number of lines of URL file to use", type="int") + cmd.AddDependency("--startline", "--list") + cmd.AddRequiredGroup(["--url", "--list"]) + cmd.AddDependency("--endline", "--list") + cmd.AddDependency("--count", "--list") + cmd.AddMutualExclusion(["--count", "--endline"]) + cmd.AddDependency("--count", "--startline") + cmd.AddArgument( + ["-t", "--timeout"], "Amount of time (seconds) to wait for browser to " + "finish loading", + type="int", default=300) + cmd.AddArgument( + ["-sz", "--size"], "Browser window size", default=(800, 600), type="coords") + + +def Iterate(command, iteration_func): + """Iterates over a list of URLs, calling a function on each. + + Args: + command: the command line containing the iteration flags + iteration_func: called for each URL with (proc, wnd, url, result) + """ + + # Retrieve the browser scraper to use to invoke the browser + scraper = scrapers.GetScraper((command["--browser"], command["--browserver"])) + + def AttachToBrowser(path, timeout): + """Invoke the browser process and connect to the socket.""" + (proc, frame, wnd) = scraper.GetBrowser(path) + + if not wnd: raise ValueError("Could not invoke browser.") + + # Try to connect the socket. If it fails, wait and try + # again. Do this for ten seconds + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + + for attempt in xrange(10): + try: + s.connect(("localhost", PORT)) + except socket.error: + time.sleep(1) + continue + break + + try: + s.getpeername() + except socket.error: + raise ValueError("Could not connect to browser") + + if command["--size"]: + # Resize and reposition the frame + windowing.MoveAndSizeWindow(frame, (0, 0), command["--size"], wnd) + + s.settimeout(timeout) + + Iterate.proc = proc + Iterate.wnd = wnd + Iterate.s = s + + def DetachFromBrowser(): + """Close the socket and kill the process if necessary.""" + if Iterate.s: + Iterate.s.close() + Iterate.s = None + + if Iterate.proc: + if not windowing.WaitForProcessExit(Iterate.proc, 0): + try: + windowing.EndProcess(Iterate.proc) + windowing.WaitForProcessExit(Iterate.proc, 0) + except pywintypes.error: + # Exception here most likely means the process died on its own + pass + Iterate.proc = None + + if command["--browserpath"]: + browser = command["--browserpath"] + else: + browser = None + + # Read the URLs from the file + if command["--url"]: + url_list = [command["--url"]] + else: + startline = command["--startline"] + if command["--count"]: + endline = startline+command["--count"] + else: + endline = command["--endline"] + + url_list = [] + file = open(command["--list"], "r") + + for line in xrange(startline-1): + file.readline() + + for line in xrange(endline-startline): + url_list.append(file.readline().strip()) + + timeout = command["--timeout"] + + # Loop through the URLs and send them through the socket + Iterate.s = None + Iterate.proc = None + Iterate.wnd = None + + for url in url_list: + # Invoke the browser if necessary + if not Iterate.proc: + AttachToBrowser(browser, timeout) + # Send the URL and wait for a response + Iterate.s.send(url + "\n") + + response = "" + + while (response.find("\n") < 0): + + try: + recv = Iterate.s.recv(MAX_URL) + response = response + recv + + # Workaround for an oddity: when Firefox closes + # gracefully, somehow Python doesn't detect it. + # (Telnet does) + if not recv: + raise socket.error + + except socket.timeout: + response = url + ",hang\n" + DetachFromBrowser() + except socket.error: + # If there was a socket error, it's probably a crash + response = url + ",crash\n" + DetachFromBrowser() + + # If we received a timeout response, restart the browser + if response[-9:] == ",timeout\n": + DetachFromBrowser() + + # Invoke the iteration function + iteration_func(url, Iterate.proc, Iterate.wnd, response) + + # We're done + DetachFromBrowser() diff --git a/tools/stats_viewer/OpenDialog.Designer.cs b/tools/stats_viewer/OpenDialog.Designer.cs new file mode 100644 index 0000000..3ef7a27 --- /dev/null +++ b/tools/stats_viewer/OpenDialog.Designer.cs @@ -0,0 +1,113 @@ +// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+namespace StatsViewer
+{
+ partial class OpenDialog
+ {
+ /// <summary>
+ /// Required designer variable.
+ /// </summary>
+ private System.ComponentModel.IContainer components = null;
+
+ /// <summary>
+ /// Clean up any resources being used.
+ /// </summary>
+ /// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing && (components != null))
+ {
+ components.Dispose();
+ }
+ base.Dispose(disposing);
+ }
+
+ #region Windows Form Designer generated code
+
+ /// <summary>
+ /// Required method for Designer support - do not modify
+ /// the contents of this method with the code editor.
+ /// </summary>
+ private void InitializeComponent()
+ {
+ this.name_box_ = new System.Windows.Forms.TextBox();
+ this.label1 = new System.Windows.Forms.Label();
+ this.button1 = new System.Windows.Forms.Button();
+ this.SuspendLayout();
+ //
+ // name_box_
+ //
+ this.name_box_.Location = new System.Drawing.Point(108, 25);
+ this.name_box_.Name = "name_box_";
+ this.name_box_.Size = new System.Drawing.Size(180, 20);
+ this.name_box_.TabIndex = 0;
+ this.name_box_.KeyUp += new System.Windows.Forms.KeyEventHandler(this.OnKeyUp);
+ //
+ // label1
+ //
+ this.label1.AutoSize = true;
+ this.label1.Location = new System.Drawing.Point(12, 28);
+ this.label1.Name = "label1";
+ this.label1.Size = new System.Drawing.Size(91, 13);
+ this.label1.TabIndex = 1;
+ this.label1.Text = "Stats File to Open";
+ //
+ // button1
+ //
+ this.button1.Location = new System.Drawing.Point(108, 61);
+ this.button1.Name = "button1";
+ this.button1.Size = new System.Drawing.Size(75, 23);
+ this.button1.TabIndex = 2;
+ this.button1.Text = "OK";
+ this.button1.UseVisualStyleBackColor = true;
+ this.button1.Click += new System.EventHandler(this.button1_Click);
+ //
+ // OpenDialog
+ //
+ this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
+ this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
+ this.ClientSize = new System.Drawing.Size(303, 94);
+ this.Controls.Add(this.button1);
+ this.Controls.Add(this.label1);
+ this.Controls.Add(this.name_box_);
+ this.Name = "OpenDialog";
+ this.Text = "OpenDialog";
+ this.ResumeLayout(false);
+ this.PerformLayout();
+
+ }
+
+ #endregion
+
+ private System.Windows.Forms.TextBox name_box_;
+ private System.Windows.Forms.Label label1;
+ private System.Windows.Forms.Button button1;
+ }
+}
\ No newline at end of file diff --git a/tools/stats_viewer/OpenDialog.cs b/tools/stats_viewer/OpenDialog.cs new file mode 100644 index 0000000..aac49a7 --- /dev/null +++ b/tools/stats_viewer/OpenDialog.cs @@ -0,0 +1,70 @@ +// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Data;
+using System.Drawing;
+using System.Text;
+using System.Windows.Forms;
+
+namespace StatsViewer
+{
+ public partial class OpenDialog : Form
+ {
+ public OpenDialog()
+ {
+ InitializeComponent();
+ }
+
+ /// <summary>
+ /// Get the user selected filename
+ /// </summary>
+ public string FileName
+ {
+ get {
+ return this.name_box_.Text;
+ }
+ }
+
+ private void button1_Click(object sender, EventArgs e)
+ {
+ this.Close();
+ }
+
+ private void OnKeyUp(object sender, KeyEventArgs e)
+ {
+ if (e.KeyCode == Keys.Enter)
+ {
+ this.Close();
+ }
+ }
+ }
+}
\ No newline at end of file diff --git a/tools/stats_viewer/OpenDialog.resx b/tools/stats_viewer/OpenDialog.resx new file mode 100644 index 0000000..ff31a6d --- /dev/null +++ b/tools/stats_viewer/OpenDialog.resx @@ -0,0 +1,120 @@ +<?xml version="1.0" encoding="utf-8"?>
+<root>
+ <!--
+ Microsoft ResX Schema
+
+ Version 2.0
+
+ The primary goals of this format is to allow a simple XML format
+ that is mostly human readable. The generation and parsing of the
+ various data types are done through the TypeConverter classes
+ associated with the data types.
+
+ Example:
+
+ ... ado.net/XML headers & schema ...
+ <resheader name="resmimetype">text/microsoft-resx</resheader>
+ <resheader name="version">2.0</resheader>
+ <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
+ <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
+ <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
+ <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
+ <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
+ <value>[base64 mime encoded serialized .NET Framework object]</value>
+ </data>
+ <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+ <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
+ <comment>This is a comment</comment>
+ </data>
+
+ There are any number of "resheader" rows that contain simple
+ name/value pairs.
+
+ Each data row contains a name, and value. The row also contains a
+ type or mimetype. Type corresponds to a .NET class that support
+ text/value conversion through the TypeConverter architecture.
+ Classes that don't support this are serialized and stored with the
+ mimetype set.
+
+ The mimetype is used for serialized objects, and tells the
+ ResXResourceReader how to depersist the object. This is currently not
+ extensible. For a given mimetype the value must be set accordingly:
+
+ Note - application/x-microsoft.net.object.binary.base64 is the format
+ that the ResXResourceWriter will generate, however the reader can
+ read any of the formats listed below.
+
+ mimetype: application/x-microsoft.net.object.binary.base64
+ value : The object must be serialized with
+ : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
+ : and then encoded with base64 encoding.
+
+ mimetype: application/x-microsoft.net.object.soap.base64
+ value : The object must be serialized with
+ : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
+ : and then encoded with base64 encoding.
+
+ mimetype: application/x-microsoft.net.object.bytearray.base64
+ value : The object must be serialized into a byte array
+ : using a System.ComponentModel.TypeConverter
+ : and then encoded with base64 encoding.
+ -->
+ <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
+ <xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
+ <xsd:element name="root" msdata:IsDataSet="true">
+ <xsd:complexType>
+ <xsd:choice maxOccurs="unbounded">
+ <xsd:element name="metadata">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" />
+ </xsd:sequence>
+ <xsd:attribute name="name" use="required" type="xsd:string" />
+ <xsd:attribute name="type" type="xsd:string" />
+ <xsd:attribute name="mimetype" type="xsd:string" />
+ <xsd:attribute ref="xml:space" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="assembly">
+ <xsd:complexType>
+ <xsd:attribute name="alias" type="xsd:string" />
+ <xsd:attribute name="name" type="xsd:string" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="data">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+ <xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
+ </xsd:sequence>
+ <xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
+ <xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
+ <xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
+ <xsd:attribute ref="xml:space" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="resheader">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+ </xsd:sequence>
+ <xsd:attribute name="name" type="xsd:string" use="required" />
+ </xsd:complexType>
+ </xsd:element>
+ </xsd:choice>
+ </xsd:complexType>
+ </xsd:element>
+ </xsd:schema>
+ <resheader name="resmimetype">
+ <value>text/microsoft-resx</value>
+ </resheader>
+ <resheader name="version">
+ <value>2.0</value>
+ </resheader>
+ <resheader name="reader">
+ <value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+ </resheader>
+ <resheader name="writer">
+ <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+ </resheader>
+</root>
\ No newline at end of file diff --git a/tools/stats_viewer/Properties/AssemblyInfo.cs b/tools/stats_viewer/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..8ffae14 --- /dev/null +++ b/tools/stats_viewer/Properties/AssemblyInfo.cs @@ -0,0 +1,33 @@ +using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("StatsViewer")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("Google")]
+[assembly: AssemblyProduct("StatsViewer")]
+[assembly: AssemblyCopyright("Copyright © Google 2007")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components. If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("0c5760e1-3ada-48e2-800d-5b104bef4a95")]
+
+// Version information for an assembly consists of the following four values:
+//
+// Major Version
+// Minor Version
+// Build Number
+// Revision
+//
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/tools/stats_viewer/Properties/Resources.Designer.cs b/tools/stats_viewer/Properties/Resources.Designer.cs new file mode 100644 index 0000000..4195c5f --- /dev/null +++ b/tools/stats_viewer/Properties/Resources.Designer.cs @@ -0,0 +1,71 @@ +//------------------------------------------------------------------------------
+// <auto-generated>
+// This code was generated by a tool.
+// Runtime Version:2.0.50727.832
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+namespace StatsViewer.Properties
+{
+
+
+ /// <summary>
+ /// A strongly-typed resource class, for looking up localized strings, etc.
+ /// </summary>
+ // This class was auto-generated by the StronglyTypedResourceBuilder
+ // class via a tool like ResGen or Visual Studio.
+ // To add or remove a member, edit your .ResX file then rerun ResGen
+ // with the /str option, or rebuild your VS project.
+ [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "2.0.0.0")]
+ [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+ [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
+ internal class Resources
+ {
+
+ private static global::System.Resources.ResourceManager resourceMan;
+
+ private static global::System.Globalization.CultureInfo resourceCulture;
+
+ [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
+ internal Resources()
+ {
+ }
+
+ /// <summary>
+ /// Returns the cached ResourceManager instance used by this class.
+ /// </summary>
+ [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
+ internal static global::System.Resources.ResourceManager ResourceManager
+ {
+ get
+ {
+ if ((resourceMan == null))
+ {
+ global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("StatsViewer.Properties.Resources", typeof(Resources).Assembly);
+ resourceMan = temp;
+ }
+ return resourceMan;
+ }
+ }
+
+ /// <summary>
+ /// Overrides the current thread's CurrentUICulture property for all
+ /// resource lookups using this strongly typed resource class.
+ /// </summary>
+ [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
+ internal static global::System.Globalization.CultureInfo Culture
+ {
+ get
+ {
+ return resourceCulture;
+ }
+ set
+ {
+ resourceCulture = value;
+ }
+ }
+ }
+}
diff --git a/tools/stats_viewer/Properties/Resources.resx b/tools/stats_viewer/Properties/Resources.resx new file mode 100644 index 0000000..ffecec8 --- /dev/null +++ b/tools/stats_viewer/Properties/Resources.resx @@ -0,0 +1,117 @@ +<?xml version="1.0" encoding="utf-8"?>
+<root>
+ <!--
+ Microsoft ResX Schema
+
+ Version 2.0
+
+ The primary goals of this format is to allow a simple XML format
+ that is mostly human readable. The generation and parsing of the
+ various data types are done through the TypeConverter classes
+ associated with the data types.
+
+ Example:
+
+ ... ado.net/XML headers & schema ...
+ <resheader name="resmimetype">text/microsoft-resx</resheader>
+ <resheader name="version">2.0</resheader>
+ <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
+ <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
+ <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
+ <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
+ <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
+ <value>[base64 mime encoded serialized .NET Framework object]</value>
+ </data>
+ <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+ <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
+ <comment>This is a comment</comment>
+ </data>
+
+ There are any number of "resheader" rows that contain simple
+ name/value pairs.
+
+ Each data row contains a name, and value. The row also contains a
+ type or mimetype. Type corresponds to a .NET class that support
+ text/value conversion through the TypeConverter architecture.
+ Classes that don't support this are serialized and stored with the
+ mimetype set.
+
+ The mimetype is used for serialized objects, and tells the
+ ResXResourceReader how to depersist the object. This is currently not
+ extensible. For a given mimetype the value must be set accordingly:
+
+ Note - application/x-microsoft.net.object.binary.base64 is the format
+ that the ResXResourceWriter will generate, however the reader can
+ read any of the formats listed below.
+
+ mimetype: application/x-microsoft.net.object.binary.base64
+ value : The object must be serialized with
+ : System.Serialization.Formatters.Binary.BinaryFormatter
+ : and then encoded with base64 encoding.
+
+ mimetype: application/x-microsoft.net.object.soap.base64
+ value : The object must be serialized with
+ : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
+ : and then encoded with base64 encoding.
+
+ mimetype: application/x-microsoft.net.object.bytearray.base64
+ value : The object must be serialized into a byte array
+ : using a System.ComponentModel.TypeConverter
+ : and then encoded with base64 encoding.
+ -->
+ <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
+ <xsd:element name="root" msdata:IsDataSet="true">
+ <xsd:complexType>
+ <xsd:choice maxOccurs="unbounded">
+ <xsd:element name="metadata">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" />
+ </xsd:sequence>
+ <xsd:attribute name="name" type="xsd:string" />
+ <xsd:attribute name="type" type="xsd:string" />
+ <xsd:attribute name="mimetype" type="xsd:string" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="assembly">
+ <xsd:complexType>
+ <xsd:attribute name="alias" type="xsd:string" />
+ <xsd:attribute name="name" type="xsd:string" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="data">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+ <xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
+ </xsd:sequence>
+ <xsd:attribute name="name" type="xsd:string" msdata:Ordinal="1" />
+ <xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
+ <xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="resheader">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+ </xsd:sequence>
+ <xsd:attribute name="name" type="xsd:string" use="required" />
+ </xsd:complexType>
+ </xsd:element>
+ </xsd:choice>
+ </xsd:complexType>
+ </xsd:element>
+ </xsd:schema>
+ <resheader name="resmimetype">
+ <value>text/microsoft-resx</value>
+ </resheader>
+ <resheader name="version">
+ <value>2.0</value>
+ </resheader>
+ <resheader name="reader">
+ <value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+ </resheader>
+ <resheader name="writer">
+ <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+ </resheader>
+</root>
\ No newline at end of file diff --git a/tools/stats_viewer/Properties/Settings.Designer.cs b/tools/stats_viewer/Properties/Settings.Designer.cs new file mode 100644 index 0000000..221765a --- /dev/null +++ b/tools/stats_viewer/Properties/Settings.Designer.cs @@ -0,0 +1,30 @@ +//------------------------------------------------------------------------------
+// <auto-generated>
+// This code was generated by a tool.
+// Runtime Version:2.0.50727.832
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+namespace StatsViewer.Properties
+{
+
+
+ [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
+ [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "8.0.0.0")]
+ internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase
+ {
+
+ private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings())));
+
+ public static Settings Default
+ {
+ get
+ {
+ return defaultInstance;
+ }
+ }
+ }
+}
diff --git a/tools/stats_viewer/Properties/Settings.settings b/tools/stats_viewer/Properties/Settings.settings new file mode 100644 index 0000000..abf36c5 --- /dev/null +++ b/tools/stats_viewer/Properties/Settings.settings @@ -0,0 +1,7 @@ +<?xml version='1.0' encoding='utf-8'?>
+<SettingsFile xmlns="http://schemas.microsoft.com/VisualStudio/2004/01/settings" CurrentProfile="(Default)">
+ <Profiles>
+ <Profile Name="(Default)" />
+ </Profiles>
+ <Settings />
+</SettingsFile>
diff --git a/tools/stats_viewer/Resources.Designer.cs b/tools/stats_viewer/Resources.Designer.cs new file mode 100644 index 0000000..2d915e0 --- /dev/null +++ b/tools/stats_viewer/Resources.Designer.cs @@ -0,0 +1,77 @@ +//------------------------------------------------------------------------------
+// <auto-generated>
+// This code was generated by a tool.
+// Runtime Version:2.0.50727.832
+//
+// Changes to this file may cause incorrect behavior and will be lost if
+// the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+namespace StatsViewer {
+ using System;
+
+
+ /// <summary>
+ /// A strongly-typed resource class, for looking up localized strings, etc.
+ /// </summary>
+ // This class was auto-generated by the StronglyTypedResourceBuilder
+ // class via a tool like ResGen or Visual Studio.
+ // To add or remove a member, edit your .ResX file then rerun ResGen
+ // with the /str option, or rebuild your VS project.
+ [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "2.0.0.0")]
+ [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+ [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
+ internal class Resources {
+
+ private static global::System.Resources.ResourceManager resourceMan;
+
+ private static global::System.Globalization.CultureInfo resourceCulture;
+
+ [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")]
+ internal Resources() {
+ }
+
+ /// <summary>
+ /// Returns the cached ResourceManager instance used by this class.
+ /// </summary>
+ [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
+ internal static global::System.Resources.ResourceManager ResourceManager {
+ get {
+ if (object.ReferenceEquals(resourceMan, null)) {
+ global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("StatsViewer.Resources", typeof(Resources).Assembly);
+ resourceMan = temp;
+ }
+ return resourceMan;
+ }
+ }
+
+ /// <summary>
+ /// Overrides the current thread's CurrentUICulture property for all
+ /// resource lookups using this strongly typed resource class.
+ /// </summary>
+ [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]
+ internal static global::System.Globalization.CultureInfo Culture {
+ get {
+ return resourceCulture;
+ }
+ set {
+ resourceCulture = value;
+ }
+ }
+
+ internal static System.Drawing.Bitmap kitten {
+ get {
+ object obj = ResourceManager.GetObject("kitten", resourceCulture);
+ return ((System.Drawing.Bitmap)(obj));
+ }
+ }
+
+ internal static System.Drawing.Bitmap kittenbackground {
+ get {
+ object obj = ResourceManager.GetObject("kittenbackground", resourceCulture);
+ return ((System.Drawing.Bitmap)(obj));
+ }
+ }
+ }
+}
diff --git a/tools/stats_viewer/Resources.resx b/tools/stats_viewer/Resources.resx new file mode 100644 index 0000000..9486b21 --- /dev/null +++ b/tools/stats_viewer/Resources.resx @@ -0,0 +1,127 @@ +<?xml version="1.0" encoding="utf-8"?>
+<root>
+ <!--
+ Microsoft ResX Schema
+
+ Version 2.0
+
+ The primary goals of this format is to allow a simple XML format
+ that is mostly human readable. The generation and parsing of the
+ various data types are done through the TypeConverter classes
+ associated with the data types.
+
+ Example:
+
+ ... ado.net/XML headers & schema ...
+ <resheader name="resmimetype">text/microsoft-resx</resheader>
+ <resheader name="version">2.0</resheader>
+ <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
+ <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
+ <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
+ <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
+ <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
+ <value>[base64 mime encoded serialized .NET Framework object]</value>
+ </data>
+ <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+ <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
+ <comment>This is a comment</comment>
+ </data>
+
+ There are any number of "resheader" rows that contain simple
+ name/value pairs.
+
+ Each data row contains a name, and value. The row also contains a
+ type or mimetype. Type corresponds to a .NET class that support
+ text/value conversion through the TypeConverter architecture.
+ Classes that don't support this are serialized and stored with the
+ mimetype set.
+
+ The mimetype is used for serialized objects, and tells the
+ ResXResourceReader how to depersist the object. This is currently not
+ extensible. For a given mimetype the value must be set accordingly:
+
+ Note - application/x-microsoft.net.object.binary.base64 is the format
+ that the ResXResourceWriter will generate, however the reader can
+ read any of the formats listed below.
+
+ mimetype: application/x-microsoft.net.object.binary.base64
+ value : The object must be serialized with
+ : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
+ : and then encoded with base64 encoding.
+
+ mimetype: application/x-microsoft.net.object.soap.base64
+ value : The object must be serialized with
+ : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
+ : and then encoded with base64 encoding.
+
+ mimetype: application/x-microsoft.net.object.bytearray.base64
+ value : The object must be serialized into a byte array
+ : using a System.ComponentModel.TypeConverter
+ : and then encoded with base64 encoding.
+ -->
+ <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
+ <xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
+ <xsd:element name="root" msdata:IsDataSet="true">
+ <xsd:complexType>
+ <xsd:choice maxOccurs="unbounded">
+ <xsd:element name="metadata">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" />
+ </xsd:sequence>
+ <xsd:attribute name="name" use="required" type="xsd:string" />
+ <xsd:attribute name="type" type="xsd:string" />
+ <xsd:attribute name="mimetype" type="xsd:string" />
+ <xsd:attribute ref="xml:space" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="assembly">
+ <xsd:complexType>
+ <xsd:attribute name="alias" type="xsd:string" />
+ <xsd:attribute name="name" type="xsd:string" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="data">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+ <xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
+ </xsd:sequence>
+ <xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
+ <xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
+ <xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
+ <xsd:attribute ref="xml:space" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="resheader">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+ </xsd:sequence>
+ <xsd:attribute name="name" type="xsd:string" use="required" />
+ </xsd:complexType>
+ </xsd:element>
+ </xsd:choice>
+ </xsd:complexType>
+ </xsd:element>
+ </xsd:schema>
+ <resheader name="resmimetype">
+ <value>text/microsoft-resx</value>
+ </resheader>
+ <resheader name="version">
+ <value>2.0</value>
+ </resheader>
+ <resheader name="reader">
+ <value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+ </resheader>
+ <resheader name="writer">
+ <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+ </resheader>
+ <assembly alias="System.Windows.Forms" name="System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" />
+ <data name="kitten" type="System.Resources.ResXFileRef, System.Windows.Forms">
+ <value>Resources\kitten.png;System.Drawing.Bitmap, System.Drawing, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
+ </data>
+ <data name="kittenbackground" type="System.Resources.ResXFileRef, System.Windows.Forms">
+ <value>Resources\kittenbackground.png;System.Drawing.Bitmap, System.Drawing, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
+ </data>
+</root>
\ No newline at end of file diff --git a/tools/stats_viewer/Resources/kitten.png b/tools/stats_viewer/Resources/kitten.png Binary files differnew file mode 100644 index 0000000..51cc9b8 --- /dev/null +++ b/tools/stats_viewer/Resources/kitten.png diff --git a/tools/stats_viewer/Resources/kittenbackground.png b/tools/stats_viewer/Resources/kittenbackground.png Binary files differnew file mode 100644 index 0000000..09a7268 --- /dev/null +++ b/tools/stats_viewer/Resources/kittenbackground.png diff --git a/tools/stats_viewer/program.cs b/tools/stats_viewer/program.cs new file mode 100644 index 0000000..2668cec --- /dev/null +++ b/tools/stats_viewer/program.cs @@ -0,0 +1,48 @@ +// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+using System;
+using System.Windows.Forms;
+
+namespace StatsViewer
+{
+ static class Program
+ {
+ /// <summary>
+ /// The main entry point for the application.
+ /// </summary>
+ [STAThread]
+ static void Main()
+ {
+ Application.EnableVisualStyles();
+ Application.SetCompatibleTextRenderingDefault(false);
+ Application.Run(new StatsViewer());
+ }
+ }
+}
\ No newline at end of file diff --git a/tools/stats_viewer/stats_table.cs b/tools/stats_viewer/stats_table.cs new file mode 100644 index 0000000..a2f87cf --- /dev/null +++ b/tools/stats_viewer/stats_table.cs @@ -0,0 +1,571 @@ +// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace StatsViewer
+{
+ /// <summary>
+ /// The stats table shared memory segment contains this
+ /// header structure.
+ /// </summary>
+ [StructLayout(LayoutKind.Sequential)]
+ internal struct StatsFileHeader {
+ public int version;
+ public int size;
+ public int max_counters;
+ public int max_threads;
+ };
+
+ /// <summary>
+ /// An entry in the StatsTable.
+ /// </summary>
+ class StatsTableEntry {
+ public StatsTableEntry(int id, string name, StatsTable table) {
+ id_ = id;
+ name_ = name;
+ table_ = table;
+ }
+
+ /// <summary>
+ /// The unique id for this entry
+ /// </summary>
+ public int id { get { return id_; } }
+
+ /// <summary>
+ /// The name for this entry.
+ /// </summary>
+ public string name { get { return name_; } }
+
+ /// <summary>
+ /// The value of this entry now.
+ /// </summary>
+ public int GetValue(int filter_pid) {
+ return table_.GetValue(id_, filter_pid);
+ }
+
+ private int id_;
+ private string name_;
+ private StatsTable table_;
+ }
+
+ // An interface for StatsCounters
+ interface IStatsCounter {
+ // The name of the counter
+ string name { get; }
+ }
+
+ // A counter.
+ class StatsCounter : IStatsCounter {
+ public StatsCounter(StatsTableEntry entry) {
+ entry_ = entry;
+ }
+
+ public string name {
+ get {
+ return entry_.name;
+ }
+ }
+
+ public int GetValue(int filter_pid) {
+ return entry_.GetValue(filter_pid);
+ }
+
+ private StatsTableEntry entry_;
+ }
+
+ // A timer.
+ class StatsTimer : IStatsCounter {
+ public StatsTimer(StatsTableEntry entry)
+ {
+ entry_ = entry;
+ }
+
+ public string name {
+ get {
+ return entry_.name;
+ }
+ }
+
+ public int GetValue(int filter_pid) {
+ return entry_.GetValue(filter_pid);
+ }
+
+ private StatsTableEntry entry_;
+ }
+
+ // A rate.
+ class StatsCounterRate : IStatsCounter
+ {
+ public StatsCounterRate(StatsCounter counter, StatsTimer timer) {
+ counter_ = counter;
+ timer_ = timer;
+ }
+
+ public string name { get { return counter_.name; } }
+
+ public int GetCount(int filter_pid) {
+ return counter_.GetValue(filter_pid);
+ }
+
+ public int GetTime(int filter_pid) {
+ return timer_.GetValue(filter_pid);
+ }
+
+ private StatsCounter counter_;
+ private StatsTimer timer_;
+ }
+
+ /// <summary>
+ /// This is a C# reader for the chrome stats_table.
+ /// </summary>
+ class StatsTable {
+ internal const int kMaxThreadNameLength = 32;
+ internal const int kMaxCounterNameLength = 32;
+
+ /// <summary>
+ /// Open a StatsTable
+ /// </summary>
+ public StatsTable() {
+ }
+
+ #region Public Properties
+ /// <summary>
+ /// Get access to the counters in the table.
+ /// </summary>
+ public StatsTableCounters Counters() {
+ return new StatsTableCounters(this);
+ }
+
+ /// <summary>
+ /// Get access to the processes in the table
+ /// </summary>
+ public ICollection Processes {
+ get {
+ return new StatsTableProcesses(this);
+ }
+ }
+ #endregion
+
+ #region Internal Properties
+ //
+ // The internal methods are accessible to the enumerators
+ // and helper classes below.
+ //
+
+ /// <summary>
+ /// Access to the table header
+ /// </summary>
+ internal StatsFileHeader Header {
+ get { return header_; }
+ }
+
+ /// <summary>
+ /// Get the offset of the ThreadName table
+ /// </summary>
+ internal long ThreadNamesOffset {
+ get {
+ return memory_.ToInt64() + Marshal.SizeOf(typeof(StatsFileHeader));
+ }
+ }
+
+ /// <summary>
+ /// Get the offset of the PIDs table
+ /// </summary>
+ internal long PidsOffset {
+ get {
+ long offset = ThreadNamesOffset;
+ // Thread names table
+ offset += AlignedSize(header_.max_threads * kMaxThreadNameLength * 2);
+ // Thread TID table
+ offset += AlignedSize(header_.max_threads *
+ Marshal.SizeOf(typeof(int)));
+ return offset;
+ }
+ }
+
+ /// <summary>
+ /// Get the offset of the CounterName table
+ /// </summary>
+ internal long CounterNamesOffset {
+ get {
+ long offset = PidsOffset;
+ // Thread PID table
+ offset += AlignedSize(header_.max_threads *
+ Marshal.SizeOf(typeof(int)));
+ return offset;
+ }
+ }
+
+ /// <summary>
+ /// Get the offset of the Data table
+ /// </summary>
+ internal long DataOffset {
+ get {
+ long offset = CounterNamesOffset;
+ // Counter names table
+ offset += AlignedSize(header_.max_counters *
+ kMaxCounterNameLength * 2);
+ return offset;
+ }
+ }
+ #endregion
+
+ #region Public Methods
+ /// <summary>
+ /// Opens the memory map
+ /// </summary>
+ /// <returns></returns>
+ /// <param name="name">The name of the file to open</param>
+ public bool Open(string name) {
+ map_handle_ =
+ Win32.OpenFileMapping((int)Win32.MapAccess.FILE_MAP_WRITE, false,
+ name);
+ if (map_handle_ == IntPtr.Zero)
+ return false;
+
+ memory_ =
+ Win32.MapViewOfFile(map_handle_, (int)Win32.MapAccess.FILE_MAP_WRITE,
+ 0,0, 0);
+ if (memory_ == IntPtr.Zero) {
+ Win32.CloseHandle(map_handle_);
+ return false;
+ }
+
+ header_ = (StatsFileHeader)Marshal.PtrToStructure(memory_, header_.GetType());
+ return true;
+ }
+
+ /// <summary>
+ /// Close the mapped file.
+ /// </summary>
+ public void Close() {
+ Win32.UnmapViewOfFile(memory_);
+ Win32.CloseHandle(map_handle_);
+ }
+
+ /// <summary>
+ /// Zero out the stats file.
+ /// </summary>
+ public void Zero() {
+ long offset = DataOffset;
+ for (int threads = 0; threads < header_.max_threads; threads++) {
+ for (int counters = 0; counters < header_.max_counters; counters++) {
+ Marshal.WriteInt32((IntPtr) offset, 0);
+ offset += Marshal.SizeOf(typeof(int));
+ }
+ }
+ }
+
+ /// <summary>
+ /// Get the value for a StatsCounterEntry now.
+ /// </summary>
+ /// <returns></returns>
+ /// <param name="filter_pid">If a specific PID is being queried, filter to this PID. 0 means use all data.</param>
+ /// <param name="id">The id of the CounterEntry to get the value for.</param>
+ public int GetValue(int id, int filter_pid) {
+ long pid_offset = PidsOffset;
+ long data_offset = DataOffset;
+ data_offset += id * (Header.max_threads *
+ Marshal.SizeOf(typeof(int)));
+ int rv = 0;
+ for (int cols = 0; cols < Header.max_threads; cols++)
+ {
+ int pid = Marshal.ReadInt32((IntPtr)pid_offset);
+ if (filter_pid == 0 || filter_pid == pid)
+ {
+ rv += Marshal.ReadInt32((IntPtr)data_offset);
+ }
+ data_offset += Marshal.SizeOf(typeof(int));
+ pid_offset += Marshal.SizeOf(typeof(int));
+ }
+ return rv;
+ }
+ #endregion
+
+ #region Private Methods
+ /// <summary>
+ /// Align to 4-byte boundaries
+ /// </summary>
+ /// <param name="size"></param>
+ /// <returns></returns>
+ private long AlignedSize(long size) {
+ Debug.Assert(sizeof(int) == 4);
+ return size + (sizeof(int) - (size % sizeof(int))) % sizeof(int);
+ }
+ #endregion
+
+ #region Private Members
+ private IntPtr memory_;
+ private IntPtr map_handle_;
+ private StatsFileHeader header_;
+ #endregion
+ }
+
+ /// <summary>
+ /// Enumerable list of Counters in the StatsTable
+ /// </summary>
+ class StatsTableCounters : ICollection {
+ /// <summary>
+ /// Create the list of counters
+ /// </summary>
+ /// <param name="table"></param>
+ /// pid</param>
+ public StatsTableCounters(StatsTable table) {
+ table_ = table;
+ counter_hi_water_mark_ = -1;
+ counters_ = new List<IStatsCounter>();
+ FindCounters();
+ }
+
+ /// <summary>
+ /// Scans the table for new entries.
+ /// </summary>
+ public void Update() {
+ FindCounters();
+ }
+
+ #region IEnumerable Members
+ public IEnumerator GetEnumerator() {
+ return counters_.GetEnumerator();
+ }
+ #endregion
+
+ #region ICollection Members
+ public void CopyTo(Array array, int index) {
+ throw new Exception("The method or operation is not implemented.");
+ }
+
+ public int Count {
+ get {
+ return counters_.Count;
+ }
+ }
+
+ public bool IsSynchronized {
+ get {
+ throw new Exception("The method or operation is not implemented.");
+ }
+ }
+
+ public object SyncRoot {
+ get {
+ throw new Exception("The method or operation is not implemented.");
+ }
+ }
+ #endregion
+
+ #region Private Methods
+ /// <summary>
+ /// Create a counter based on an entry
+ /// </summary>
+ /// <param name="id"></param>
+ /// <param name="name"></param>
+ /// <returns></returns>
+ private IStatsCounter NameToCounter(int id, string name)
+ {
+ IStatsCounter rv = null;
+
+ // check if the name has a type encoded
+ if (name.Length > 2 && name[1] == ':')
+ {
+ StatsTableEntry entry = new StatsTableEntry(id, name.Substring(2), table_);
+ switch (name[0])
+ {
+ case 't':
+ rv = new StatsTimer(entry);
+ break;
+ case 'c':
+ rv = new StatsCounter(entry);
+ break;
+ }
+ }
+ else
+ {
+ StatsTableEntry entry = new StatsTableEntry(id, name, table_);
+ rv = new StatsCounter(entry);
+ }
+
+ return rv;
+ }
+
+ // If we have two StatsTableEntries with the same name,
+ // attempt to upgrade them to a higher level type.
+ // Example: A counter + a timer == a rate!
+ private void UpgradeCounter(IStatsCounter old_counter, IStatsCounter counter)
+ {
+ if (old_counter is StatsCounter && counter is StatsTimer)
+ {
+ StatsCounterRate rate = new StatsCounterRate(old_counter as StatsCounter,
+ counter as StatsTimer);
+ counters_.Remove(old_counter);
+ counters_.Add(rate);
+ }
+ else if (old_counter is StatsTimer && counter is StatsCounter)
+ {
+ StatsCounterRate rate = new StatsCounterRate(counter as StatsCounter,
+ old_counter as StatsTimer);
+ counters_.Remove(old_counter);
+ counters_.Add(rate);
+ }
+ }
+
+ /// <summary>
+ /// Find the counters in the table and insert into the counters_
+ /// hash table.
+ /// </summary>
+ private void FindCounters()
+ {
+ Debug.Assert(table_.Header.max_counters > 0);
+
+ int index = counter_hi_water_mark_;
+
+ do
+ {
+ // Find an entry in the table.
+ index++;
+ long offset = table_.CounterNamesOffset +
+ (index * StatsTable.kMaxCounterNameLength * 2);
+ string name = Marshal.PtrToStringUni((IntPtr)offset);
+ if (name.Length == 0)
+ continue;
+
+ // Record that we've already looked at this StatsTableEntry.
+ counter_hi_water_mark_ = index;
+
+ IStatsCounter counter = NameToCounter(index, name);
+
+ if (counter != null)
+ {
+ IStatsCounter old_counter = FindExistingCounter(counter.name);
+ if (old_counter != null)
+ UpgradeCounter(old_counter, counter);
+ else
+ counters_.Add(counter);
+ }
+ } while (index < table_.Header.max_counters - 1);
+ }
+
+ /// <summary>
+ /// Find an existing counter in our table
+ /// </summary>
+ /// <param name="name"></param>
+ private IStatsCounter FindExistingCounter(string name) {
+ foreach (IStatsCounter ctr in counters_)
+ {
+ if (ctr.name == name)
+ return ctr;
+ }
+ return null;
+ }
+ #endregion
+
+ #region Private Members
+ private StatsTable table_;
+ private List<IStatsCounter> counters_;
+ // Highest index of counters processed.
+ private int counter_hi_water_mark_;
+ #endregion
+ }
+
+ /// <summary>
+ /// A collection of processes
+ /// </summary>
+ class StatsTableProcesses : ICollection
+ {
+ /// <summary>
+ /// Constructor
+ /// </summary>
+ /// <param name="table"></param>
+ public StatsTableProcesses(StatsTable table) {
+ table_ = table;
+ pids_ = new List<int>();
+ Initialize();
+ }
+
+ #region ICollection Members
+ public void CopyTo(Array array, int index) {
+ throw new Exception("The method or operation is not implemented.");
+ }
+
+ public int Count {
+ get {
+ return pids_.Count;
+ }
+ }
+
+ public bool IsSynchronized {
+ get {
+ throw new Exception("The method or operation is not implemented.");
+ }
+ }
+
+ public object SyncRoot {
+ get {
+ throw new Exception("The method or operation is not implemented.");
+ }
+ }
+ #endregion
+
+ #region IEnumerable Members
+ public IEnumerator GetEnumerator() {
+ return pids_.GetEnumerator();
+ }
+ #endregion
+
+ /// <summary>
+ /// Initialize the pid list.
+ /// </summary>
+ private void Initialize() {
+ long offset = table_.ThreadNamesOffset;
+
+ for (int index = 0; index < table_.Header.max_threads; index++) {
+ string thread_name = Marshal.PtrToStringUni((IntPtr)offset);
+ if (thread_name.Length > 0) {
+ long pidOffset = table_.PidsOffset + index *
+ Marshal.SizeOf(typeof(int));
+ int pid = Marshal.ReadInt32((IntPtr)pidOffset);
+ if (!pids_.Contains(pid))
+ pids_.Add(pid);
+ }
+ offset += StatsTable.kMaxThreadNameLength * 2;
+ }
+ }
+
+ #region Private Members
+ private StatsTable table_;
+ private List<int> pids_;
+ #endregion
+ }
+}
diff --git a/tools/stats_viewer/stats_viewer.Designer.cs b/tools/stats_viewer/stats_viewer.Designer.cs new file mode 100644 index 0000000..f697c5b --- /dev/null +++ b/tools/stats_viewer/stats_viewer.Designer.cs @@ -0,0 +1,417 @@ +// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+namespace StatsViewer
+{
+ partial class StatsViewer
+ {
+ /// <summary>
+ /// Required designer variable.
+ /// </summary>
+ private System.ComponentModel.IContainer components = null;
+
+ /// <summary>
+ /// Clean up any resources being used.
+ /// </summary>
+ /// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing && (components != null))
+ {
+ components.Dispose();
+ }
+ base.Dispose(disposing);
+ }
+
+ #region Windows Form Designer generated code
+
+ /// <summary>
+ /// Required method for Designer support - do not modify
+ /// the contents of this method with the code editor.
+ /// </summary>
+ private void InitializeComponent()
+ {
+ System.ComponentModel.ComponentResourceManager resources = new System.ComponentModel.ComponentResourceManager(typeof(StatsViewer));
+ this.listViewCounters = new System.Windows.Forms.ListView();
+ this.columnHeaderName = new System.Windows.Forms.ColumnHeader();
+ this.columnHeaderValue = new System.Windows.Forms.ColumnHeader();
+ this.columnHeaderDelta = new System.Windows.Forms.ColumnHeader();
+ this.pictureBoxTitle = new System.Windows.Forms.PictureBox();
+ this.panelHeader = new System.Windows.Forms.Panel();
+ this.labelKills = new System.Windows.Forms.Label();
+ this.label1 = new System.Windows.Forms.Label();
+ this.labelInterval = new System.Windows.Forms.Label();
+ this.comboBoxFilter = new System.Windows.Forms.ComboBox();
+ this.panelControls = new System.Windows.Forms.Panel();
+ this.buttonExport = new System.Windows.Forms.Button();
+ this.buttonZero = new System.Windows.Forms.Button();
+ this.comboBoxInterval = new System.Windows.Forms.ComboBox();
+ this.labelFilter = new System.Windows.Forms.Label();
+ this.saveFileDialogExport = new System.Windows.Forms.SaveFileDialog();
+ this.menuStrip1 = new System.Windows.Forms.MenuStrip();
+ this.fileToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
+ this.openToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
+ this.closeToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
+ this.quitToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
+ this.listViewRates = new System.Windows.Forms.ListView();
+ this.columnHeaderRateName = new System.Windows.Forms.ColumnHeader();
+ this.columnHeaderRateCount = new System.Windows.Forms.ColumnHeader();
+ this.columnHeaderRateTotaltime = new System.Windows.Forms.ColumnHeader();
+ this.columnHeaderRateAvgTime = new System.Windows.Forms.ColumnHeader();
+ this.splitContainer1 = new System.Windows.Forms.SplitContainer();
+ ((System.ComponentModel.ISupportInitialize)(this.pictureBoxTitle)).BeginInit();
+ this.panelHeader.SuspendLayout();
+ this.panelControls.SuspendLayout();
+ this.menuStrip1.SuspendLayout();
+ this.splitContainer1.Panel1.SuspendLayout();
+ this.splitContainer1.Panel2.SuspendLayout();
+ this.splitContainer1.SuspendLayout();
+ this.SuspendLayout();
+ //
+ // listViewCounters
+ //
+ this.listViewCounters.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] {
+ this.columnHeaderName,
+ this.columnHeaderValue,
+ this.columnHeaderDelta});
+ this.listViewCounters.Dock = System.Windows.Forms.DockStyle.Fill;
+ this.listViewCounters.FullRowSelect = true;
+ this.listViewCounters.Location = new System.Drawing.Point(0, 0);
+ this.listViewCounters.Name = "listViewCounters";
+ this.listViewCounters.Size = new System.Drawing.Size(505, 221);
+ this.listViewCounters.Sorting = System.Windows.Forms.SortOrder.Descending;
+ this.listViewCounters.TabIndex = 0;
+ this.listViewCounters.UseCompatibleStateImageBehavior = false;
+ this.listViewCounters.View = System.Windows.Forms.View.Details;
+ this.listViewCounters.ColumnClick += new System.Windows.Forms.ColumnClickEventHandler(this.column_Click);
+ //
+ // columnHeaderName
+ //
+ this.columnHeaderName.Text = "Counter Name";
+ this.columnHeaderName.Width = 203;
+ //
+ // columnHeaderValue
+ //
+ this.columnHeaderValue.Text = "Value";
+ this.columnHeaderValue.Width = 69;
+ //
+ // columnHeaderDelta
+ //
+ this.columnHeaderDelta.Text = "Delta";
+ this.columnHeaderDelta.Width = 86;
+ //
+ // pictureBoxTitle
+ //
+ this.pictureBoxTitle.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
+ this.pictureBoxTitle.BackColor = System.Drawing.Color.Transparent;
+ this.pictureBoxTitle.Image = ((System.Drawing.Image)(resources.GetObject("pictureBoxTitle.Image")));
+ this.pictureBoxTitle.Location = new System.Drawing.Point(257, 0);
+ this.pictureBoxTitle.Name = "pictureBoxTitle";
+ this.pictureBoxTitle.Size = new System.Drawing.Size(248, 86);
+ this.pictureBoxTitle.TabIndex = 1;
+ this.pictureBoxTitle.TabStop = false;
+ //
+ // panelHeader
+ //
+ this.panelHeader.BackgroundImage = ((System.Drawing.Image)(resources.GetObject("panelHeader.BackgroundImage")));
+ this.panelHeader.BackgroundImageLayout = System.Windows.Forms.ImageLayout.Stretch;
+ this.panelHeader.Controls.Add(this.labelKills);
+ this.panelHeader.Controls.Add(this.label1);
+ this.panelHeader.Controls.Add(this.pictureBoxTitle);
+ this.panelHeader.Dock = System.Windows.Forms.DockStyle.Top;
+ this.panelHeader.Location = new System.Drawing.Point(0, 24);
+ this.panelHeader.Name = "panelHeader";
+ this.panelHeader.Size = new System.Drawing.Size(505, 86);
+ this.panelHeader.TabIndex = 2;
+ //
+ // labelKills
+ //
+ this.labelKills.AutoSize = true;
+ this.labelKills.BackColor = System.Drawing.Color.Transparent;
+ this.labelKills.Font = new System.Drawing.Font("Arial", 9.75F, System.Drawing.FontStyle.Italic, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
+ this.labelKills.Location = new System.Drawing.Point(12, 33);
+ this.labelKills.Name = "labelKills";
+ this.labelKills.Size = new System.Drawing.Size(280, 16);
+ this.labelKills.TabIndex = 3;
+ this.labelKills.Text = "During the World Wide Wait, God Kills Kittens.";
+ //
+ // label1
+ //
+ this.label1.AutoSize = true;
+ this.label1.BackColor = System.Drawing.Color.Transparent;
+ this.label1.Font = new System.Drawing.Font("Arial", 15.75F, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
+ this.label1.Location = new System.Drawing.Point(12, 9);
+ this.label1.Name = "label1";
+ this.label1.Size = new System.Drawing.Size(140, 24);
+ this.label1.TabIndex = 2;
+ this.label1.Text = "Chrome Varz";
+ //
+ // labelInterval
+ //
+ this.labelInterval.AutoSize = true;
+ this.labelInterval.Location = new System.Drawing.Point(11, 9);
+ this.labelInterval.Name = "labelInterval";
+ this.labelInterval.Size = new System.Drawing.Size(73, 13);
+ this.labelInterval.TabIndex = 3;
+ this.labelInterval.Text = "Interval (secs)";
+ //
+ // comboBoxFilter
+ //
+ this.comboBoxFilter.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
+ this.comboBoxFilter.FormattingEnabled = true;
+ this.comboBoxFilter.Location = new System.Drawing.Point(302, 5);
+ this.comboBoxFilter.Name = "comboBoxFilter";
+ this.comboBoxFilter.Size = new System.Drawing.Size(121, 21);
+ this.comboBoxFilter.TabIndex = 5;
+ this.comboBoxFilter.SelectedIndexChanged += new System.EventHandler(this.filter_changed);
+ this.comboBoxFilter.DropDownClosed += new System.EventHandler(this.mouse_Leave);
+ this.comboBoxFilter.DropDown += new System.EventHandler(this.mouse_Enter);
+ //
+ // panelControls
+ //
+ this.panelControls.Controls.Add(this.buttonExport);
+ this.panelControls.Controls.Add(this.buttonZero);
+ this.panelControls.Controls.Add(this.comboBoxInterval);
+ this.panelControls.Controls.Add(this.labelFilter);
+ this.panelControls.Controls.Add(this.comboBoxFilter);
+ this.panelControls.Controls.Add(this.labelInterval);
+ this.panelControls.Dock = System.Windows.Forms.DockStyle.Top;
+ this.panelControls.Location = new System.Drawing.Point(0, 110);
+ this.panelControls.Name = "panelControls";
+ this.panelControls.Size = new System.Drawing.Size(505, 32);
+ this.panelControls.TabIndex = 6;
+ //
+ // buttonExport
+ //
+ this.buttonExport.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
+ this.buttonExport.Location = new System.Drawing.Point(187, 4);
+ this.buttonExport.Name = "buttonExport";
+ this.buttonExport.Size = new System.Drawing.Size(75, 23);
+ this.buttonExport.TabIndex = 9;
+ this.buttonExport.Text = "Export";
+ this.buttonExport.UseVisualStyleBackColor = true;
+ this.buttonExport.Click += new System.EventHandler(this.buttonExport_Click);
+ //
+ // buttonZero
+ //
+ this.buttonZero.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
+ this.buttonZero.Location = new System.Drawing.Point(427, 4);
+ this.buttonZero.Name = "buttonZero";
+ this.buttonZero.Size = new System.Drawing.Size(75, 23);
+ this.buttonZero.TabIndex = 8;
+ this.buttonZero.Text = "Clear All";
+ this.buttonZero.UseVisualStyleBackColor = true;
+ this.buttonZero.Click += new System.EventHandler(this.buttonZero_Click);
+ //
+ // comboBoxInterval
+ //
+ this.comboBoxInterval.FormattingEnabled = true;
+ this.comboBoxInterval.Items.AddRange(new object[] {
+ "1",
+ "2",
+ "5",
+ "10",
+ "30",
+ "60"});
+ this.comboBoxInterval.Location = new System.Drawing.Point(84, 6);
+ this.comboBoxInterval.Name = "comboBoxInterval";
+ this.comboBoxInterval.Size = new System.Drawing.Size(55, 21);
+ this.comboBoxInterval.TabIndex = 7;
+ this.comboBoxInterval.Text = "1";
+ this.comboBoxInterval.SelectedIndexChanged += new System.EventHandler(this.interval_changed);
+ this.comboBoxInterval.DropDownClosed += new System.EventHandler(this.mouse_Leave);
+ this.comboBoxInterval.DropDown += new System.EventHandler(this.mouse_Enter);
+ //
+ // labelFilter
+ //
+ this.labelFilter.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
+ this.labelFilter.AutoSize = true;
+ this.labelFilter.Location = new System.Drawing.Point(268, 9);
+ this.labelFilter.Name = "labelFilter";
+ this.labelFilter.Size = new System.Drawing.Size(29, 13);
+ this.labelFilter.TabIndex = 6;
+ this.labelFilter.Text = "Filter";
+ //
+ // saveFileDialogExport
+ //
+ this.saveFileDialogExport.FileName = "results.txt";
+ //
+ // menuStrip1
+ //
+ this.menuStrip1.Items.AddRange(new System.Windows.Forms.ToolStripItem[] {
+ this.fileToolStripMenuItem});
+ this.menuStrip1.Location = new System.Drawing.Point(0, 0);
+ this.menuStrip1.Name = "menuStrip1";
+ this.menuStrip1.Size = new System.Drawing.Size(505, 24);
+ this.menuStrip1.TabIndex = 7;
+ this.menuStrip1.Text = "menuStrip1";
+ //
+ // fileToolStripMenuItem
+ //
+ this.fileToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
+ this.openToolStripMenuItem,
+ this.closeToolStripMenuItem,
+ this.quitToolStripMenuItem});
+ this.fileToolStripMenuItem.Name = "fileToolStripMenuItem";
+ this.fileToolStripMenuItem.Size = new System.Drawing.Size(35, 20);
+ this.fileToolStripMenuItem.Text = "File";
+ //
+ // openToolStripMenuItem
+ //
+ this.openToolStripMenuItem.Name = "openToolStripMenuItem";
+ this.openToolStripMenuItem.Size = new System.Drawing.Size(111, 22);
+ this.openToolStripMenuItem.Text = "Open";
+ this.openToolStripMenuItem.Click += new System.EventHandler(this.openToolStripMenuItem_Click);
+ //
+ // closeToolStripMenuItem
+ //
+ this.closeToolStripMenuItem.Name = "closeToolStripMenuItem";
+ this.closeToolStripMenuItem.Size = new System.Drawing.Size(111, 22);
+ this.closeToolStripMenuItem.Text = "Close";
+ this.closeToolStripMenuItem.Click += new System.EventHandler(this.closeToolStripMenuItem_Click);
+ //
+ // quitToolStripMenuItem
+ //
+ this.quitToolStripMenuItem.Name = "quitToolStripMenuItem";
+ this.quitToolStripMenuItem.Size = new System.Drawing.Size(111, 22);
+ this.quitToolStripMenuItem.Text = "Quit";
+ this.quitToolStripMenuItem.Click += new System.EventHandler(this.quitToolStripMenuItem_Click);
+ //
+ // listViewRates
+ //
+ this.listViewRates.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] {
+ this.columnHeaderRateName,
+ this.columnHeaderRateCount,
+ this.columnHeaderRateTotaltime,
+ this.columnHeaderRateAvgTime});
+ this.listViewRates.Dock = System.Windows.Forms.DockStyle.Fill;
+ this.listViewRates.FullRowSelect = true;
+ this.listViewRates.Location = new System.Drawing.Point(0, 0);
+ this.listViewRates.Name = "listViewRates";
+ this.listViewRates.Size = new System.Drawing.Size(505, 270);
+ this.listViewRates.Sorting = System.Windows.Forms.SortOrder.Descending;
+ this.listViewRates.TabIndex = 8;
+ this.listViewRates.UseCompatibleStateImageBehavior = false;
+ this.listViewRates.View = System.Windows.Forms.View.Details;
+ //
+ // columnHeaderRateName
+ //
+ this.columnHeaderRateName.Text = "Rate Name";
+ this.columnHeaderRateName.Width = 205;
+ //
+ // columnHeaderRateCount
+ //
+ this.columnHeaderRateCount.Text = "Count";
+ //
+ // columnHeaderRateTotaltime
+ //
+ this.columnHeaderRateTotaltime.Text = "Total Time (ms)";
+ this.columnHeaderRateTotaltime.Width = 100;
+ //
+ // columnHeaderRateAvgTime
+ //
+ this.columnHeaderRateAvgTime.Text = "Average Time (ms)";
+ this.columnHeaderRateAvgTime.Width = 110;
+ //
+ // splitContainer1
+ //
+ this.splitContainer1.Dock = System.Windows.Forms.DockStyle.Fill;
+ this.splitContainer1.Location = new System.Drawing.Point(0, 142);
+ this.splitContainer1.Name = "splitContainer1";
+ this.splitContainer1.Orientation = System.Windows.Forms.Orientation.Horizontal;
+ //
+ // splitContainer1.Panel1
+ //
+ this.splitContainer1.Panel1.Controls.Add(this.listViewCounters);
+ //
+ // splitContainer1.Panel2
+ //
+ this.splitContainer1.Panel2.Controls.Add(this.listViewRates);
+ this.splitContainer1.Size = new System.Drawing.Size(505, 495);
+ this.splitContainer1.SplitterDistance = 221;
+ this.splitContainer1.TabIndex = 9;
+ //
+ // StatsViewer
+ //
+ this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
+ this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
+ this.ClientSize = new System.Drawing.Size(505, 637);
+ this.Controls.Add(this.splitContainer1);
+ this.Controls.Add(this.panelControls);
+ this.Controls.Add(this.panelHeader);
+ this.Controls.Add(this.menuStrip1);
+ this.DoubleBuffered = true;
+ this.Name = "StatsViewer";
+ this.Text = "Chrome Varz";
+ ((System.ComponentModel.ISupportInitialize)(this.pictureBoxTitle)).EndInit();
+ this.panelHeader.ResumeLayout(false);
+ this.panelHeader.PerformLayout();
+ this.panelControls.ResumeLayout(false);
+ this.panelControls.PerformLayout();
+ this.menuStrip1.ResumeLayout(false);
+ this.menuStrip1.PerformLayout();
+ this.splitContainer1.Panel1.ResumeLayout(false);
+ this.splitContainer1.Panel2.ResumeLayout(false);
+ this.splitContainer1.ResumeLayout(false);
+ this.ResumeLayout(false);
+ this.PerformLayout();
+
+ }
+
+ #endregion
+
+ private System.Windows.Forms.ListView listViewCounters;
+ private System.Windows.Forms.ColumnHeader columnHeaderName;
+ private System.Windows.Forms.ColumnHeader columnHeaderValue;
+ private System.Windows.Forms.ColumnHeader columnHeaderDelta;
+ private System.Windows.Forms.PictureBox pictureBoxTitle;
+ private System.Windows.Forms.Panel panelHeader;
+ private System.Windows.Forms.Label label1;
+ private System.Windows.Forms.Label labelInterval;
+ private System.Windows.Forms.ComboBox comboBoxFilter;
+ private System.Windows.Forms.Panel panelControls;
+ private System.Windows.Forms.Label labelFilter;
+ private System.Windows.Forms.ComboBox comboBoxInterval;
+ private System.Windows.Forms.Label labelKills;
+ private System.Windows.Forms.Button buttonZero;
+ private System.Windows.Forms.Button buttonExport;
+ private System.Windows.Forms.SaveFileDialog saveFileDialogExport;
+ private System.Windows.Forms.MenuStrip menuStrip1;
+ private System.Windows.Forms.ToolStripMenuItem fileToolStripMenuItem;
+ private System.Windows.Forms.ToolStripMenuItem openToolStripMenuItem;
+ private System.Windows.Forms.ToolStripMenuItem closeToolStripMenuItem;
+ private System.Windows.Forms.ToolStripMenuItem quitToolStripMenuItem;
+ private System.Windows.Forms.ListView listViewRates;
+ private System.Windows.Forms.ColumnHeader columnHeaderRateName;
+ private System.Windows.Forms.ColumnHeader columnHeaderRateCount;
+ private System.Windows.Forms.ColumnHeader columnHeaderRateTotaltime;
+ private System.Windows.Forms.ColumnHeader columnHeaderRateAvgTime;
+ private System.Windows.Forms.SplitContainer splitContainer1;
+ }
+}
diff --git a/tools/stats_viewer/stats_viewer.cs b/tools/stats_viewer/stats_viewer.cs new file mode 100644 index 0000000..27fbda1 --- /dev/null +++ b/tools/stats_viewer/stats_viewer.cs @@ -0,0 +1,535 @@ +// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Data;
+using System.Diagnostics;
+using System.Drawing;
+using System.Text;
+using System.Windows.Forms;
+using System.IO;
+
+namespace StatsViewer {
+ public partial class StatsViewer : Form {
+ /// <summary>
+ /// Create a StatsViewer.
+ /// </summary>
+ public StatsViewer() {
+ InitializeComponent();
+ }
+
+ #region Protected Methods
+ /// <summary>
+ /// Callback when the form loads.
+ /// </summary>
+ /// <param name="e"></param>
+ protected override void OnLoad(EventArgs e) {
+ base.OnLoad(e);
+
+ timer_ = new Timer();
+ timer_.Interval = kPollInterval;
+ timer_.Tick += new EventHandler(PollTimerTicked);
+ timer_.Start();
+ }
+ #endregion
+
+ #region Private Methods
+ /// <summary>
+ /// Attempt to open the stats file.
+ /// Return true on success, false otherwise.
+ /// </summary>
+ private bool OpenStatsFile() {
+ StatsTable table = new StatsTable();
+ if (table.Open(kStatsTableName)) {
+ stats_table_ = table;
+ return true;
+ }
+ return false;
+ }
+
+ /// <summary>
+ /// Close the open stats file.
+ /// </summary>
+ private void CloseStatsFile() {
+ if (this.stats_table_ != null)
+ {
+ this.stats_table_.Close();
+ this.stats_table_ = null;
+ this.listViewCounters.Items.Clear();
+ }
+ }
+
+ /// <summary>
+ /// Updates the process list in the UI.
+ /// </summary>
+ private void UpdateProcessList() {
+ int current_pids = comboBoxFilter.Items.Count;
+ int table_pids = stats_table_.Processes.Count;
+ if (current_pids != table_pids + 1) // Add one because of the "all" entry.
+ {
+ int selected_index = this.comboBoxFilter.SelectedIndex;
+ this.comboBoxFilter.Items.Clear();
+ this.comboBoxFilter.Items.Add(kStringAllProcesses);
+ foreach (int pid in stats_table_.Processes)
+ this.comboBoxFilter.Items.Add(kStringProcess + pid.ToString());
+ this.comboBoxFilter.SelectedIndex = selected_index;
+ }
+ }
+
+ /// <summary>
+ /// Updates the UI for a counter.
+ /// </summary>
+ /// <param name="counter"></param>
+ private void UpdateCounter(IStatsCounter counter) {
+ ListView view;
+
+ // Figure out which list this counter goes into.
+ if (counter is StatsCounterRate)
+ view = listViewRates;
+ else if (counter is StatsCounter || counter is StatsTimer)
+ view = listViewCounters;
+ else
+ return; // Counter type not supported yet.
+
+ // See if the counter is already in the list.
+ ListViewItem item = view.Items[counter.name];
+ if (item != null)
+ {
+ // Update an existing counter.
+ Debug.Assert(item is StatsCounterListViewItem);
+ StatsCounterListViewItem counter_item = item as StatsCounterListViewItem;
+ counter_item.Update(counter, filter_pid_);
+ }
+ else
+ {
+ // Create a new counter
+ StatsCounterListViewItem new_item = null;
+ if (counter is StatsCounterRate)
+ new_item = new RateListViewItem(counter, filter_pid_);
+ else if (counter is StatsCounter || counter is StatsTimer)
+ new_item = new CounterListViewItem(counter, filter_pid_);
+ Debug.Assert(new_item != null);
+ view.Items.Add(new_item);
+ }
+ }
+
+ /// <summary>
+ /// Sample the data and update the UI
+ /// </summary>
+ private void SampleData() {
+ // If the table isn't open, try to open it again.
+ if (stats_table_ == null)
+ if (!OpenStatsFile())
+ return;
+
+ if (stats_counters_ == null)
+ stats_counters_ = stats_table_.Counters();
+
+ if (pause_updates_)
+ return;
+
+ stats_counters_.Update();
+
+ UpdateProcessList();
+
+ foreach (IStatsCounter counter in stats_counters_)
+ UpdateCounter(counter);
+ }
+
+ /// <summary>
+ /// Set the background color based on the value
+ /// </summary>
+ /// <param name="item"></param>
+ /// <param name="value"></param>
+ private void ColorItem(ListViewItem item, int value)
+ {
+ if (value < 0)
+ item.ForeColor = Color.Red;
+ else if (value > 0)
+ item.ForeColor = Color.DarkGreen;
+ else
+ item.ForeColor = Color.Black;
+ }
+
+ /// <summary>
+ /// Called when the timer fires.
+ /// </summary>
+ /// <param name="sender"></param>
+ /// <param name="e"></param>
+ void PollTimerTicked(object sender, EventArgs e) {
+ SampleData();
+ }
+
+ /// <summary>
+ /// Called when the interval is changed by the user.
+ /// </summary>
+ /// <param name="sender"></param>
+ /// <param name="e"></param>
+ private void interval_changed(object sender, EventArgs e) {
+ int interval = 1;
+ if (int.TryParse(comboBoxInterval.Text, out interval)) {
+ if (timer_ != null) {
+ timer_.Stop();
+ timer_.Interval = interval * 1000;
+ timer_.Start();
+ }
+ } else {
+ comboBoxInterval.Text = timer_.Interval.ToString();
+ }
+ }
+
+ /// <summary>
+ /// Called when the user changes the filter
+ /// </summary>
+ /// <param name="sender"></param>
+ /// <param name="e"></param>
+ private void filter_changed(object sender, EventArgs e) {
+ // While in this event handler, don't allow recursive events!
+ this.comboBoxFilter.SelectedIndexChanged -= new System.EventHandler(this.filter_changed);
+ if (this.comboBoxFilter.Text == kStringAllProcesses)
+ filter_pid_ = 0;
+ else
+ int.TryParse(comboBoxFilter.Text.Substring(kStringProcess.Length), out filter_pid_);
+ SampleData();
+ this.comboBoxFilter.SelectedIndexChanged += new System.EventHandler(this.filter_changed);
+ }
+
+ /// <summary>
+ /// Callback when the mouse enters a control
+ /// </summary>
+ /// <param name="sender"></param>
+ /// <param name="e"></param>
+ private void mouse_Enter(object sender, EventArgs e) {
+ // When the dropdown is expanded, we pause
+ // updates, as it messes with the UI.
+ pause_updates_ = true;
+ }
+
+ /// <summary>
+ /// Callback when the mouse leaves a control
+ /// </summary>
+ /// <param name="sender"></param>
+ /// <param name="e"></param>
+ private void mouse_Leave(object sender, EventArgs e) {
+ pause_updates_ = false;
+ }
+
+ /// <summary>
+ /// Called when the user clicks the zero-stats button.
+ /// </summary>
+ /// <param name="sender"></param>
+ /// <param name="e"></param>
+ private void buttonZero_Click(object sender, EventArgs e) {
+ this.stats_table_.Zero();
+ SampleData();
+ }
+
+ /// <summary>
+ /// Called when the user clicks a column heading.
+ /// </summary>
+ /// <param name="sender"></param>
+ /// <param name="e"></param>
+ private void column_Click(object sender, ColumnClickEventArgs e) {
+ if (e.Column != sort_column_) {
+ sort_column_ = e.Column;
+ this.listViewCounters.Sorting = SortOrder.Ascending;
+ } else {
+ if (this.listViewCounters.Sorting == SortOrder.Ascending)
+ this.listViewCounters.Sorting = SortOrder.Descending;
+ else
+ this.listViewCounters.Sorting = SortOrder.Ascending;
+ }
+
+ this.listViewCounters.ListViewItemSorter =
+ new ListViewItemComparer(e.Column, this.listViewCounters.Sorting);
+ this.listViewCounters.Sort();
+ }
+
+ /// <summary>
+ /// Called when the user clicks the button "Export".
+ /// </summary>
+ /// <param name="sender"></param>
+ /// <param name="e"></param>
+ private void buttonExport_Click(object sender, EventArgs e) {
+ //Have to pick a textfile to export to.
+ //Saves what is shown in listViewStats in the format: function value
+ //(with a tab in between), so that it is easy to copy paste into a spreadsheet.
+ //(Does not save the delta values.)
+ TextWriter tw = null;
+ try {
+ saveFileDialogExport.CheckFileExists = false;
+ saveFileDialogExport.ShowDialog();
+ tw = new StreamWriter(saveFileDialogExport.FileName);
+
+ for (int i = 0; i < listViewCounters.Items.Count; i++) {
+ tw.Write(listViewCounters.Items[i].SubItems[0].Text + "\t");
+ tw.WriteLine(listViewCounters.Items[i].SubItems[1].Text);
+ }
+ }
+ catch (IOException ex) {
+ MessageBox.Show(string.Format("There was an error while saving your results file. The results might not have been saved correctly.: {0}", ex.Message));
+ }
+ finally{
+ if (tw != null) tw.Close();
+ }
+ }
+
+ #endregion
+
+ class ListViewItemComparer : IComparer {
+ public ListViewItemComparer() {
+ this.col_ = 0;
+ this.order_ = SortOrder.Ascending;
+ }
+
+ public ListViewItemComparer(int column, SortOrder order) {
+ this.col_ = column;
+ this.order_ = order;
+ }
+
+ public int Compare(object x, object y) {
+ int return_value = -1;
+
+ object x_tag = ((ListViewItem)x).SubItems[col_].Tag;
+ object y_tag = ((ListViewItem)y).SubItems[col_].Tag;
+
+ if (Comparable(x_tag, y_tag))
+ return_value = ((IComparable)x_tag).CompareTo(y_tag);
+ else
+ return_value = String.Compare(((ListViewItem)x).SubItems[col_].Text,
+ ((ListViewItem)y).SubItems[col_].Text);
+
+ if (order_ == SortOrder.Descending)
+ return_value *= -1;
+
+ return return_value;
+ }
+
+ #region Private Methods
+ private bool Comparable(object x, object y) {
+ if (x == null || y == null)
+ return false;
+
+ return x is IComparable && y is IComparable;
+ }
+ #endregion
+
+ #region Private Members
+ private int col_;
+ private SortOrder order_;
+ #endregion
+ }
+
+ #region Private Members
+ private const string kStringAllProcesses = "All Processes";
+ private const string kStringProcess = "Process ";
+ private const int kPollInterval = 1000; // 1 second
+ private const string kStatsTableName = "ChromeStats";
+ private StatsTable stats_table_;
+ private StatsTableCounters stats_counters_;
+ private Timer timer_;
+ private int filter_pid_;
+ private bool pause_updates_;
+ private int sort_column_ = -1;
+ #endregion
+
+ #region Private Event Callbacks
+ private void openToolStripMenuItem_Click(object sender, EventArgs e)
+ {
+ OpenDialog dialog = new OpenDialog();
+ dialog.ShowDialog();
+
+ CloseStatsFile();
+
+ StatsTable table = new StatsTable();
+ bool rv = table.Open(dialog.FileName);
+ if (!rv)
+ {
+ MessageBox.Show("Could not open statsfile: " + dialog.FileName);
+ }
+ else
+ {
+ stats_table_ = table;
+ }
+ }
+
+ private void closeToolStripMenuItem_Click(object sender, EventArgs e)
+ {
+ CloseStatsFile();
+ }
+
+ private void quitToolStripMenuItem_Click(object sender, EventArgs e)
+ {
+ Application.Exit();
+ }
+ #endregion
+ }
+
+ /// <summary>
+ /// Base class for counter list view items.
+ /// </summary>
+ internal class StatsCounterListViewItem : ListViewItem
+ {
+ /// <summary>
+ /// Create the ListViewItem
+ /// </summary>
+ /// <param name="text"></param>
+ public StatsCounterListViewItem(string text) : base(text) { }
+
+ /// <summary>
+ /// Update the ListViewItem given a new counter value.
+ /// </summary>
+ /// <param name="counter"></param>
+ /// <param name="filter_pid"></param>
+ public virtual void Update(IStatsCounter counter, int filter_pid) { }
+
+ /// <summary>
+ /// Set the background color based on the value
+ /// </summary>
+ /// <param name="value"></param>
+ protected void ColorItem(int value)
+ {
+ if (value < 0)
+ ForeColor = Color.Red;
+ else if (value > 0)
+ ForeColor = Color.DarkGreen;
+ else
+ ForeColor = Color.Black;
+ }
+
+ /// <summary>
+ /// Create a new subitem with a zeroed Tag.
+ /// </summary>
+ /// <returns></returns>
+ protected ListViewSubItem NewSubItem()
+ {
+ ListViewSubItem item = new ListViewSubItem();
+ item.Tag = -1; // Arbitrarily initialize to -1.
+ return item;
+ }
+
+ /// <summary>
+ /// Set the value for a subitem.
+ /// </summary>
+ /// <param name="item"></param>
+ /// <param name="val"></param>
+ /// <returns>True if the value changed, false otherwise</returns>
+ protected bool SetSubItem(ListViewSubItem item, int val)
+ {
+ // The reason for doing this extra compare is because
+ // we introduce flicker if we unnecessarily update the
+ // subitems. The UI is much less likely to cause you
+ // a seizure when we do this.
+ if (val != (int)item.Tag)
+ {
+ item.Text = val.ToString();
+ item.Tag = val;
+ return true;
+ }
+ return false;
+ }
+ }
+
+ /// <summary>
+ /// A listview item which contains a rate.
+ /// </summary>
+ internal class RateListViewItem : StatsCounterListViewItem
+ {
+ public RateListViewItem(IStatsCounter ctr, int filter_pid) :
+ base(ctr.name)
+ {
+ StatsCounterRate rate = ctr as StatsCounterRate;
+ Name = rate.name;
+ SubItems.Add(NewSubItem());
+ SubItems.Add(NewSubItem());
+ SubItems.Add(NewSubItem());
+ Update(ctr, filter_pid);
+ }
+
+ public override void Update(IStatsCounter counter, int filter_pid)
+ {
+ Debug.Assert(counter is StatsCounterRate);
+
+ StatsCounterRate new_rate = counter as StatsCounterRate;
+ int new_count = new_rate.GetCount(filter_pid);
+ int new_time = new_rate.GetTime(filter_pid);
+ int old_avg = Tag != null ? (int)Tag : 0;
+ int new_avg = new_count > 0 ? (new_time / new_count) : 0;
+ int delta = new_avg - old_avg;
+
+ SetSubItem(SubItems[column_count_index], new_count);
+ SetSubItem(SubItems[column_time_index], new_time);
+ if (SetSubItem(SubItems[column_avg_index], new_avg))
+ ColorItem(delta);
+ Tag = new_avg;
+ }
+
+ private const int column_count_index = 1;
+ private const int column_time_index = 2;
+ private const int column_avg_index = 3;
+ }
+
+ /// <summary>
+ /// A listview item which contains a counter.
+ /// </summary>
+ internal class CounterListViewItem : StatsCounterListViewItem
+ {
+ public CounterListViewItem(IStatsCounter ctr, int filter_pid) :
+ base(ctr.name)
+ {
+ Name = ctr.name;
+ SubItems.Add(NewSubItem());
+ SubItems.Add(NewSubItem());
+ Update(ctr, filter_pid);
+ }
+
+ public override void Update(IStatsCounter counter, int filter_pid) {
+ Debug.Assert(counter is StatsCounter || counter is StatsTimer);
+
+ int new_value = 0;
+ if (counter is StatsCounter)
+ new_value = ((StatsCounter)counter).GetValue(filter_pid);
+ else if (counter is StatsTimer)
+ new_value = ((StatsTimer)counter).GetValue(filter_pid);
+
+ int old_value = Tag != null ? (int)Tag : 0;
+ int delta = new_value - old_value;
+ SetSubItem(SubItems[column_value_index], new_value);
+ if (SetSubItem(SubItems[column_delta_index], delta))
+ ColorItem(delta);
+ Tag = new_value;
+ }
+
+ private const int column_value_index = 1;
+ private const int column_delta_index = 2;
+ }
+}
diff --git a/tools/stats_viewer/stats_viewer.csproj b/tools/stats_viewer/stats_viewer.csproj new file mode 100644 index 0000000..2b6fa6c --- /dev/null +++ b/tools/stats_viewer/stats_viewer.csproj @@ -0,0 +1,107 @@ +<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <PropertyGroup>
+ <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+ <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+ <ProductVersion>8.0.50727</ProductVersion>
+ <SchemaVersion>2.0</SchemaVersion>
+ <ProjectGuid>{41735CD9-3E35-47F7-9FD1-4A9950B6B131}</ProjectGuid>
+ <OutputType>WinExe</OutputType>
+ <AppDesignerFolder>Properties</AppDesignerFolder>
+ <RootNamespace>StatsViewer</RootNamespace>
+ <AssemblyName>stats_viewer</AssemblyName>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+ <DebugSymbols>true</DebugSymbols>
+ <DebugType>full</DebugType>
+ <Optimize>false</Optimize>
+ <OutputPath>$(SolutionDir)$(Configuration)</OutputPath>
+ <DefineConstants>DEBUG;TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+ <DebugType>pdbonly</DebugType>
+ <Optimize>true</Optimize>
+ <OutputPath>$(SolutionDir)$(Configuration)</OutputPath>
+ <DefineConstants>TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ </PropertyGroup>
+ <ItemGroup>
+ <Reference Include="System" />
+ <Reference Include="System.Data" />
+ <Reference Include="System.Deployment" />
+ <Reference Include="System.Drawing" />
+ <Reference Include="System.Windows.Forms" />
+ <Reference Include="System.Xml" />
+ </ItemGroup>
+ <ItemGroup>
+ <Compile Include="OpenDialog.cs">
+ <SubType>Form</SubType>
+ </Compile>
+ <Compile Include="OpenDialog.Designer.cs">
+ <DependentUpon>OpenDialog.cs</DependentUpon>
+ </Compile>
+ <Compile Include="Resources.Designer.cs">
+ <AutoGen>True</AutoGen>
+ <DesignTime>True</DesignTime>
+ <DependentUpon>Resources.resx</DependentUpon>
+ </Compile>
+ <Compile Include="stats_viewer.cs">
+ <SubType>Form</SubType>
+ </Compile>
+ <Compile Include="stats_viewer.Designer.cs">
+ <DependentUpon>stats_viewer.cs</DependentUpon>
+ </Compile>
+ <Compile Include="Program.cs" />
+ <Compile Include="Properties\AssemblyInfo.cs" />
+ <EmbeddedResource Include="OpenDialog.resx">
+ <SubType>Designer</SubType>
+ <DependentUpon>OpenDialog.cs</DependentUpon>
+ </EmbeddedResource>
+ <EmbeddedResource Include="Resources.resx">
+ <SubType>Designer</SubType>
+ <Generator>ResXFileCodeGenerator</Generator>
+ <LastGenOutput>Resources.Designer.cs</LastGenOutput>
+ </EmbeddedResource>
+ <EmbeddedResource Include="stats_viewer.resx">
+ <SubType>Designer</SubType>
+ <DependentUpon>stats_viewer.cs</DependentUpon>
+ </EmbeddedResource>
+ <EmbeddedResource Include="Properties\Resources.resx">
+ <Generator>ResXFileCodeGenerator</Generator>
+ <LastGenOutput>Resources.Designer.cs</LastGenOutput>
+ <SubType>Designer</SubType>
+ </EmbeddedResource>
+ <Compile Include="Properties\Resources.Designer.cs">
+ <AutoGen>True</AutoGen>
+ <DependentUpon>Resources.resx</DependentUpon>
+ </Compile>
+ <None Include="Properties\Settings.settings">
+ <Generator>SettingsSingleFileGenerator</Generator>
+ <LastGenOutput>Settings.Designer.cs</LastGenOutput>
+ </None>
+ <Compile Include="Properties\Settings.Designer.cs">
+ <AutoGen>True</AutoGen>
+ <DependentUpon>Settings.settings</DependentUpon>
+ <DesignTimeSharedInput>True</DesignTimeSharedInput>
+ </Compile>
+ <Compile Include="stats_table.cs" />
+ <Compile Include="win32.cs" />
+ </ItemGroup>
+ <ItemGroup>
+ <None Include="Resources\kitten.png" />
+ </ItemGroup>
+ <ItemGroup>
+ <None Include="Resources\kittenbackground.png" />
+ </ItemGroup>
+ <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+ <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
+ Other similar extension points exist, see Microsoft.Common.targets.
+ <Target Name="BeforeBuild">
+ </Target>
+ <Target Name="AfterBuild">
+ </Target>
+ -->
+</Project>
\ No newline at end of file diff --git a/tools/stats_viewer/stats_viewer.resx b/tools/stats_viewer/stats_viewer.resx new file mode 100644 index 0000000..117f389 --- /dev/null +++ b/tools/stats_viewer/stats_viewer.resx @@ -0,0 +1,1129 @@ +<?xml version="1.0" encoding="utf-8"?>
+<root>
+ <!--
+ Microsoft ResX Schema
+
+ Version 2.0
+
+ The primary goals of this format is to allow a simple XML format
+ that is mostly human readable. The generation and parsing of the
+ various data types are done through the TypeConverter classes
+ associated with the data types.
+
+ Example:
+
+ ... ado.net/XML headers & schema ...
+ <resheader name="resmimetype">text/microsoft-resx</resheader>
+ <resheader name="version">2.0</resheader>
+ <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
+ <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
+ <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
+ <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
+ <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
+ <value>[base64 mime encoded serialized .NET Framework object]</value>
+ </data>
+ <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+ <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
+ <comment>This is a comment</comment>
+ </data>
+
+ There are any number of "resheader" rows that contain simple
+ name/value pairs.
+
+ Each data row contains a name, and value. The row also contains a
+ type or mimetype. Type corresponds to a .NET class that support
+ text/value conversion through the TypeConverter architecture.
+ Classes that don't support this are serialized and stored with the
+ mimetype set.
+
+ The mimetype is used for serialized objects, and tells the
+ ResXResourceReader how to depersist the object. This is currently not
+ extensible. For a given mimetype the value must be set accordingly:
+
+ Note - application/x-microsoft.net.object.binary.base64 is the format
+ that the ResXResourceWriter will generate, however the reader can
+ read any of the formats listed below.
+
+ mimetype: application/x-microsoft.net.object.binary.base64
+ value : The object must be serialized with
+ : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
+ : and then encoded with base64 encoding.
+
+ mimetype: application/x-microsoft.net.object.soap.base64
+ value : The object must be serialized with
+ : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
+ : and then encoded with base64 encoding.
+
+ mimetype: application/x-microsoft.net.object.bytearray.base64
+ value : The object must be serialized into a byte array
+ : using a System.ComponentModel.TypeConverter
+ : and then encoded with base64 encoding.
+ -->
+ <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
+ <xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
+ <xsd:element name="root" msdata:IsDataSet="true">
+ <xsd:complexType>
+ <xsd:choice maxOccurs="unbounded">
+ <xsd:element name="metadata">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" />
+ </xsd:sequence>
+ <xsd:attribute name="name" use="required" type="xsd:string" />
+ <xsd:attribute name="type" type="xsd:string" />
+ <xsd:attribute name="mimetype" type="xsd:string" />
+ <xsd:attribute ref="xml:space" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="assembly">
+ <xsd:complexType>
+ <xsd:attribute name="alias" type="xsd:string" />
+ <xsd:attribute name="name" type="xsd:string" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="data">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+ <xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
+ </xsd:sequence>
+ <xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
+ <xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
+ <xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
+ <xsd:attribute ref="xml:space" />
+ </xsd:complexType>
+ </xsd:element>
+ <xsd:element name="resheader">
+ <xsd:complexType>
+ <xsd:sequence>
+ <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+ </xsd:sequence>
+ <xsd:attribute name="name" type="xsd:string" use="required" />
+ </xsd:complexType>
+ </xsd:element>
+ </xsd:choice>
+ </xsd:complexType>
+ </xsd:element>
+ </xsd:schema>
+ <resheader name="resmimetype">
+ <value>text/microsoft-resx</value>
+ </resheader>
+ <resheader name="version">
+ <value>2.0</value>
+ </resheader>
+ <resheader name="reader">
+ <value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+ </resheader>
+ <resheader name="writer">
+ <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+ </resheader>
+ <assembly alias="System.Drawing" name="System.Drawing, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a" />
+ <data name="pictureBoxTitle.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+ <value>
+ iVBORw0KGgoAAAANSUhEUgAAAPgAAABWCAIAAABtpczCAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8
+ YQUAAAAgY0hSTQAAeiYAAICEAAD6AAAAgOgAAHUwAADqYAAAOpgAABdwnLpRPAAAAAlwSFlzAAAOxAAA
+ DsQBlSsOGwAA4PdJREFUeF5k/QVAXNf2/4GmdnutvZV4gjPu7oN70KBBEggJcScBgru7u7u7u2sggQTi
+ JLgmaa/1tnnrDGl///de++3umTMDDMPnfGfttdde81nzg+B3vz3/+edf9n3++W+f7fvss88+//zzfb99
+ /O3jr3C07+PHj8g5yR2f/fb5Z59JTnzc93Ef/PPZ55J/4d59+3777TfkzGdwz8fPP/8CTsMDkX8+R87v
+ g9OS7yF5DIzwGORbwQ+AB38BP0jyGPhayXeAx8AT2PspyH/wUyRPA77H3j/IwyR3Sb7hp/8kj5b8VHi4
+ 5CnDEdyUPAz53faeMzJ8hGe79+sgPwueC/JbID9xHzxbyZcgD5c8FeSLkV9Y8v0k5yRPDHk94IuQZ/L7
+ k/r03H5/Wnsvx96vtfddfn8FkG+GvFx7J+CZSJ6G5Bf6DXkCezf++CLJzb1fcu8l3btr74X69KrBr7f3
+ 6/8G3wN+H+R3/PRg+LWQ7yn5Q8Idn++9QvDCS37ZvdcE+Ta//iZ5QSS/oOQvgbzMkh8gOUBuw13Ic4Xv
+ uPc67/0PeSmQZ4P8ADiBfIu9v7Hkzr0fIXll954sPDP4ayLfU/JtADXJS4N829/gGcIvAH8deMrwf+Rb
+ 7n255IF7Z/aQkPxR4FXc+xGffl/kYb/uIbv35fCH3vfZF5/99uvHfakdJ2MGyQFteL9Ogm8nyb+bEtBL
+ Deym+HcTA3pIAd1k5EwPJbCHGtxHCRmkhgyQQwaIIQMkRIPk0CFK2DA1bIgiOaCEjyCKGKNHTjAixkH0
+ yEl61BSIGjFNi5qhg2IegWhxcyBqwiw1ZZ6ZscBOf8pJf8pKf8rMeM7KeM5Mf0FPe0ZLe05LfUZNXqCk
+ PqOlv2CkP2ekP6NlPKNmvKBmvqBmvaJlv6LnLIIYuYv03EVaziI1RzLmvqHkviXnviXlL5Hyl4mgvE83
+ KfnL5PwVcv4yKX+FULJOLt4gF6+TSzZoJZuM4i0acnODVLJJKtkil21RyjYpZRtUicifhJyhlK6Ty7eo
+ lds0UPUOtWaHujfW7tD+UN07Wu07hkT0PdW9p9f/xATBQcMOo36b0bjDaH7Han7PbHxHb3xPh7HhHa3h
+ HbXxPa3hPbXuHaVuV6J3lPr31IYPyMkGyV3I+IGGnJScb/qZ3vgTPIZY/wFf/wFb/wHXgIhQjwhuous/
+ YGrfYepA7/HIw96R6t6R63ZItdvEmm1i3TtS7S6hegdX+45Q+55QvYuv2sFVvsNX7hAqdwmVO8SKbVLF
+ NgXRFrVskwq/fskqsXiVhGiFXLwKrwmtZI0KByUrlOIVSukqtWSFWrRMLl6BA3rJMr1wiZq3TM1foecv
+ M/KXmHlvmXlv2Dmv2Vkv4C/OyHjBTH/KSF2gpy0wUhYYSXP0xFl6/ENK3Aw5bpoc+4AU84AYOYGPGMdG
+ TeAix0FYuBk+hgsZwoQM40KHCcFDhOBBfFA/IbCXENiHC+jDBvRi/bqxfl24gF6ibydhX3aPTXAL1r0S
+ 5VaDca3DuNZi79fjPOpw9+sw9+sxHg0Ej0aiRyPBswnv3ULwbsX7tOK8WzFeLYjgwKcN69eO92nD+bbj
+ /ToI/p1E/05CQDcpoIcMo38XXC3kwD5iUB8+cIAYNEgKHSKFj5BBEaPksCFC+DA+ZpwcP0GNm6AkTCGK
+ nSLFTBJiQVOE2GlC7ANi9CQ+ZgIXN0WMBU0S4idxCQ/widOIkmbwKY+IEhFSHuGTH+JSHkk0iwWlPsal
+ PcamPcakP4ERhMuYJ6Q/wafP49KeYDIWMNnPcVnPEGU/J2S/IGY9J2S9wGU+hfOY3Jf43JeEvJfE/Jek
+ /BfEvBf4vJcSvcDnvyLkwk04eEnIf4kvfIUreo0vfI2HsfgVvmRPrwkli4TiRbxEcEAofg0ilr4hl70l
+ ly6SS16R9lS+SC1/A6KUL5LL4d5FYvlbUsVbMowlyDfEgeCg9A2h7C2hdBFXuoj/fYQDfBnoDaFiiVD+
+ Fu5Cl75Blb5RLHuLKl9Cly9hyt5iyt6gy94iZ+A8cu8inMGCSt/gSl/DN8cUvcSUvMYWv0IVvFIsWkQX
+ vkYXvlIseKmQD2deo/NfofNeYnJfYHKf40A58Io9x8MrBq9exgI+Yx6fuUDIeorPfkbKfkbOWiBlzRMz
+ nxBgzHhMTHuMz3hCSn9CTp0jpsziUx6TUh9TUh9TU2apSbO0xIeMhGla3BQ1dpIaM0GLHCWHDxMjhknh
+ Q+TgPmJIPymoFx/QjduTfxfWp13Ru03Bt0MRDnzaFbzbFL1aFTya5T2aFT2aFD0a0e6NCLFuwG0dyr0O
+ eFZ0q1J0QcBGu1Zh9uX02oa24z1rMO71kscB3I0470a8ZyPWoxHr1UzwbiEB5V4I6DivFiyQ7d2G8W5D
+ +7Zj/DvhGcDzIAT2EGEEBfUR4ZIK6iUF91IQ9ZFD+smhg/DsEb4jJIoao0SNkSJHiDFjhPgJcuw4MWGK
+ nDRNTZgixU0SEmdICaBpQsJDYvwMPu4BHo4TZ4hJM6Skh6TkhyTAOn2WlDEHryAIXk1C6hw+ZRaBO3UO
+ yAbh058A0IQMeNEX4C8B7GIzFrAZ8wA6PuspIec5CcjOegZAY4H1nBf4nBcEEMCd9wqPUP4CU/CaACp8
+ RSx4RYKx6BUeEQD9+wgHCOKv/sAOOAN0sKWvsABQ2Wtc+Rt8+VtsxRts+Vt8xRsQoRyIBIiXyJVvyBWL
+ VFDlW3r1Eq16CRkr31Ar31CQB7whVL4lgsoXCeWLeBAcVy2RqpfJ1ctEUJVk/ENVS4TKJULVErFyCVvx
+ Fl25hNkbK5exlZInADd/F6b8DRzDGTiPA5W9xoDKF+E5IxdJGXJtoIvfoIpeK5bAmUVMySKu8DW24BW2
+ 4CU2H7ns4eKHF4qQ/RxeKEL2MyIIeT2fEbOekhDoF/BAeTYQ/5SUPk/InCenz5OTHxOS5wipQPwTaupj
+ WsocLekRNWGGnjBNj38AYsRNMqNGqQBJBAQFg8AMKbiPFNJHDgSvBN/sIvh3433a0d7tKJ92lHebvFer
+ PIDu2Yxyb5T3aEJ5NiOsArfuDXi3OpxrDcatGuNaiXatxLhUYO7XEl0r8fty+2zDOvFedRjPBuTR7o04
+ gNu3mejdDJTjfFqJXs14j0bwbzByrGczxrcd69eJCK6zwB5AHBFcgkG9YNvE0CEksAkeIIUOUCQihw3C
+ sydFjVKiEb4RgYXHjpMkfJMSp8gJk8SkaQCdnDBFiJvAA+iJj0iJDwlJswA6Lv4BBmw7aYaQAog/JKbM
+ EJIf4lMf4tJmcRlA8xw+bRabPodD9BhBGcjOWgC+4UVHXmvAOusp4tnAetZTbPYzfM5zct4rSu5LIvCN
+ OPQrQv4rIggx71fg1rh8+KO+hr8uYtIFr/AFL8G5cUUv0EUv0TAWvkAVvgRfxJW9QRAHDiQGKRkXJfQs
+ 4irf4Kvf4muWCDXLWERL+JplXN0ysQZgfUMEVb0hVb2lggDxulVG7Qq9eplWtUSpekupfEsG0CVCWN8T
+ UF61RK5ZIdWsEiUiwFiLiFS7BifhB+FrVgjVKzjguwr4BsqXsNUr+KolXOVbOAln9gQ3kbv2BPdKcEeu
+ h/I3mKoVXNUqvmIZW74Mro8qhysEfq+3WPD7olc45Ap/RYALO/8FIQ95iyPmviAVvKSC8p6TEWsHE4FX
+ Hv4EwPcCUfLmCXCTUh6Tk+dISWDqjyXHs5SkR5SEGQrYeTwCOlDOiB5jRo7QwgaBFiowE9JPCewmBnaT
+ A7pIvu0QLODAXr1a0J4t4OLA9ycjd29E3a8H1sGaEZt2q4cwBO9Wi3OrxrtVEO6V4p2LsbcL0a4VRJdy
+ 4r78AbuIbqJPA9a3Ge/djJO4OMQnYOQYwBoOfJCIBX4S8vMgPgHzDgLP7gHbxgf3AeIgfEg/IWyQiNj2
+ MGBNBsEb0J4kFk6OmaDGTlBiwb8nwLaJ8ZOkxGmwZ0rSDDnxAbg1OQk8exoOgGN4myOkPSalPZa83z3E
+ AtNpc3h4KwSlPsKlPsSmPpLADazP4jLnsJmPsZlPEGXN47Ln8dkL8E6KOE32U0LuM0Ie6Dkh5xkW3nbz
+ XkCwQSl4SUZ8+jWxAFCGkGORWLRIKgL/XgQLlwQhb/DFb/GlICRUwJYgAm9Dlb0GoUHli+CaYNVgkMAN
+ kISrAG7eYqvfSsYlXO0yoW4FX7eCrl1B1S6japYxcLN+lVC7BALiidUS1S6T61aptSuUmmVyzTIJVP2W
+ WP2WUA0O/RZfvYSvAoKXgWMQPBhfu0aoXSPWrQPfxD0B5bXrJDhTBwcr+Opl4BsHrMNYs4pHWF/GIZcB
+ IlzNiuTCQ87gqiWgS94K4A0Ewb0KuSDhJlwe+Oo3+MpFQpXkkoN3p+KX2OKXuJIX+JIXhKKXpMKX1MKX
+ tKKXjOJXrKKXrLyn1KwnpOwnxJwFcuY8CQwI/nZpc3t/RErqLC31ETXlERxQkKDlES3hIT1umh47SYsb
+ Z8SN0aNH6JEjjPAhKvAd0gdRAFBO9u8AL4eR7NuGBMwQSniB8wKTTWjAGuRWh3apRrnWIvG2cxXqXhXa
+ pRLjVkW4X0lwLcPfLcY7F+Ju56Nv5CneKcLcLcLvKxiyjeol+DVifMG/94ITCMTbEP8GAdlw81P83YUP
+ 6AIXxwHZiPpxoQP40AFC2BAR+IZoBBQxQoweB8+mIuY9ivh39DhQToqdBKAhBCclTBERsqdJqchvToY4
+ ZM+wkx9CkE1InSWmzhIgDoGABCIQiEPS5zCIVQPKEs/IeAKGAc5ByF4g5s6Tc+fhxSVkz2OzFzCAct5T
+ XC4Ct8RvnuOB8oIX5MKXYDxwgC98AX8wYjFExq/JJa+JEEMj8S6Eqm/wZcD0G0LJW4mQm7jyJVz5Mhbw
+ rUBiAAzic0hIgKl8g6l+C0JwAar+QAogQ0CUMFS7BP6Na1jFNayj69dQ9WvoulVs/Rq+YY0Iql8FkepW
+ ENWvkhvWKKC947pVcu0yqX55716wbUS1q3DBIKqHEb52jdSwToJxT7UrBDjZACdXCXAhwcNqJV8FAtYl
+ uMPJvasOB08DeSYwrsCliNsDHd4ogHUIruBaqlsh1rxFVPuWXPOWUg1vL29J8MoUvoIgHignFj8nFb2g
+ FL1mlLxhFb9i5z9n5D1nwFjwggljzgIta4EGQXnaHIiS8ZiW8YSRNktPfQis09Me0mFMesiIf8iMfcCM
+ mWDGjjJiRxjRw/SIIXpoPzUIpnPdJEiEBHRS/Tsofu1U/3aabxvFq5mIBM/NIIIkwJaEKLU4l2qwcIJL
+ Le5OJfZOOfZeGe5+BdmtjHi3AHcrFwu6k4+9U4C7mYe6mqW4r3DILroX79eA8mnCeDfvzS//j/IAyMN0
+ EiAhgwTivbjAXmxgLyaoDxPcjw0dxIcPw2ySGDFCiBwFvkkQk0SNEWPgYIIMN4H76AlyzCQpbooUP00G
+ C0fi7BkiePZeqA1Yg3nDJFLCNzF1Dt7siMj73Rzu92gbmwFWPf9JWQtYCAFznoE9IzTnv6DkQYD4FJ/z
+ FKZKGAgw8p7j8yA4gXvh7RX4RsJrECH/Ba7wBRbmiGUwHUREKH0N0zjEkmGWVraIhvfuqhVC1Rqxco0I
+ iAPflcu4yhVkBBqqIR5YkTC9jK0FLUmEAI0YZO0qrm4N3BovQZkAONav4OpXsAA6IF6/pli3CiMcI/cC
+ c3A9wAhY71HeuA6gwzECOsL6CqkBoEe4J9aBea+RYES+5yqhYQ/o/xNyEx4GF0/jBtAP3xNXt4KgXLuG
+ q13/XXCdAOur8G6Aq13F1Kyia1ZAmOrlvfci8G8ChPu1qxAakashOgLQ3xBrl8hVSzCFoJQvUUqXyMib
+ 3hty8WsKUF4I3vGaVvKWWbTIArjzXzDznjH2lPuMmf2UlfGEnv6Ymj5Hz3zCyHjCTJtlpj1kpjwCMZJh
+ AjrDiHvAiJ6gR48zIodpgHj4IFBOgxkdBCoBncA3xb+DCnx7t1C8m6ieTRSPRtL9eoJbHZImAb5da3D3
+ qjDOFWjnCuzdCtzNMsz1YtTNYsytQrRzEe5eMf52Pv5mNv5mDgh3Ixt3LRt9KR0B3TaqBy9xdKwEdCwk
+ T8DCfTvAv5F5QCAyy8QH9eLAwoP70cH9qNBBdOggJnwYGzGKjxzFRY7hosZx0RP46EkCkiGZxEcjNyFz
+ QkLyJFP4eJhZQmQi8ey99IgkMYLfAxq0N3dE8iGI4CY2/QmkSjAZ85gsyI08xUmEBSERCIIycib3OQJ0
+ 7jNczjNMznN0/kts3gsITrCQEoEQvOA1Gf42EJAgs6hnqKLnMOVC3oglMzwcTL8k0zJU2Vv58jcKlcvo
+ qlV09Rq2Zg1cEMJcdDWcWcIA4ojeoqqR8ANVs6QI0Uj9CkYioBkBq34N07CObVhDnBLgbgSt4WFsWMPU
+ rQLlChC9wAMa1+E8Yrp7yDauI8bcuEZCDiQ2j9wEre7pE9O/P4wIX9u0ToARrpY9waW1d9C0RmzeBNbh
+ R8AzwTRuYOpXUXXwTrKBrV/H1K1h9qCvWcPUrMHviJIcYKtXkSAeYn2gHEy9bo1SAxEUaIlS85YMXi6Z
+ GVMrlully1TklVykFr2iQcRS+JKS/5Kc/4KW95Ke94KZ+4yV+5SV+5wNB9kLrMwn7IzHADct5SE1BTFy
+ ZspDVso0K3mGlfSQlTDDigMvn6RHjlIjhmlhQ7SQfmpILy24hwYu7ttG9muj+bZSvZop3ohoHvUUd0RU
+ 1xqCcwXmbiUIB+Z9sxh1owh1vRB9tQB1vRhztRB9OVfxSrbitRzU9Sz0jSzcrRzi9Sz8lXTM5TTU5Qz0
+ 5XTMvsJh29gBUmArHlLpvhCIQ0K9g4AkCiFQgbRON04Si+NC+nEhA5iQQcXQIcWwYVTEKDpiFBMFec1x
+ GBFFT2Bjp3CxU9iYSWzMBCbuASH+ASFuCpcwjUucgSkmLvkRZi8x8in3NwfHIAyMvyMO6SokPQIJPkme
+ BJ25gMp+is59DgTjc18g6S04Bklw38MaDy6OJL9eYPJewjxyTxB5kwoXScVvSUVIXg9b/AJV8hJdsYhk
+ P6reIuEvRNKVS6iqZUVQ5ZI8omWF6hXFmlVU7aoksF5ByK5Fgmx07ZJi7YoCUFu/gmpcQTetYprXME2r
+ 2EYIA1YQyiWQIZw1rmGa17EtG/iWDVzTOqZpHQ1qXMO2bOJbNolN6ySAEgR8N20galxHiEcE5+HeDVIz
+ CO5FHkluWqc0bZDh3qZ1YvMGoWWD0LwOWOOb1/BNq3g4aFpHxuY1uAvftIFp2kA3rqP34qWGdeQaq1tT
+ rF9HS0CXUA52voauWUeuZ8klDaAjoQuwXgkTg2UI/Wm1a/TqFZgZw1yZXvWWUblEL12kFL4kFr2kFL+k
+ Fz+nF72g578CxEGMvBec3GecnAU2KHuelfmYnfmYmzHHSXvETJ6hJ88wUmaAcnbyA07SNDdxmhf3gBMz
+ yYoeZ0YgU09aWD89pJce0EUP6KAD4j6tVJ8Wuk8Lw6Oe6tFA9Wpk3K+lu9XQXaupzuWEWyXY26U4GG8U
+ Ya4XYq4VYK7kKl7IVrxagLuci3bKVLicibqShbmSgbqagb6WibmUqngpReEiKE3BKUV+X/GIXfwQOagN
+ F9iG9+/AI5lvMHIkYwjhOFZi5PgQiMUHscEDqNAhhdBhhbBhBSA7cgwNY8wkIA4jNvYBpAKxcVOo+Ck0
+ KOEBVpLnxiXP4JIeYhNnUEkPFVNmMZL0H5LVTp1D/57hRvLcv+cBMVnPIOUHggNU9nP07xADx9g9gXlL
+ sMb+jvUffOMKIBH2GkYcZEsKJZnskje4kjeY0tcYyWQL0hcwBUSmepJZI6pqSbF6WaHqrXzlW/maFVTd
+ KgrxQuTNHQ7QDYgw9YiRy1cvy9SvKjSto5pXFFvXMO0buNZ1YBpMGtgCwjDNm0AzrmUT27aFb93Ete1p
+ C9+2CTcJbVvE1g1iyzqhZY2IaJ3cuiXRJqKWTRIIUAa1bpLaNkitG5SWDXLzOqV5A0RGvhAunnX4oYja
+ Nwjt64S2dXzbOg4ZN/Dtm4SWTXgOmJYtbDOCO0wMgHIFGBs20HXr2Dqw9k+CwAaZqsKkYi80ql2lAOhI
+ ngden1Vq1Qq1cpla/ZZW84ZZvcgof00tfkGWiFr6gl7ynFH0glkAE9BnzLyn7LynPFDOPCf7CSfrMRsQ
+ z5jjp8ywk6clLv6AlTjJTpzgJoxzY8e4MeO8mDFO9Cg7eoQVMcQM62cEddMDu1iBnayADpZvKxMQ92pk
+ ejYw3esYrtU0lyqqSxX9bjnjdin5egH2ah7qWgEWUT72Si72Ujb6QibKKQN1MQtzPhN9Hg7SFC+moi6n
+ oa+koy6lyjslyZxPlAVdSJU/nyyzr2jENm4QH9CCDmjHBiJzTcioECGpgqwt9WACesHFCaHDxJAhWIVC
+ h42gwsfAzhWjxlCRYygJ4ujoKVTsA0zcNDYGKH+gmPAAnTgNOUFcChKrwCIONvEhOvkROmUWlToH+oNv
+ dPo8Gll3mEfWbiRGjpXktveEzgGf/p3mPcT3yP6Dckkme08wW0IEee4iWKB5gytCEn/IsghMJSExXPoW
+ VQGh9gqEoZSqZTKoGtILMJtcgT85GolbVsDwMODiQHkdBLJwAKwvo+uWUaDaZfmaVan6NdnGdYWmVbmW
+ NVQbWPWqYhP4t8RHAfSWDTByRK1bgB2+YxPftU3o2CJ0bBKBSxiRg01S+8YntW0SgM52OLlNbNsiwE1E
+ W8jDQG2b5LYNSusmrWWT2roJB3AG37GB21PnJr5zU3L8x8lNuKLgp8PTQHBHnhJY+5oiEA8RVMM6vmEd
+ ZsaIGjdw9es4JOxBgiUygF6zCuE4JPgJFfDKrFArliiwblX5llb1hlHxmlb6iloMXv4KQnNK4XNq0TNa
+ ISD+jJO7wM1d4Oc9FeYtCHOe8EHZT3hg56kPuUnA9xQzaZKVOMGOH+PGjfDiRvjRw4LIIV54PzuslxnW
+ Sw/toYd2MwPbmf5tLL92tm8r27uJ5dXIcq9jutcy3aoZd8upd8oot0upN4to1wtIV/Nwl3PAwhHEr+Ti
+ LmRizqejHFMVz6bIO6YpOqQqOCTLOybJn0uUO58ofyEZKJc9lyB9Nk7aIUbqXKLs2XhpCF1s4oYIga2Y
+ gDbMJ9AhI46so2ID+zBB/VhYWQ0dIoQNE8JGcOEQkU9ARA5ejihmCowcRnTcNA5Aj55UiH+gAOYNcEsC
+ cXwyZAPnYJESghY0QvljRVDaExSQvUe5JETZAx0sHBZuYJESsXPEy19iQX8gvnewRzkCPbKqRyhBRIT1
+ xeKXsMpIhFxK6SIBMtxlr5GsdjkSgsOioGL5MrpiBV+5SqoEx1qmwvtyDdgYhKeQgFvFISO8j69BvgJJ
+ wNVCQLKOBxeEGAYicjB1IKZ2DUCXalyTbViVbVlTaN1EIdCvoVoRvP5PretYBMQtAqhrm9i5BSJJ2CV1
+ bZM7QVsgCjIisOK74GF7ktxEwN0EwwbBOwC1bYvWtklt26R0IBcAAnfnJlYyfhJ8IfJVG9iOLRw8k+Z1
+ 9B7ryLNCrB3TtAnEQwSFxPf/j5DYCea7tSBJgrIKEi/L5MoVsHZaxRK1DFh/Sy97Qy9ZpBW/pUAQWLhI
+ zn9FzX1BzQM9YwLiufOC3CfC3CeinMfCrFl+1iwva5aTMQuxODPxATNhihE/wYob48QMc2OGBDFD4qhB
+ cXi/MKKHF9bFCumgB4PamQEtTJ8mpk8zx7uJ49nA8mxg30coZ7pUMu+U0m4WUa7nk67mEq/mEq7mES5l
+ Y4H1i9mYCxmYcymKDklyoNMJMqeTZO0SZc4kyDgkyJyNlzkbJ+OISNohVto+WtY+WsYhVvZMtPS+omGb
+ eCgV6MQHdmBhoTUQmXdKcuT92BBAfBApIUAOgPURHDL7HIdJJzZ6Ah09CXxj46Yxv9s5OmpSIWFaMfkh
+ BviG1XhkQX4WUoRIljD1MQQtKED8/x90YH1PMIPMf02C5UkAHew8X5Lk3tOekYMgtQLZbmTZ8iXkCiWg
+ I4gjKgMh6+cgWLKR5M6WYfVbofSNfDlkCVeIVTDTWmXWrbBr37LqV1h1y3TIoMHaTfVbUu0SBVS/RKlb
+ JtcvExtWyY1rZJgRQijcDHHwJrZxU6ZpU6YZtCHXsqnQto1u2UQ1b6CQ4EQSn0hCFBx4+R7lEhG7NkGk
+ TqB8i9S9Te7eQg6Q4y1S7xZxTz1/aJPQs0XoRi4P+FpS+xa1Y4vWvgmigPfDBdC5iekGbWH31LOF7d3G
+ 92zhujfRnRuY9i2IlHDt2/B8cIiv7wnimQ0gHkL8T5JE/zDZJTesQDYTWW9CTH2NVAHRC2RaVmgVK7Ty
+ ZWrxG0rRIhnh+y05Z5GQ84qY85qS84qWAxPQBXb+nFLBnFLeQ3HujChnRpg1zc+a5mbNcDOmOckPICLn
+ Jk5y4sbY4OVRA9zIfkFkn1J4ryi8RxDeww/tZAe10YJa6UEtTP9mllcd06eR59XIvV/DvF/DcqlkOJfR
+ 7pTQbhSQr+QQL2cTLsGcModwOQcHUYpTOupcqoJjsvzZZHn7BDn7BPnT8bK2CTK2cdK2sVKn46TsY6XO
+ REudiTxmF3HMJvyYbZiUXYTs6Sh52wjZfYUjVnEj2BAEcUxAD9YfwpUeTFAvNqgPEizYkAEcpBGhKCV0
+ EBc6hGRaosYJUZNg6sA6zDhxSGg+BeE41J9g4qdQiTMQnaOTZrAgSeUJHlk4gOVfZHEeyaWkPUZL0imS
+ ueZTcHEwdWQEIwc7z4MVylewRI/MPoteE4tfwzoOTCgheQJL8TigHPiG4BsJwSFd+BJX+pJYArHjS2r5
+ a1LlK1wlrGYjC+8I4rWQD1km1L4hVy9S616zml4K2p6Lu1+odS5odcxpdsxqtk+rtU2pto4rt4wptU0o
+ tY+Lu8ZEnQ9E3XMqPfOqfaCnot5XvK4lZhuE1JuYtm1U67Z867ZM67Zsx65i+za2cxt8mgLO3QWOu0Vs
+ g9AZiNwm7glCl27QFhEZt/F7Y88OAdS7S+jfJQzsEPq3CX2fROzbJvbsEHt2ST3bFFD3Nq1ri9aDjHBt
+ wPsDrnsbBw/uBW3he7eRxw/skvp34GrBd2/CvciPg+8AY9s6FiYSrWt4ZGIAQiJ4eItAZgItm5TmdTLM
+ dBvXKPWwVgWzz1UK+HoVsrRErl6lA+UlS5RiJJkIaURy/htqLrjPC1LeAqPwibB4RrVsUqdyxKhi2LB0
+ WL94SLdoSLdgULtoWLtoSCtnUDN1QD21XyOlTzWpWzmhWymmSxTZJYroVgrtEYaAuoWB7Vz/FqZ3I9W3
+ kerXzPauY3nV89xreeDi9yoYt4qpEKhcyyNfziZdSCc6pRHPp+LPpWIdU9AOyQpnkxXsE+XOJMrZJyqe
+ SVCwjZW1i5a2jZGxjZK2jjhuHX7MJuKoTcQx2/CjdqFHbEOO2IBCj9lGSJ0KObqvaPRU7DAmpAsd2IOB
+ cCWgF+3fgwroQQX2ooL7P+XLJSlzKLiBZCIe8oaRE0iyJWYSB6mVOMgeTuEkoAPuGBDgvsf6XvQCC0Bp
+ c8g62e95Q2y6JCgHC98LxyWTTrBwJGiRVJsgNSeQ+QbDBhWCkOBbsmaJ1JkggTgsXkKpUxnkwl/B0g+9
+ +BWt7DVBspQDGXHEuSEfXL9EbX7Nbnkuan6i2jKt0T6m09Gv09mp096k21yr1Vip2VSq2Vyi1VSk2VSg
+ 3lKk0Vyk3lKo1lyq0lGv1d2s292s1duu0T+k1jut1LXA71pkdq/SumAKuKHQtiXXtgMjqmOH1L1D3iOs
+ e5uEOPE2oXOX2P2OhGiX2LtLAiH4ItpDHE4iGkREGtwhDe2QJAfk/h1y3w4ZAR1GhHUqqHcHRrLk+8CX
+ 4/t3gGx4NyDsHYAAd7jZs0f5Flw2yM0uiKCQOSsEPJT2LRKE/ogkE1+I+CGx07SOvGXVr8BCFRVWqWrW
+ kFR6JRThLNHLlhglb+nFizTIrhS+YBS95BY85xbPC0oeqJYN6ZV3m1S3Wtc3nKmrt6tpsK2st65oOFVW
+ b1HRYFnRaFXaaFnYaFXYaJnfYJpbb5TZcCK1SSe+WTO6WS28RSW0WTm4SSmgQeTfyPOuZ3jWUj3rYd7J
+ 8gDKKzh3Shg3i6jX8iBQIV3OJl/KpDilURyTiWeTcA6JWPsE9Ol48G95gNs2VsYuVt4mWu5UpLRNJFAu
+ axclax0uZRV61DL4qFXIUZuwY3ZhQPlh65Ajp2AMO2wZdHhf6aRt/CguqBMqtFCAOEI5eHkfgK64tzC0
+ Z+oAeuQoIWqMEIWELjigHBLkUHGFzEcnINmCiZv85OWJ04id71H+/w06ZA+hQAUqUjAZkgSiJBxHJqAg
+ AB0pCXxFKFgEUwfzxha9wBU9xxUgaz0QjuP3qv9KFpEVTajmQwr63kI4DssZUAVKK4HpFCwEwnLgMrV2
+ mdv0RqnlmWrbjHrroHZLm25DlV5DoV595onaOO3KYLWKAOVKf9Uqf40aP81qH41qX80aP40qP80qX60K
+ f/WqUPW6CM3GKI3mePXWLI2Ocq3OFs2eMZWheZXhN6LeVXonEiEotmzKtW0rdm5jwGjBZft2CH27hN4d
+ fA8Q+Y6IaBfgllC7uycSMiJniD1gxtvkwW3y0A55eIcytEMd3KHAmf5tYJ0kEaVvG0Tu3Sb1wsldOEPs
+ 3SEM7ML1gMD9ifItxNcRuBGB00tA38T3bkA8g0MCpy2I7yltyFwW3nBILeskSTIHyV02rhDrl/B1S8ga
+ Vs0KEVZAyxappW8YpW9YJYus4kVWwQte0TOlsqdaZY9OVI4bV3WbVdVZ15Sebci71Jp1rSXzSmv21eac
+ K825VxrzLjfmXarPu1ib41SXd7Gx4FJd4fmKgjMlhacKSi2zy81Sy42SyvTjSvWiS3TCy7SCKlX9qoUe
+ lZJApQJiFdbtIsaNAtrVXMTIL2dTJF5OOp9KcUgknInHnIlH2cWCf8vZxcnZxMhYR0oD4lYRMlYR0tYS
+ 0AF3i5Cj5iFHLIIR1k+FHgW+rYIPWQUfsQw6YhF00Dzg4L7SqdPxIxCjQ50WGDkaKA/qBy9H9EfcIqmn
+ hRVQxNEjxwDu/wM9agITO4kFyuMmETvf0x9xC9j5nqNDpSEU+kB1G+Lr83uUw7IOVA5CHdwnL8+H+ATm
+ PW9IBYtIYSACN1JNBSO+5CWsZRLLFpEoHKabEuFhrb4USl6B8mVy6SqhbAUHqYPaRWbTS5X2af3GLt3a
+ crXqNNXycNUSH41SV62y2zolV7VKnVTLHFXKz6tXOmnVOOlUOGpVnNOucNKuvKBXdcm46rJh5RXd6qva
+ dTd06m9rN7rrNPrrNkVrtxVq9LfrjD/QHX2t0r/O6NrGtu3IdbyT7dqV793B9u/iB97hBnbhQKJ3+D7Q
+ LqBPAvUijAKvyNi3C9SS+3cBcdrQNnV4mzK8TYZxaIcCpj4A2v1/RQGbB8HJPiB+C2IViuQYgXvP2vfc
+ Hbk24IxECOgSdW/AHJcM6lgnIBlJmOCug0gta+RmCeh1klIFWKmFDHr5ErlkkVLyllayyCh6xSh8xS54
+ Ji55ol31yLRm2Kau9Uxj+fnmrGsdSfd6Yu/3Rd3vjbzfH+PeH+veH+fel3C/O96tPc6lI/Zud9y9rljn
+ joRbrYnXm1Iu1aU5VWaeK892LMs+U5Rlk5N9Ki3XKjbfKLhIw6tIcK+QdbuAeT2fcS2XfiWbeimTfDGD
+ DIifSwEjJ5xNJNnH4+1iUbYx8qci5WyiZa2jZKwipazCpS3CpD4p9LhF6HGzoCOmAYfMgemQo8C6ZAS+
+ D5sHHbYIPGIZeNjM7+C+4onTMUO4oA6MfxfGrxsFoYtk+VNRErpgkYIWSYz+B+jg6LD2idg51ItPIY4O
+ oMdPYiB02eP7/4fyPdAznpAlFYVAPNTKgoUD3EjBt6SKEBkhOv9/QCdCcILYNtD8e2oFWbpfJJa9+T/Q
+ IRwvg0qMt6TiN/iCRagoxNa+YrQ8FTePatSV6JREK+V7cAqu83LP8bJPi/Lt1AptNIpOaRRYKOWfFBSY
+ ikosVMos1QtNVfJNVQvM1YottcosT5Rbnai00a2y0ym306k4o13jpFt9Savqhmq1j0pzkkZfve7QhObI
+ a9HABrVnF931TrrnvSyQPbCLG3iHGdzFDEnGwXdYhPt3eLgAJNcAAdEuYEocfE9C9I488o4yvEsd3aGO
+ bJNHgfVd0hDQ/44Mdw2+ow69p0qIB6wRwZl+xOPhCz+Bvsf3H6BDpP7/A3rPhgT0DWLnOqLfk5vEtrXf
+ QYfKlk+gQ8oFybSUSF7M0jeUkpfU4mfcoseq5dP6VcOWdS32LeWXujJv9sa5Dkf6DYf6D4Z4DYd6j4SD
+ fIbDvQbCPHpD3XtC3fvC3QfC7w9EuPWHu/SEO/dEOXfH3umIv92RcLs94WZL3NX6hMvlKRezUu1i0439
+ 09Xc0nl3MjjXM5mXMhkX0qnnU0iOyXiHRNzZRIJDAvFMHNEuBmcbgzoVKW8eKmUZLmUZJmUOZMMYJmUW
+ chxkGnzsZNBR44BDpoFHzAKOmAUeNQ86ZhZ89GTQYZOAgyb+B0z8DpgFHDT1PbCvZNwueggb2ImsDfl3
+ IdF5UB8qpE8hqO9T6PL/gA41LeDokGGE5U8kdEEWQcHOIUZ/AHzD8hAC+h/ai1v2xvTHJKSubR5qxHFQ
+ N7tXBf475aQ8KPV8CaWzJMi67Jk6xOVg5ODf4OKS0pRPiMN2hDJIIMIyPpTXvYVlDqjeRs5AuWzNa1rL
+ glLLiHpFgSjPhZvhSEu1Iqeb0DONOZkmomxTlXQjcbohP8OYk2HIAmUZc7KN+ekGnBR9dqoBL91QkKUv
+ zj4hzDUSZxkrpxkppZkoZ1uoFp4Sl50WlV0Qld8T1McotVerDT1QH3krHtqk9W4r9u4o9O2Al0Pogh7a
+ lWgHNQi4S1jvf4fph4P3mMH3ONAe/Yh28UO7xJFd0sgOaXQHGUfekYbfkYfek4c+UP6gfOg9IL5n8Aju
+ QPb/G7oA9H9YO4COGD84PXj8JrZ3E9e7iZh9F0yOIYv/KZFPhiw+TKzBzptWSODojQjrUAIABS0wASUX
+ vyVCTVvZa2r5c1bFE+XKqRPVg2bVzbb1Jec6sm/1JroORniMhPgNBfv2h3oMhHoMhtwfDL4/EHy/J9Cl
+ K/BeZ6Bzd/C93tC7QHl/6L3+kHsDIa6DYfeHwu4Ph7sPR7r3h7l0h99ribxTEX0xK9YmMk7PK07pbjzv
+ SiLzfAr9fArVMYmMxCoJuNNxuNOxBJB1JPpUBMoqQl4C9BHT4KMSuI9Kbh4/GXTMJBAoP2Lod8jE/8jJ
+ gCMn/cHaj5oGHjUKOGzoe9DI54CRz35g3cRn/76icZuIAZR/O0xGocQcKMeEDGBDwdQBdziQ2PmeIHRB
+ YvQxyDBCygWRZD6KZF0kM9FPXr4H+qeIBQo155CCxPQ5qN6EqmUoU4atD7BFBQnNP3n5C0jQ0vYEleL5
+ rykFi5QiJDWO5MiLXkIZNJREwz6aPUcnSOoNoU4a1jjIVYvkmjdQxk2G2u7G1/z2ae3aCtVUd0rsSVy8
+ Lj5OkxijTo3VYMfpCaJPCCK0OZG6jHh9avwJSrwuOVGflmzIiD9Bi9GlxurS4/WYSTrMBC16uBrFW0S5
+ r0S7r0T3UaHG6DLyzAXFtqJ8e06RM782TrW3SWd63mhmXWtsi923BhNBiBzIfVu4gR3MMCI04usI6IA4
+ GjT4ATP0AS8B/Xd3f4cfekcYlmjkHXn0PWXkA23kAx2c+5OL74C7U8H74a1g8D1i85IEy/9NQPeg3wtg
+ EPrhMpBcCRC6wDMBQeaxewvy9JCYR5alICnUDjNRJEyHlIuklmaF2CCpcqlYphS/pRa+IcDSBGwxKXnG
+ LplVKh/Xq+owr6i2Kc85W5VwuT7yTlPQ3VZf13af+21+ri0Bd1oDbrf73en0d+7wu9Pqc7PZ52aT363W
+ wDttwbc7Q+72hbkOhroPhXiMhnmPh3hPhPpOhPuOhnoNhnj2BLk1Bd4oCbFPCjMOjFC/Hym+GstxTKKf
+ TaTaxxNPx+JPx+FtozE2UZhTEWiLUEXLMEWLMLmTQVImQUD5MdMQCd8Bx4wDjhkFHDX2P2LoewhAN/Q7
+ DDLyPWTse9jE/6iR31FDn0NGPgcNvQ8Y+R4w8gbQJ20ih1B+bQr+nYoB3ejgXmz4AD6sH9KLSOgCi/97
+ lP/fZBRAn8DC+v8e6JLJ6N5MFMm6/GHnn0BH8i17+gN0ZOcVgJ75FL23DSLnOfUP5b4gg/JekgtfEIue
+ E4pfwL4HZG8LbEUreYPMRKFktPgtDtY+YfZZukQqh1JSycYcWMZrfK5U26mWEkjzM5cNUJIJV8aGKpP9
+ RRRPIc1VTL0nIrsokf20mFF6jGgdWrg6KVqLGqfHjNVhRGjSwjXoUZqMWB1ahBbtDht9Gi/vwKSd57Ev
+ sUjeYmq6kSjfQpRnxc93UipxV69O0Gip0xyZ059c1R5aF3RuQm6EhoTX27iRHezwDqp/BzUAAcwnCwcv
+ Jwx9gKAFyQZK7JkED4asy9B70sgudWSXNvIOEKcNvWMMf2APvGMMbFGG39FGP9CHweDhYe9IoEHw7O3/
+ Yx2YBkf/HX1JukZCOTymbwffu4Xr2sB0bGLaN7HtSKYfybq0IJQTm5FKGyiHJELuFWp3q1cIZcvkwjfk
+ 3EV8DpK0ZZU+Vi8Y1c9rMM1Jt0gLs0r1dshwuZxz51rWtUuZV51yblzMu32pwPlSyd3LFc5Xq+9eq3W5
+ Xn3vWo3LjWq3W/Wedxp9bjf7OncF3e8L9ewP8RwJ952I9J+KDJyICRiJ8B0O8e73d2/3vl3j75QTYhUd
+ ru8drnktimcfTz0TTzkdS7SNxgHl1pEoq3BEFqEoixAFs2AZ0yAp02CIVaSAcmP/Y0Z+xwx9jxj4HzX0
+ O3rC+yAcG/oeNfA5ou992NDnsLHPESNvyehzWN/zgIH3fgOvH/YVTVgD6P7tCn5dKP8uAB0X3o8PR2rN
+ gfK9Wlxi+AgUmpMg6yIRODom+pOpwwEmdgoJ0EHxD9AJ0yBMImRdJNNQSR4dAT39MZTkU/ccPROqbRFH
+ h/kolB9SQDnPqDnP6DDC2hso/xU4OqX4BQnWPvfyiUA5sF70Gtb2sYWLGGRf4yK2aIkIb7iwzFG2SK54
+ yayaEeUWcYMuK7qqH/USyAaIsF48/F024RwRZY2TtyUpOtCxzmJ6sAY7UosTqkoPVaJGqTKi1VnhyoxQ
+ JVaEGidSix6oznAio3WlpI0J1DN8ZUcm8w6bGqPFzzTiZZ3k59qpFtzQyvFQTguhxsdLV7bTRpY0+zY4
+ PZv0vi0KJMWHd3HDu+gBRBjJ3BQsHIAmD3+gSsJrEHg2BNzgwSQIUUY+MAZ3GcPv2EPvuEPvhaP/VB58
+ LxjYZgy/h/MQw8BjSMMfIKohDkJW5/f04h7fiKNDmmULMfU90GHW27ODgyWkLkB8A90OuX9J7Q1SabOF
+ FIRBWSXUh4GdS6qFkYL18hVcMewTB9CRpToIzcVFDwySG00i441DnA28HPU8rU/6mFv7m9r4nbT0MTP1
+ tjTxszYNs7OMOWObaH865cyZdHv7DAeHbKfzeVcul9y6UX77RvmdG/Xud5u8Xdp83Xohmo/wHY8KGIsL
+ HIzxHwjzAdA7vG41+F4uDjqdEG7qG6p9JYx3JppsG0O0icKfisRYRYCFy5uHyFuEKsB4MlD2ZJA0gG4S
+ cNwkQGLVCNYQrhzR8zmo531Q3/ewvvehPel5HTTwOmjkfcTA84i+5yGJDhh47dfzBNDHTkUPogPbMX7t
+ GF+I1LtxIT3YsD4slCvCTFTi6PjIYULUMDF6hBg1QgLFjMKOZgKiMXzsGC4OHH0KYhh07LRi3Aw6bgaW
+ S/GwHS55lpw0C7unSCmwZoRs39wLXWBfLfr3zZpQNQ6g7zk6DVjPfU6H4s+C14yC1zBSkCLE18RCSMIs
+ wvoormBxr2wLNnTCHl4srGggk6dl2DVMqnrKrehVSolm+lrLuyhJuQkVvUUEdzb+KhFtdPyI+rFjxnj8
+ GTrtCovmKqQGaHBC1FmBInKAkBSoRA5WpoUoMSJUWZEazEAV9jUq+cQRKZWDx4xRuNME3C06MVyVmaJH
+ TzdgZ1qJ0pxEMZfIkdeOR7v8kBFzfHhSY3ZLZ2Rd1LUMKzvEvg8QroCdK/bvKEJE3vMO1wvhygfSwAck
+ 0zIIgoDkHQ0gBoceekfs/0Dpf88a+6f62Dvttpfiuse8nhXVoV3R6C4LJqY97/A9PwHrlFHEy3Hd77CQ
+ yUFmtO/gy+G6IiMLq8giFL7nHZK77NqF5Sp8xza4OKZ1Cym/Qcq8NnHNG3hQ0wbUukC1OlKejmw8XSKW
+ LRGB8qK31ILXUHNLzXvKKJlTz+kyCEzUdrmh7Wypd+WExiUtzavq2jfUdG6oaV9V14Sbl7U1buhoOOtq
+ u+npeZw44amn7613ws/IMMjMLOyUVaSNTezp00ln7dOdHHOuOJXcvlrnfqfNz609yLMz3Ae47/R3bfa5
+ 3eh7o8rvfH6oTUS4gXOo8Hw4zTaSYBWJtY5AnQpDmYYonAyRMwcjDzxuFHTMEGJukP8RA7/D+n5HDIBs
+ 38MwnvA6qOuxX9fzgI77j3Cg53kAxhOeB/S9EOKRuzwO6CHjDzoeP+wrHjsVC6C3YfzasL5QBdCJCexE
+ BfegYYMFrB8hFV2DuPAhHLJaBLuHRsgxyL4hUgxsqoDNFiP4mFFcNBLGYGIeoGJmUDEPMXEzuNgZQsIj
+ ctIcJXEW2S8IG2PTnsA+QpiM7oEOK0TIOigSoyOxCuVT6IKAzsh/ycx/xUAKQV9T8l6R9jbkwmb7vYou
+ SXEiwnrha1zpKwri9EsQwROr5/llLUpxPhQPQ1kXgYybAO0lJLqziOfQ8io//sj68YAWCm1LZV6kM5x5
+ FG8VZoAqK0CJ6i0keArxfsrkQBVasBI1WJkapMJ25TLtFFEGR48ZHjvmgJHz4dPjNNgpuowcI262hSDx
+ NCP2HCbx0vGMWwfT7h8caFF9+Ex/5KX64Ipy9waz9z2x/z26fxfVt4OFzDeQ2gv5lg/kvvdInnEvdAHK
+ IWU+tEmE9HnvNmvsJ63JHZOSXrpb5N9u+f05t5Uwsq46scUb2KZ2vCe0Q8CzRR7foAxtk5Dvhpg6rKdK
+ QhSgHFwc1qd28d07+K5dYscOlIXh27axrRIB2U1QPLyG1McjdV0bSGlXzSpsK4GVY1jwJ8GbZOFbqNqn
+ FYC5vGAUzPPyxtVii1WdXTgXLYWXdbWdNDQcVVXPqaheUNG4oKxxXqzmqKLmqKbqqKJ0TlnpgpLKJSXl
+ K8qqV5RUrquo3VRTv6mheUtT+66OrtsJfS9DAz8To2BLszh728xL5/NvXyt3d27wvNfo7dzg69zgc7vB
+ 91pFyPmkSAuPcNULoVSrUIx5mOKpcNSpEMWTQXJGQTInA46Z+B82CDx6IuCofgBEKQD6kRM+R/W8D+t5
+ H9L1PKjneUjHfb+m6w/abgA64L4fpAfnvQ7pehzUcYczgP4BPS/kAthXNmadMIgJgdLFTmSnHIAe1AlZ
+ F4jRoagLSnMRR4c9RNCXAvYTRUgaEsBN5Aw4/SA2YgTqdSFkR0VNKkZOKUZNo6OhugtqYKAGfZaUOEtM
+ miMkPSakwJ78T6DjJfVbku33yDZyUs5zAF3COoQu4Ogv6DArzX9Jhc2dEKwj3SYQ7bEOVS571S/I5uWS
+ l0RItxcuomEjY9UjXnGNMOouyVVN6g7l2F2mgicPf59DtMfIKu3/nvb9d+Ljxy0I5EsMmiuH5MkHkX3E
+ FC8x0UOI9xTgvYUkPyHRX0z0V6Z6iRl3ubSrDNIVCsaNRYxX5WVo8TP0OHnG/BwLXpodI+8qvcaNXn+f
+ UOAsleEtNTZ4cv71mdEXJ4bXVPrWaX1beIhhBnepfbv03neUXpgpvqeCkPQ5ZAl3KbAAhETkEJ9sC6d2
+ DaY3bLLKKV7e/7h34yuXy1/FRR0eeqzyYE1teIvb9Z7W+YHav0UdWacNb9KRVdJdYB0H601/qAtBHN/5
+ Dg+UQ6ELUsu1CRWUuNZtKNEhNG1AuIJs2kC2q6592ksKTQRgeaj8DYCObJaF7UKwiwI2B+U/FmX0qgQl
+ CK5doF8w5F1S13RU07BX0zitom6vrO4oVndUUnNUVnNUVXNQVT2rrOIoVj4nVDrLE5/li51EKhfEqheU
+ 1C4qa1xS0bisqn5VDaR6XUPdWUfLw8gg2MYq4eK5gtvXK1xvV7k713rerve+WRl0OSPawS9K9yKAHoIx
+ hXAlWNYyWM44SE4/UBos3AgB/diJgOMQjgPlBhCR+xzT8zqi53lY1+OwjvtBLbcDWm77YdS+f0DT9UcQ
+ HOh6AOh/6LCE+0MAum3iEC60AxfcRQiGfhXduOBudCgSuuCDYL8FUomOA8rD4DFQ7oKUeUGBF3IGEeL0
+ 2PBRTNgolO8qhk+iIh9goqYRRU+j4x7i4qBGdw6b/ASfOg/dVJCd4RCjI50nJKxL2oPA/nxi5jNSJnQF
+ eUbOBtzhRX8J6ZdPAtx/Zx3KXSC/LglmECEpyIKXEMagSl+Qqh/wC4v4oZdwLjyp25jjt8iyrmzUPRbm
+ AgVlgpLWkD2mryh/lkp1ZjN8uWQfNs6ViXJhodx4GC8RwUuA9xLg/MTEIDEpQEz0E5N9RBQfFVqQGjNC
+ jZWkxsrU4KbpsHONBXkW/FRLcr4TudWF3e3Kar5Hy7+LL4kTPRhxnJyzHn+lP/RWMLhKH9hk9G9x+3cE
+ fTucnm1m9xazd4fds8Pq3mb27LB7d9kDO9zhTeHouu7kW+u6Nl3vu4cCrxxKuSWdfv1YTRRh5aXN8xXj
+ kRVx7zte93tG3zZtYAOuH0b3Dq1jB9e+hYWl2c4tPFJusA0jrn0H375LQLQjAX0L1wT7PCAo3yQ1bpIb
+ Nsj1kj2mdetEmHpWLkM5LkziySVvYIUIiuHIhVCW+IyeNc/OnlFOalLzCuBfOcW4rC26qKJ1RknTWlXT
+ Wk3LRkXTXlnTUVXzvLL6WWVVexVVB2UVeyVlB5HSab6SPU/sKFJ1UtJwUtZ0UtW6qKbtpKZ5QU3zvIrq
+ eWXl80riCypK17S1PMxNE5zO5ty8WuRys9Ltdp3HrUq/K1lhjn5huk6BJIsglGmwnFmgtFmgjEGQzIlA
+ KcOA40b+xwwDpPR9j+v7HAMv1/c+putxRPv+IZ37APoRbbdDmi4A+kFN1wPaMEqO4QAZ7x+CA8kjD2m7
+ H9S5Dyujo7ZJQ/iwdmxwBy6oC4QJ7saE9GBC+5BN0AA6TEmDB6AWAI3MTaG6CxFi82GDOKAfWA8bwoQO
+ o8KB9UnMH6DHzKBjH6JjZxQToEZ3/g/Q91iHcheo6PpUtAhJmIynRETQGwRYf0HJfknNeQmFcrTcV2Dt
+ e6Ajo2TrPlJptKcCCeiFr1EVL8j106LCfEHIeZwbV9YFJ3eTLHWLIXOHIXeDhb7IwjnQcedpxLtsuheb
+ GsQlBnAJHmwMXAn3eWgfEc5XhPMRYf3F+EABIURIChOTQ1TJIRrUcC1GtCYjXo2eoslKPcHNNORlmXET
+ zQiptpiqS5SuO4JeN7UaN+UQJ2xmmPDtK9epecux51rjb1WHl5X7VtUGN3X61jV619RBPevqvZvq3Rsq
+ XesQ4ah0rygNLWk/XD9T06rjdv2Ar8PB7Kvo5nvEfndKrz/+bZ/J7tr50Zeavevi7m1O9xa9CyoZdyit
+ u5Q2KE7cxkMiBSiHcBwqFsHFwbz3LLxlG9e0hWncxDTAjqdNAmwkrYeK83UoZZH0yVghVq6QkLh8kVj8
+ BkrlKAXPSYXPiQXPqdkL9Iw5XtqoalSxuus9wWVj7hU1NUeRpjlHyYCrZChSM1PSsFbSsFfWcBSrAtxn
+ xOIzIkSnhcLTArG9UPmsUMVRpHZeSd1RWf2cioajiuZZFQ0HseoZoZIdl2vL5doJBZe01f0sTePOn8m8
+ eakY0jX3b1T5XM4MsPf21z7rQzD1ByOXMg04bhoopR94XC8IAd3Q75iB7/ET3uDiR3U9j+p5HAV2tVwP
+ /iGNe/vV7/4II5yBEdAHSe6FUXLgcgi+BK6BT6CHtEHEggnuwgZ3oUNhMgpJxj5cGDSxGEBidITyfgyQ
+ HQ55RqSSUZJZH8BDIjIc0jII8RjYXxc+jomYQkU9gOgFoTxmWiFuRjHpMZg6JhnZCQqUQ6MVctZTABrp
+ GQQLRrA4ivQPQvY7Qx8cpBVOLggC9+eU3OdQ/UzLgx3QLxDQC16BIJIBxCl7Qvb2w+701+iql5SGaXFe
+ DjfQAePKlbtHVLxBlb7FPH6XLX2XLX+HjbnFxt9h4iFk92MR/JkYbw7Wkw8ujvfm47zYaD8Ed4jp0f4C
+ YpCAEiqihqlQwzQYYZqMCC1GjBYzUYcLGcYsY1G6KTfekhxjhco5Q6i+yK6+Iiq8phxzmRt8hVKcrNnf
+ bT49Zzn72vLBG4vJJeuRt9Y9LyxAA4tWnc+Mu1+ZDq5YDq5a9Lw1GVg2f7Bk3zVq6e5y/J7tj1lXCa13
+ qJNutNcB3Hl/2kQCe3PO6clb+8G3hn3r6l2bvJZtav07Yv0uoRmi8B0iCAlUdpANTZBOadnCN28C4p/U
+ sIWt38bXbZKgDYYEcXIVUogPQTm+dAl2pRBgp3Mh7I57TYXd5fkvSDlPyZmPaZmPhCn9moHpajcvCa7o
+ K19S0bJiK2sQWAIsXUzl6HCUTvKVrQXKAO5ZschBLDgjFJwW8O34fFs+HCvbC1UcBCoOItXTIhUboYqN
+ QNmKKz7FU7bmK1mxeafYXCse96yq0l0D3UBb86QrZwvvXK5yu1LlcynT387DR83eG2cWIA+gnww8ZhYk
+ rR8opRd43CAAKD+m74OArut5RMf9sK474uJ7KAO7GvcOqN/dr+b8I4yaLgD6AYAb7tW5f0Tn/jFdd9Bx
+ 5MDjqJbb4X3FwzYJULbVig5uxwR3YkI60eHdmIgebHgPLmKAGNGPjxzARQ7iIkADgD4IGzYANwmRg8TI
+ QZJkJCC4I3ulMZHjKGQfxgwGMjAxD+TiHyokz6GTZlHJyCY6KOqCdivI7BNJlkMJ6GvSXq3iXv+33w/g
+ GpC0zkJS7Ajl0LtCYuef4vU/QIfCxoKX6OJFTNUrSt2UIDOD7mUn7cyRvktWvMkAvmXvMaXcWLJubJQb
+ C+vGxHmwcL48oicbf4+NdRUQIUHuL6IF8ilBYkqAOtVHg+YP2RhNQZS2OEZbFK0jCtXkhmgyI7TZUZrM
+ BB12hqEg1ZQfd4oRboWLMkfFGCtEmWLDrWkhZ9ne9mS3s/LjnZfnJq90thvPzF6cenK1oEYnp1y7oFov
+ v1o7s1ytoF6ncciqbdK2esC4bdK6d8zey1Pxut3BpBv02jvsgdvUl16cD2HKSwGCkWDBu1nP5c2gJ5v3
+ prfO9a/rNW5wqnfINbuExm1S8zapZZvQukNELBy27W2TWrdITZvEhi0CqHGb0LBNqN8i1m4Qa9YJ1cD6
+ OgVAr1iFZQdk/UGSwoLKOdhIQcmBN88XFKQuA1pTzIhSuvV8E9SvnhNf0FE9I1DXpwkFGBpFjkCVJ4oI
+ TEO2yJIntuUL7UUggZ2AZ8vn2ggE1gLhaXBusfppoRrIiq9szBLpUXhaBLYOiW1EF5gxBBYsviVXcFqk
+ dFVD3cPUIPKcTeb1syXOTuXeFzL87O77qJ7xxZsC6MFSJ4OPWQTLGARK6QYc10eS5ccNfKUN/aT1vYDa
+ I7ruR2EEyhGa3Q5ruhzSuAd8A/Tg2YiFa7vBY46d8Dx+wlNK30vawFsGxhNex/U8j8KeUdt4SJkD6G2o
+ 4A7FsE50RDcmshsb3o2L6MWH92AjejERYOe94PRocHrE7BH6iVFA+QApauAT63AlhA3Djmk0bK6Le4CK
+ nZKPBdBn5JJmFGD7HOww2gMdci/AumQOiti5BG4csmnoxafNRHvHSFcWsPZn5L2mUL+D/sfcFAJ02MUI
+ 3Vowe6DXPuBn5tB9HWWdxVLONPlbNEVnurwrTcabqejHxnowwcLJfhrsAG2+v66Sv6F6iKlOjKl+uoVx
+ ka1lmZNd8bXTOTfO5N46V+p8qc7lRtPdm023rxY52sYaqYVrciPVGAlqtMwTvDRTQbQlK8iS4Gck76V1
+ zF1P5p6B4n1Lkrc9654t/lGX98KwT2IgLzlCnBijHuQnCPTkezhTPe5RAnw4Pl700DBecCgrIpqflaMT
+ GEC9fvZ4+A1Ozg1R2TlC/yX86/ucDwFqT+6wR4JO/Hsx95//rd/5X8mbnyNG1xwalpWrN+i1W8T6TVLD
+ FrkR9llvwUYKyBjim7fIsIEa6fSygUThtRv42g1czTpsmMJVruEr14lVG+SKVWLZMq7oDbrgFfRSxEJx
+ KKSzcl6RM16QUp4S06Cr1hwtY0qU1Kp7P1zN0VZor6VmLdTQpgu5OAZRnkCQxnBQ5BMMnjlXYMXl2CB8
+ 8234/FN8gZVQfEqsaiNWtxFp2Ai1rISahhwVDTJfhGFx5Wk8Oao6gWtAF5mxlC05yjZcZZizOp/Q9bc1
+ T7x4Ov+WY5H7+TT/Mx7+Wvb+ZOMAOaNgaVj4NAs6bhQoDWG6of9xI18pI39pY39ZA/B1r6P6XkDwMV2g
+ 3PUg6PcABvg+DOjDNQCxjYG3lKGPjKGPrJGvHAgOjP1glEFAj4PirWZUcKticJtieAc6qgsb3YWL7MaF
+ d2LCOlHh3ajQbsWwHlRYNzrsU1QjAb2fCFdCZC8+oo8QAXNWJIhXCBtRjBxXjJ5UjJ2Ui5uUjXsgkzAt
+ B7tFJfugJc0QoU0c0iyOiKwWfQIdyMbkS7S3DpoP0CMdWoh5zyCS2WvnCdG5JFx59X/oF72AanVorIOu
+ fEmufyTMLWYFXFFwVpNyZincpqLuUORdKXLedJQ3C1aOyGEGapkXrUpcLjYGunfFhgwmR0+lJCykpjyN
+ iZmPiXydn7bUVPy2veJVZf7r/Kx3ZWU/1dYspiWXOdiEKLGiVVnp6uxcfRGUvsRY8n1NSW4n5O5pSTnr
+ yV7Xk719Eu9hz/U8y+kuurc0mRR5T3zJ7Pg1W3RqiHnADVXPC7wQZ9WUQGN/Z8FFG+nbF7Aet2jBnsLb
+ FzDuTpTwC7wEG3KuqVynPXr2Km3hMqPfgfos5+ZvO52/fZz578fR3V8qHm24tb01rF8T1G3Sajcp9VuU
+ hi1S4xaxCeaa0DJgk9q4QYFAvG6TXLsB/k1AEIcc4hq2YhWPtKmBus5laMwEqxCwOxGV8wqd8wqTDdu4
+ XhLTn5MSnxKTF4jQnSJtTBTfqOMapH7aSminoXZKSVObJebjWQwUlY2mqZA4hkyBGYdvwWZZ8tiWPL6l
+ QGQuVDIXKVuK1ayUtKyVdK1EOid5WjosFRGBz1RgUWToNGmaGMvToYmNmCqmTJVTLJVzfJWbmtoe5saR
+ DtYZV+2z751N9bH3CtRzCKKeDFI0CZWGmi3zECmTUFnjUHmTIBnjAGkjkJ80Yu0+iGA+qu959AS4+/3D
+ MNGUCAlpQHANwJVg6CNt5Atwy5v4K5wMUASZBaFM/OX3lY7YJsAGiyb5kFb5sHbFqE5sfA8pvpsc00OM
+ 6MKGQyTTBR6PjurFRPfho/oQssP7CRDSIOojhIN6CRDShA4oBg/Kho7IQ/QSNQn7phXipuTip2STZuTT
+ 5hQy5tAZj/GZT4jZ86Qc0FMitNSC9izQwQL2WBQ8wxS+QJQP7VlgVyi0+US6tUCrLUnDreeQW0RCl73o
+ BWnnAmcgA4OU8mJKXmIrX1JqHvFyKhh+N2VvqB68TpW6TpW5Q5N1oyjcpyrcY2IDdZVr7t98XpK91Vq/
+ 29b4U1fbL+NDvz4Y/62raz0m4ekd13cxMR9bGz72df2UnfX89t0dD9+PmVn/io9ptrUIZZFjhMw0NX62
+ vlK2hUaCtbKPOeOGnuJFdamLasev6CpcNSW6OvCDb2rmR54dawhJ9DZxO8t1OcPxdlK5bycKvKDlfVYU
+ cEnZ/5L4mgXuojnG44ooxOWEq6PA11GQeFE525ZWZKLQYoXutEK1W8i3nmdudsR8fD/18ZcXv/z67J//
+ G3zzLql/0b5pRbN+Q9CwzqzfoNZDIgV8fZvauElr2qQ2bVIk2RVa3TpVwjqxcpVQvoovByNfIZctU5CC
+ ROj88ZaQDYi/xmQvYgH0nBekzOeUlAVyyhNyykNG6ohabI2hi6+Og4WKrZqqlVjdgK+sQeerEnmaVLEe
+ S9mYq2TGEVpxOJZsjgWPb8EXmgnEZkIlM6GqhUjDSknXQqB1kq91gq0uJApoaA5ZgUmVo4twHC2qQI/G
+ g3gGTP20UOWimpqL0YkQW/M4J5vk22eSPc74+J84H8I4FaxoHnrULOywefgxs1DZk8EKxsHyJiFykIo5
+ GShjEiAFK/+GvrD4fxxGA29w96Pg8RIdQ+iXIC6hHBLwcib+IHmg3DQQkjko0yD0vvJhm+R+TGizXGir
+ fOge6L3EhB5ydDcBHD2iCxPViwXKI7pQENIA69G9QLkE9D5cVB8hEny9jwTohw0ohgzKhQ3LIzn1CVT0
+ uELMBJi6JHSZVUybVcyYxWc9JmZDk755aCKHz0a6JaKhyVb+c1zRM2SbRcFzdN5zVB5si36GlrQrgi5F
+ cDEgoEP0skc5HMB5OAldFCV2DsJVvCRXz7Fza2n+zjLXVA5cIh6+TD5+kyrrQgVTV7hHw0SeUJsK9//Y
+ 1faxq/N9UupKcNj/Sss/tnf8lpn77NS5XopgVtPg57tuv/gGL546209gTRBZb7UN5vUNKtnMGBw6jk1J
+ UOEk6whTTVXjrFVcDMkO6tJnNKRtxUfOays4n+a5X1CN8TIvT7tZn+tSmnTN+6q68xneDSvWZWO61znN
+ aydp10zJbg788HsGd+y5QfcMEgJOB9464WnPD7Vj5tgzq+woJQbSBVoHu85S+9y0dkezP75/+PG/b3/9
+ beV/H+fe/7fy0frdjmXjhhWl+jVW3RpCM/h33QalFoEeKKc0SUCvX6PWrJOq1wiV0CsUYZ0EoJcuU0uR
+ bluUoiVyzms8KOsVIeslZLdomc/pqc/IafP09IeCzBG92LKT99z0HE4q2ygrWYpUTETKBhyxPl1syFQx
+ 5Kgac5VN2HxzFseSxTXnCMy5IjO+khlfxZSnasZXM+Wqm3HUzfgaBmxVZQqfQ+Sy8FwuhqOC5+mQ+fpU
+ 7kmmyJytZCNQOa+qfldfP8DaKvr86bgbpxPdzwQEG12O4NiFoa0ijllEH7WIOm4ZIWsRLm8WLmseLmsR
+ CrUAcuYhslDxchKSj8FypkGyxv6IzUuiGhkjfxkYIbw5GQAuLmcaqGAWhLi4RBjzYIxlGNYiBLevYsQ2
+ bQAX3qIQ1qYQ1o6K6EBHd2NjIXSR2LnEy7HRfVg4COuCYxy09YJ5aiQQ30+I7iNE9xKj+ylR/RCvY8MG
+ 5RHQR1FRY4rRY/Ix4wjoiQ8UUx8ppj1CZTzCZT0mZAPrj/HZjzHZj1HZT1C5C5jC57hiYB3xdXQeCAEd
+ GikiNP8Beu5zJEyXdFRE0EdAf06Ewi+kmhfabkELtWfMojZmiLvCDbVDl0nHLpGPXwHWiVL3SLKeFFQ4
+ h9Jrb/ufqMhf/IIeaxr3ELkvjaz+dfXOO4szs/LU0T8fHv/u2AKWvEjiPTqCGvvL/omvf5j+9vDg/mNV
+ R49nKcimsEhxquwEPUG8iTjCUnxDB2clOmKhfMz9nEqCp3Woi6nXdb1o71PRftZ+dw1crmoYqUvZm1LO
+ nqSYKEnfPqPkfkHL55rOVRv2ZRuWy2U150tqgW4nwz1M75/lwUQ2y4lXepaRbSCdpnmw3oE+FGS2Opr2
+ 67vJX/75+tf/rX387flvH3vW/x0zsnq6aVG1fpVTu0qvXafUbFCqNwBrcHEaeHzjJnUP9No1sgR0AsTl
+ Fauk8mUokSDDOj/sTYGlfqgMzXtNzX5FyXpBy3rBTn/BSn1By3jKzXqglt1jHJtlce+WnoOBsrVIbCkU
+ nxSKTbhiE7aSMVPZkCmGyagJi2fO5JgzuaYsoRlHyYIHlCubwAXAUTblqJixVU6ylQ3YYngfUKIJlKgi
+ FSJfiygwoAiN6XwTmtCMqWwrVD+vpnNTx8jD1DLEwS7ymm2Mm51/qPHVGJ59JN46WsYqTsoathHFKljH
+ yJ2KkbKEbRYRsGVOHip1AXeofkHKvEIVTYOgBgYhHsY/ZB6iaB6CsghGI2MIxioUZx1OBNlEEq3C8fsq
+ hqzT+rERLYrhCOgKYZ2KEd2oqG50ZBdMSdHRfZiYfkw0xC092OheHHRpjOohRPbgYwaI0PYofoAU10eO
+ 7aPEDJCiB3GRA4oRQ3JRo0C5Ysy4QvyEYvykfNKUYtpDDCjrES57Fpczh8+ZhQN0zhwq7wk6/ym24Cm2
+ 8Bmu8Dm+8Dk2H1ooIgG6pDUXwjRxD+u9cGVvVvopdIGOuC8haicWvkEXQ9fPl9SKfk5UIPam5sEr1GNX
+ KDI3qHIwK/Wko0Np2HisXA2ZNCtUeUEXTO6XH/jLocn90nNSqPnDcs+//uHt59+9/tPfn/3pry+/+Obl
+ l9+++urvb7785uXn30x/9W3b9/sL5Y+lsYlxGpx4fUGciTjwpOCSBuYk/7Ch6Og9J/Xc2Jup4ZfuXdK6
+ dFpw+azIzBCvrnyUgv9aT0ORTf5WmbXfQhdnpU+6eV5dR+mYuuDwGSuWrQX91iX1YHfjGHejpDtaiY7s
+ fEd2oS01zVAhxUihzd9061H+L9sjv/z86tf/rH383+uPH4d/+jXz4daVtrc6jeuChg1Wwxa9bptRu82q
+ 3WLWbTDrN+l1G1TE6ZE2pbDTGfY4k6rWoJ8HBTpYIKCvUgqXaIWwdWiRWfyGkQ+fCPKanvWSk/6Cm/qc
+ m/ZInNajlVxmEhJqcuustr222FogMOPxjbhcxMLZIjOm2IQuMGbwjRkcUwbLjAn2LDDliMz5yic5ImO4
+ GHhKJ3liM67YmCXQZ/G0mFx1pkCdIdaii09AgE4VnqSDBBZsJWsBrK3qXNYwdNY39bKxCrxoGeZi5Rds
+ eD2ady6GZBenaJ0gZ5uIsotH28LeuTgZ6xhp2BtqG6NoE614KgKEsgxDW4ZhrMLBp4F44B7qeOEkcnwq
+ AnsqAncqHJF1BB74tosi2UbB7g1kBNBPJfeiw5sVwlohTJcNbpMO65QN75IL75CP7EbF9gPo6KgeVHQP
+ Jr6PGNtHAEcH1mP68HH9hIQBUsIA4E4G6GMHcVEA+qBc1Ihi3Dg6YQKbALW7k4rJD1CpM5iMR9ish/ic
+ R7jcR8iY/QiTM4fOm8fmLWDy5jEFC3ug4wB0CNCh3ZwkNAfKIVABX5fE67+z/snRIZh5DcXrxLzXirBm
+ VP2SVj0siI8k3tY7dIV2+DpZ5hZN3oWp6MvAJjOIZVhU04FDo3/7YfZP3z/+6vv5r75//OU3j7746/yX
+ f1n57Ovdz75e//zL5c+/2Nj31drnf1r76uvVz79a2ff108//2v33bwql9qcwMHFqrBhtzh7odnxpDco/
+ bA0BX41b5zWvOapePCMy1EWfc1A2MaaQyd9gcX/W1MQQCN8I2Af1tbFGJ0hWpkwy/q+qIiljfbz5SfIF
+ e4H3be2cSPvqaMesW5p5F8U5pzmRJxQ8xQfybmn+d7nx40+Tv354+uvPbyBS//hxaOd/mQ+2r7Yt6TZt
+ CJu3eI077MZddsMOp26bU7/Frt9k1kJ0Dlv6V6n160A8dPVA+m9BcznYJ162QilZpRctM4tgM+hbdulb
+ eglUn7+GOgtu9oI4c049pV8rqkgrIFTH/eaJG5baDmpiax7XjMsx4nJM2dxTHJEdR+UMX81BrGHHV7Fh
+ C62R3AukGkVmPJEJh2/M5ZvwBSY8vgmXb8zmGjBZOky2NpuvBUlGprIBQ2xIE5owhKYMgTlbCMnHM0pa
+ TuonbpwwcTE3dT9r4n/H1C9I/06M8HIc1SEBY5eCPpNKsE8mOySRYKsRbPWHDdGwZ/R0LMomCoS1jsTa
+ ROGgztE6EgrWYfPRXkGvItSv20Yjd9lG4+2iiadjSGdiKTDaxRDt48n2cZR9lSPWKX2K4c1yIa1ywW0y
+ Qa3HgzukQtqPh7ZLR/UoxPYrxvShIrsVIroUYvvwsX3EiE4IabBREN70YRIHcImDxIRBUsIQKX4IGzWs
+ GDkgHzWkkDCOTZrEJk5gkqYwiVOo5AfojEeErIe4nBl83kNC7iw+G3Cfw+U+wQLlwHr+Aq7gKR6UD14O
+ O6ahx8sn0AlAOTTR3YN7T59idKQJDAjplFvyjNT4XFQ/qJYSyXAzkb7FOX6HInOLLO1MlfWnodOouHpF
+ hYHvfpz/6m9Ln/996ctvlj//29Jnf3rz+ddLX/xp+7Ovft735e5nX21+/uXO519vf/n16pdfAPebn339
+ 6vO/Df3t24qjBzMpigkqjBhNVoyRMMhCbME7JiT81VQH63iKb2lIUeEfNjEg2lrzHR01beyUeUIZedSf
+ lVRRLN5RJuugtjbO+pRYUwPP4xzXUscYnoBHck5bMS5a03yuqGT7W9WGnKl0O5lwiumtIXtX+Uium8F/
+ lxp/We/5ZWvq47v5X39+9M//dLx4HzOwdr5lSad5DUDnN+9wW3Y5Ddvsuk2gHAEdCWBWIXShIwHMOoJ7
+ PRLVMGpWadCqpWyFUbbCKV3hlS2zkHgddpS/ZJcsKBXM6KR26oXmarp6K107J75hrX3DRO+cuoodn3dK
+ KDATCKw4/NNM0XUl3QATu4SzN2JOX/I3POVpYO5ldsrT7NRdw5MXNXTOIqUvamdUVGyVxNZ8oQVHYMjk
+ 6jC4J1gifZbYgCHSp/MNGXwTJt+MLbDgia1FqvbKWpe1DW4aGzrbGrhfNvTx0b8fp3onhXcpneSYiT+X
+ TXXKZF1IZzul0s+l4hxToJELyiEJtkhjTsciO4/OwOaMWJxdLAZ22cGOUkQxqDNxuDPxcB4LB7CBwyGB
+ Cjs5QI6JNKc0hlMqA0C3SumTC2uSDm6WCWiWDmg+FtR6NLT9WHgngC77u+QBdCR6gQC9AxPegYrqQccN
+ oBMG0fEDmLgBXOwANnYIFTOsCGPsMCp+FJM4jk0YRydOoBMn0SnT2PSH+OxpYu5DUt5DUs4jAoCeDbjP
+ YbMfYwH3/Hl83jwODD73KQamp5I20IiL72H9/w/63ty0ADpfPiNXPOfUPRY3P9CpbdBO8mHfN5R34cu6
+ MuRvQRUAWdqPgYDeoCA/+v2Pz77668rnf1n7/M8A8da+r9Y//2rji6/fffb1T/v+tPvZnzY+//PGvj+v
+ f/7n1T99vf7VX1a//NvTL7/p+8u3VYcP5pPRaWqsRD1+pLHQx1Jor4c31UXbWDANdDCayjKaajIn9HBm
+ ZqxTNkqnz2pr6FLkMH9ncqV0YcVFJCsQyIqV0ATCfl1t2gldqqEe8ZyD2M6CdsYEd0ZfwcNRUOBrU+Bq
+ 6n+SfEP5mAPnh/g7urtPq9+/bPrPyuDHdw9//Wly50PD9HJAxxv75hW91jVR6yavdZvbvMNp2GJBGNO0
+ xWnaZjduMpvWGQ3A+jq1AShfgyYWIKCfAZ+oUb3KqFrjVqzyy9ZY0J+ocpFV/Uy1Ysogp9U4NFn3lrPo
+ jCXbTld0WV/3qp6Oo7KSLZ9jIxSc4otsGPxzFKGHyKDkvMtkcPqgV0yp/a2yc3f6A2JGwpOa3AOzLtxI
+ drySfuV26uWb0Q5Ofma2rgaWN3RNTitrm/PUTrKUDGl8AxpPH8IeltCULTzJEVgKlOyUNM6r617Q0b1i
+ rHvnjIGni5FPtK5nhurtXMblPMKFfJJTDu1yFvNKFuNSFhFaK17MxDql4c4l4/c2TSObSpPwjsk4x2Ss
+ ZNwT4VwKAbacOibBAflcChUZkylOqTTYjQqdM/ZVjlqk9MuGNh0PapIKaDoe0HQksPlwSNvRiC7pyG6Z
+ 8E4pUGS3bHinPEQvMb2EyE5IsWNi+tBx/YpxAyAUKGZAMXpIIWZIPm5QMRY0oAAHCSOoRIhhxhWTJtGp
+ M7jsaVLeDDn3ESXnESkbDP4RPmsOn/UE0i/4vMcEwD0Hpqfz6OwFNLSBzoauRpKIRdIT/f/L0fc+QAsW
+ jIpekMsXGHWzStX9asUVKumRvEAH3F1lqXssGVem7E2a1E1YMGKhUxm4OkX5wR9/fPz1X15/+ae3X3wJ
+ iG9+9tXaZ18uf/blBkL53zbB6b/45s3n37z98u9v//S3V1/97emf/j7z9x9av/2+6OD+XCImQ4WdoMsP
+ NuL62avdv6F79ZqGmSVVU0PG3IRsZ8szNqFo6eBO2ala22mctFTiCBWk5f/GYEsRiT+IRCgU+gcMZr+p
+ icDhtMalcxq25nR7S9ql0+zzluQrlpTQa5qJt/TvGeLPCg9bcr+PcTV5t9j+01LP+zfdPy/1/bTZu7JZ
+ Pv7aq3PRoXPVsH1d3L7Ba93iNG+xGjcBdHbTFrd5m928xWzaYALoiNapjev0xjUQo3GN2bTGbFxn1q5x
+ gPWydWYlHC+Jauf085vNIuMN7t1QtTVkGvHolnyVC1oGl7R0HZTEdgKOLdg5U2hH4l1E89wIqgWGFybv
+ RPVf9CvVP9d+5t5OUsWvxR1PgtKaLnv2uIa9Sq9Yya+fisxsuBdUeS+wwiMk+dr9O0Y21hw1YwrXmMoz
+ pPGMGEIjupIhQ2AKpQFQLCDWPi1SP62ieslEz+3GyYAIE98MTbci7t0K6o0K0rUSCrRcvFVEu1FIvJYP
+ /eigZRcJOr1czIBmL6RzqYTzaQSndPyFdDyMTum482k46JBxMYN0ARoewV1p5AvpSFsY0MUM2kWkRQwJ
+ 0oumib1SwQ1HA+qP+jccDWw6GtB0OLj5cFjnsfBuqZCOowA9uHtYh2xkFzqmBx/dDTE6JqpPMbZfIW5I
+ MW5QIW5AIWZQIWpQPnJANnpQIXpAIbJPJqpPFohPGEUnjCpCDJPygJA1Tc6ZoeQ+pGY/ImfPkrJmiZkA
+ OiRh5om5SB4GmzWHyXqCzgLQod5rHp31BAsfWYHE6OD0kHGH9v5P4UNaAHH43BwyfMBI+VN67SNhZac4
+ I4MVdp/gZiN7RXjoEvHALfLRe3Spmwzpm2x5Tx46kYMvx8h3Htw/8dc/z//5q+dffvnmM9BXL7/60/O/
+ /PXln/725otvXn357bO//vD6B6nXB6Qf//3H6b99//i43FMqvQuPz5I9nohVSBTSYrU54ebi0GsGrs76
+ TtfUDSzIxhBtX1C7cFHT6CRd34R+9aaFiblYU4dhbq0uh/qHlOyfFdF/JZMPMJnSp6w0zE2Vrl8+ee/G
+ yfO2MG0V3r6seuOc0NGMdNWUHHnzRJCTyjUjvI3KkfQwx1/+Nfuv3cnV561PJopePC5+vZz3YNF/4O3l
+ 3nXTjhVh5wavfYvdssFoBjtfZzVtgoBywJrZuEprXKM1roPgXmbzOlNykoJ0cVln1WxwKtc5NSvC2mfa
+ RV1G0ckGztdUzxnyjThUTTzNhC46p6YHoch5VRV7pJRFdIaldI4guCzNvnuME4nTLOBZ5tBNUjDa1SK7
+ 59ci1zwz+qzd8lTtWu1cd5JqfivpfRqY3XTBu88j7mVW7URSUcI5ZyeBriVFcBIBnWNAExhQlfWpIiOW
+ EGrFLDmq5gwxRO32mtq3zxr5+xoHJOr4FIk8ahl3q/G3K/F3yknOZeRbJcSbxaRbRdTreZSrudDeiH4l
+ h3olm3Ilm4Q0rMuBUXKQTbyaQ76eT72WS72aS7maQ72aQ5eIcT2PdaOAAWf2Fffpx7YdDKo7FgBqPAaU
+ BzYfCm45HNC2P6jjUFDH4aD2Y6EdMqEdcpBKj+0lQKQe3asg4Vgufkg+YVg+fkguFhpJDypEDsjBZBQU
+ PiAb2S8X0Y9wHz+GSX5ATHtIznxEzZ1j5s8yc2dpObPUnFlKNrQGmCNkPgZBYyNM2hPoVofKeAKmjs2B
+ SOYJvmAeOqGR8+cJhQu4oqeEwqdQcMcoekovnCeUL5Ab50R1nao5iZyw6xgXYxknrpQd+qgD6sAN0gE3
+ 2vHb9OPX2dJuQnS0kJRPxjTJSI8dP/ZSXv7FwaPP/vLDk28PL+BJL5XEzynMme+Pjv39x2cE6k9GVu/M
+ bGeFqoMs3vJp+397uj92csji0iLICnHKsIGaFWkpjr5tcvOKhrkt+6Qd395J+8pNk7NXdTWMKCIt/C03
+ B11DPp0j5+x6VkuPgcF/iyH+DYX/i72j5vXrZkyG9I0rloHeTrcuG9y6rO1yXcv1qsYVW85ZQ4L/VZ30
+ gNPJPqfi3E3bK/3/+5+Zd9sTa4t9M2O5C7PZm9vlz9YThpddepZt+lZU+9b5PZvMrnVaJ7RSWoV2eXSk
+ yxw03V1ntW0wkP5EiOCA3bLOal6lNq1B70Vc0xqrcUXUvKTW8vREVZ95fLKhxx21i6aCU0K+HpWmSiAa
+ cBkO2spOmmrnVJQdhKKzAuXzHJWLeMFNKfb9/ayg79lx+wVJ3/OSv+XmHFapp1i0sO0KUSeSj6sU08xm
+ zvgt3orvMr2TybWoM7/51D9jyjslxezyVaaGNVl4EqIXBt+AzjeiCA0oXCMm7yRXpM8Q6JE5JkyBvZqW
+ s8WJkLtGkZEnQ7N1/Mr57jUk+GQ5tzqSaw3NtZp+r4LiXEa5VUQCd4eGdbeKabcLqXcKabcKocsuBVo0
+ wmUA3o+omHq7hHobRuSA7lzKdC5l3ynh3C1jQ4+kfUU9+jGtBwOqjvjXgpcfDWg+FNhyKLj1cGD7Qf/2
+ gwHth4Pajga1HQ/rQEIXJNMySIgfRsUC3AMy8UOySaMKCcNyMUMyYOdAdlifdGifdDg4er8cKH4YnTRB
+ SJ4ipc1QsmcZBU84hU84+Y9ZEtYp2XMI6Olz8PktmIwniqlz8qmP5dOfyIOp5y9gC+ZxBfOE/AUi4F4w
+ jy9YgNkq1JSSixZo5U9p9fPsukF+dgbTx0nuls7hi9yDp7EHreWO2CsevU485sqQBdCv0Y85M+WC2fg0
+ OqmKRl3Q1//v5Ss7p07PctUeKult33H7X0Lyvzz859QMhxmizYs3Pmbmfyyp2I2KWfIL+KW48GNXy7uc
+ pEpboygxNVaNBh0Ewi1F/hc1HWzYBmbk05e1brjZ3HY/fcnV3Payrv4p0W1PJ0t7XRUdWmDk7Ys3TNR1
+ cWzBIRb/YFDohYsX9Gjkw8G+133uO9pZCK9f0Az0tIzyt/O+pX/3vIr7BbW0oDOVqbfaSrzmRtN/+e/4
+ Pz9Mba0OPX5Q9GQ6c3O94s1OzvR60MjG5ZFNw75VUe8Gqw/aMq5Re9bones0SYd1Sscmq2OL3grN1Deh
+ tzqzZZ2DaIMOe6KhsWjLsqBzSbsNGhJ1a6Vn6nncUr1ozrPRYJrwGHockqEy+bSR4KKZ+nldNQdl5dMC
+ 4Rm+2JGtfAEvuC3F8fqRFfotJ/4bfsrfeGl/4Wb9hZv7V37uXwVZf+Wn/ZWb+Z2oGm1cT7bMk9GJPihK
+ xxvUaJ4v0DjrT9e7hBHaEHimNL4hEyJ1vjGVb0SFGIZnxOLrUDlaJIYxU+Cooe1mpR99yyQ+2Dw6VT+8
+ TCWwnuHbSPRqIns00t0bmG610CidcqeMdKeMeK+Cerec5lxGvVvGuFfOdC6jQ9NdGPf6NkJ3UrgLRmg5
+ fa+C5lrFcaviulZyXKtB3H0lfUax7Uf9q4/61R72azjo37Q/sGl/cPOBoLZDfi0HA1uPAOWQignrkIvu
+ QcX1Y2CXRuKoYvyITOzg8YQRmaQxoFkmevB49LBs1KAsIB7WKx3eIx3ZKwuxTcooPm2SnDJBSp+kZD2g
+ 5j9iFs6xCsDUJQGMJHqBj72FShj59Dm5tFmZtDnpjHmZrCdyeQuKBQvoQknmseApCQR5mGxYMX2mWLRA
+ qpnn1o7y8goIAXekr2gevMA47ID/0UL676ZH/2ErdeQKVvY2TeYGODr1+G2SlC8VE0mnlGprrgT6f6yt
+ /a2yajc+dScp47f2ro8PZj52D36IzVj2CP5PfunHicmPc3Mfxyd+7ev5daT/p46Gx4mhBQ4no7V5sRqM
+ 6BOsEAvBNRO6yQm0ngnJ4doJt4AL93yd7vifveHtcMfvgn+s6xWXM2evmQTH3LnuYm5oQdeGOastPyPT
+ /cZ1E2MDTlKcp4eL/eVzuq7OpjFhTplJt6L9T4e4nrx3Tux5TS3ez7Imx/nJZNr/fhl+vz20s9L//FH5
+ k8mspZelWx+qXn5Im1y/N7FtPbih0b3O6tkk926Q+tZoPev0jg1a+wa1ewv6hyGtd1u3aK0brOY1AJ3d
+ tsFq3aQ3rzE73mh2z5+saddNSxb7OQsvm/OtVZkGXMIJMeaUCdH5lqrPfeO7Fw3PG2raKinZQaG5WOkM
+ R+yE598+yvb5jhX1DT/pr4LUPwvSvuJlfc4p2Mcu+oxb8Dkv/0te3he83K/5mV/z0r7iJP6ZE/+tMP6A
+ SshBoesR7lUFvh2eZ0rhGTKQMN2Ewjeh8owZXH06T4vE1ibQYXn1mqFx4BmL+GtmqX4WGSknU0q0ous4
+ Ya3UgHaqZzvNvYUGH31xv57iWk12raFKPgCDIfkMDBiZrlXQYJrhVoW04YVmvG5VDNcquIvmVk2DPuue
+ 9VzPBp5HLYxcz3revpIBk5i2Y/7VR/zqDvs27A9o+iGw8fvg5h+Dmg/CrDSo5Xhwi3Rwi0xIK0xM5WL7
+ UYnD6IQx2biR43HDx+KGjsYNHYuH4xHphAmFWCh0QYxcNrJHJqZXPmkAkzFOzpth5k4zciZp2VOU3AfU
+ nAeU3GmI1CnZj4hZyEcoolJn5dMfyWTOIcp6IpM9L4uMEtbz51EFsHT6lJj3lAIfWJz+VC5rQa58nlwz
+ zs8poYa6St02/OEa/9hlguxZhaMWx78zPvh388M/OKKO36TL3mHK3aEr3CXLeVDRARxqroXJfHzUf7rb
+ Pk4MfBzu+Tjc93Fu8n9Ppv4zPPBTRcVqatpifu5Cfc1CQ/2LhvqFipKH6SmdPm55DhZxhurQBSBBi5lg
+ xA024V7Uwaty9/OVjp2/aXTxjpWZg7a6GY+piedqUyydjFwCbwTFuwXG3LrkbGJgTrO04dxzs0hKdQ2P
+ vBUZ4ZKfG52RHlSQH5aT7VNU4NtUF1WQfi/K3/rGWbbTKaLrFVG4p9Fge8j//ju09qZ9823fq7maJ+PZ
+ zx7mbe/Urf4z/9GGz+TmuaENg951HrRl7N8kDGyQ+9eBdVrXJoBOhhbsEtDhswOYwHr7BqNzk9G2QW9d
+ EbTPG9R1mGXE6/hfF1w349qpsU8KyAZKGFsboneQOKfcKivf3tfd3MlM10ZVxU4ZEiPKpzliCF3uHeEG
+ /IMb+2dB0pf89C+FmV8K8j4XlH4uKvtMWPa5qPRzYclnwoLP+NmfcTM/52R9JUz9nJ/wOTv8TyzP79k3
+ ZLj2WL45GDmTawSgk3gwNz3J5BvS+bpkrh6RZSNSdbU6FXvRPv2aTa6ndVG8VUGhfka9OKaDGdRL9e6j
+ +nTT4MOMvJpI8KlGHo0gqnsD/X497X4dHT4Vw70WPvsFPgcG+YQM9zq6Rx1y7FEPIx0+FgZ6rvu28Hyb
+ eYEdAv82AYB+Mqb9OIDuX3/It/HHwOYfglu+C275Ibj1UAjk1EEt0mFtciFtshCpQ3YlYRSVMCaTMC6V
+ NCaVOAqUH00ck06eUoCFodhRCMpl4yDJ2C+fOIDNHqcWzLCLZ7mls/zSOV7RI2buFDV7kpzzgJzzkJQ9
+ S4DZZ/qcQuojmYyH0llA+Zxs9hOZ3HnZ7MfAulyuxNRBeQv43AXYVArbMtAF8+jyCWpBNT3MS97N7NhN
+ 0SFnltx1nMI5eTlLqUOGR/5mcvyv54iH7jDlnanyt4hy9+io+2yMt4gUbaxWcf38cLjvbEbEQnb005zY
+ ucL4yazIoUivHs+bTVCge/VMzrXz2dec8q9dyHFySLU2izPQCtcQRqrzYlTZ8er0WB16pAnPWZ+mQvmG
+ iP8zVyyroksxtlHH8eSPEg8cIRxUZMkpGwjCkn0iUj2uuViaWvPOXtTyDjqXkuWdlO7n6GRuYKxqdFLV
+ /ryRT+ClmLjbpcWBrQ1RiVGOV88ynWyJN84x7l7hD3aE//rv8fdrwz9vTL1727v4qOzJRObyYtHm+7wX
+ OyFjKxcGlo0GtsRDu9CXFDe0iR9cJ/Wtk3vWyd2b0HYdQIfG0ERoDN2+QemGXmKbjJ4Nbttrtao+g6RE
+ A59rGjdOCu01OFZqNCs93HlHim+USmGrQfPkqaIGm4AAswtWOrYaKnYqynZiZXum6DKG73qEF/QNN/Zr
+ QdLn/IwvhNlfifO+EJV8ISr7QglAL/tcDGPx56KiL0RwARR8Kcr9jJe5j534FTfoO+4dKZ4jTmBJExgz
+ eSZ0rgmJa0LlQDbdhAF1AXxDKu+0WM3dyirxyrnCO+dLvBzKo2wrCkxz6tVjOjjB/VT/EVrgAN2vleLT
+ TPRtJQPuno0kD+RD6ijuDfAJXmDb4Pd0ryamRwMNPtDLu5nu28L0bWX5trH9/k+coG5+UBd/X/GAWUyH
+ dEDtMb+6A951/who+geAHtr6Y2j70VBk5UgqpE0uvFMxpF0uuEMappuxo/Ipk/LpM6i0BwppU/KpU3Jp
+ Uwqp06iESYXYUZnYYbmEYVTiICZthJQ3ySp7JKiYFZU9FFQ+EVY94RfOMHKmKDkPSLkzpJw5AhQCZD6S
+ zZyVzXwknfVQKvuRVPacVO6cdM5j6ZwnsjkL8rlPFfLnFSG/nj9Pyn9KKlwgls/QSproMaEoV+vDd5Tk
+ nNkyd6hSV1DS9nIypjIHDBT/bMv55rrqobscqdsU6ZtEmbsMlAsH5SZAeSkRwnUEySc1Mmx0s+30888a
+ 5547mXnWOMVaO8FEJUZfGKrNCtVghWlyIrWFYWq8ICVmkJgeqgL9XtiRyowYZXK0KilSnx18SuyggxMy
+ f6TS99uf17vt4YjhyCowZGVpsnI0WVmS1C2vyzGpvm7e5x0v6V+/axkef7ewKv6Gi8P+49+gCDJEJkpV
+ n6usSzlpwb1yVTcz7XZ22lX3O6pXz1JtT8oEe5/YXGr477vRf21Nf/z3SyjtWpmvnhlKer2Qtb2TtfQu
+ fGLpyuCS6eCWMnQkHdnFjGxihtbx/esEpGGY5OMGoCMXfB4B8sEb68S+DebwOn/gjWrdiGZShobLNeWL
+ JqLTQLk61cGCfPceNSpDubTXuHX+RNu8XkmHaUCwsZOV2mlNpdMqamcEqueZSlcxvLuHWd5/YYR/zYn/
+ kyDxS17ql/zsLwWFXwhKvwTEBRVfiCo+E5WDvhQXfQHWzsvbx8nex0n6kh38D85dGcFZgtACCl0YvJN0
+ vilNYM4UQG7RkqdiDKUvDCFsMPWzsU65er7Q+UK5l1NNlGNVrlVurU5CtyBsiB48Rg0Zood0UQLbSUHt
+ JP9WENkPPtCr9dMnM/o007ybQFTfFqpfKw3k38aAT0QK7uKGdCMK7mIHdbLD+rmhvdx9hQMnozuPB9Qe
+ 8as/4NvwXVDrP0JavwdHD2k/Etp5PKhNOqhNLrhdIahdNgiSjP0y0cNyKZPojBlc2gN0+gNFYD1tShFA
+ jx+XixmViRlCQE8bI2ZP0AumuOWPxBUPxSVTvLIZbuVjXskjdt4DGoCe84iIVLw8ks96KAOIZz48mjlz
+ OOvh4ZzZI3lzR3OfHM+dl8p6IpU5L53/FIle8h9jCmbxZdP0qn5+djrd3+nYbbUDd9goFyb6DknqouIx
+ K7nDBqg/W4q+9rmCCr1CuKd29Br14A2y1E2SrDND3oUr58aV9+bhAoSUQBV6kCozWJ0drsOP0OWH6/BC
+ NdkBKjQ/JUKwEj5EiRCiTAlWpgRCDwxlSog6I0iDDmeQrIsKOVybGmGj5HNO44Kd0P6s6sXrhq7+54lC
+ +e+k/3wcf+h7mW+/OfpXC3ujqEQf/6Dr125aePg6ZRYEx6X6ssX4/VLfMkVkRYqMQJtK5B6/cM3gyvUT
+ erqyVuZYQ50jBpo/nFD7tqLozsf/Tf53d/S/7x799q/nH3+eXZ6vmR5IWlzI2drKXP8pemH73tSm7TCA
+ vk0a3cWObaGH17FDG7jBTTx0v4DO0V3IdmmkGR1MVYdXlYYXtdtHdXOztf3uKV+2FNrpsKx0iGetia5e
+ 9MQiXvWYVucL/c43wtYX4pIuo6AQw4unlOy1BA6w51+oeZGjehnLu3WMef87RsC3rPBvuWF/YUZ9zUz6
+ EzvzSw5E50Vf8Eu/FFR8KSz/QljylbDgS17WF+z0L1hpn7Pi/8QI+pFzR17gQBZYQgUYW2jGEkKtixVP
+ 2VasbivWtOFr2As07ugahp2xS710Lu/G+fL7F+rCnaoybApq9FN7VGJHODHj9KhhanQfLaKHEtZNDe0C
+ 0YI7qP5tEMHTAiUKaKUFwudndNCD2uEuRlgPO6qfG9HHj+jjRfbxIvrgGD5bhh81wN9X0G8c1XE0oA5m
+ ogcCW/aHdx0I7zwQ3PpjcPuhkM7jge1SAa2ygW3ygeDosH40CDQrJE9i06eJaZOY1AnFlEmFzBlM5iwu
+ cVIhekQ6alAGQpf0cVLOBLPwAb90WlQ+LSqb5JY+4JTP8Upm2QUPAXRi1gw25xE6d1Y+55FM9syRrIcH
+ sh7uz5r5IefRj7lzB/KfHsl7djRz/nDG/NFcBHRU0RNU+SyxapSTV8qKvI/2MD7szD98B9yagblNkHdC
+ yVig95txvrrrdKgkW60wUcPjlOwl7vcwGb1FlL9Nlr3HkoMIxxXZKkrxUaaBfIFmdUaoGogZog59i6jQ
+ BcBfieiLdLygBKhQA9WoAaoUP2Wijwo+QIkQrUyNU2dG6DJCzPlBl7TdbujdczMzPyM0tOEZ2SpRxHKK
+ tCNS+APHsQcNzdX8Am/5+lx1dz8bFnEzuyD46i1LWdR3f//hc3ncEXni4WPYb4Ta2KzCwMwcT3t7trmp
+ 4ilTBauTUvduCF4tlH787cE/t/r+uT3167+effz3439u9r9+VPRkInH5Tcr7/yav/tf3yQeHkU3loW3y
+ 2C52fBsztoEd3cQMb2MGdnDQ7rR7G4v0a98kDa7yR9/odz0wKi06EXtP19Va7cIJvt0J0vkzBK8AdmqF
+ cvWkSveiWteSoGOF2LHIL+02DA4xuGItdNTknFPSPC/Uu8BVuYDjXpNh3pfmRuG0Ugknoo6KQv7BiP2O
+ m/6jUtZ34uy/C7K+5ub9hZ//F0Hm15yMP7FT/8JJ/Asj8U/06L/R/Q9xbqB4p6lCS47YggMpc7EVX8Va
+ pALLog5qeufVDK9rmHiftI5zdMy+dqns7vXq+1erA85VpFgXVBmld6snD/OSRxkJQ7T4AXpsPz2qlxGJ
+ iBnWTQ/uBDHgs77gs5BCOuAyYIV3M8O6GFH97OhP4kf18qP6+NH93OgBbjzyaWECcHSTmK7jIU0Qjh8O
+ bPkxtPNAeMeB0Lb9IV1HQnukg7tkA8HOO1AhsMmoTyFiAGhGJU3g0x6Q0iZwKePo1El09kNC5iwBidHH
+ IEYH0FGpE6TcKVbJQ1HptLhyRlT9UFT5SFg2yy+aZRXOMvMeUrKmMJkP5LNmpHNmjuc8PJQ9+332o28R
+ zf4jb25/wbODuc8PZC78mLlwOHNeLmdOvuwJtvoBtbiBHh2M8bKVvq9y/D5b1pmucIescBMv74g+Zkn5
+ xtHkbwlx+K5+w5pqbb+r8peV/nGTfhz2j94hKdxlKN6gyd1ioO9xSO5CgrsID+1coLNFmDI1WIkSICYh
+ jS5EJF9lKrQX9VOi+YkpkqYXRG8h1k8ZF6JKiFKhxGqxIg3ZAebc8NtGAV6W3iH2Z2+dIIsOm9qLnX0c
+ bro7XLhprWss4omwV66a+3g5Bfo6xUTfjE9wsbASaenQOAK0PHa/NOZbEvfwbQ+rnBJ/ZxfDgACLwrzr
+ mclnwv1ODHSG/PqvoY//Gvmw0fVudfB/H+Y+/ufxx38/WJ6v7Kr3mpsO+9d/M1f/6TO3dWZqW2N8lzG2
+ gxvfwYxto8e2FIe3FQclPXv7oV/SDnFwkzH8VrVn2rC8Rj/eXzvo9AkXE80rJtyb55mhkaL8Bs3GGS2A
+ u3eN3b1G6VrDdL3hVvaeCAvTuWrLO6fJdRJrOwlPnOcrnSNxL2O4EaKTbWfvj1wOKFQ/HaaolEEzaNG5
+ 2KBknymvGfMdO+OIShnGIF9eO/5HXtg3jIjvmTHfs0P+QXc/zLyC4doyRJZcJZAFV/mUUBX23dkpazlp
+ Gl7VNnYzsgi1PZPodC77xqUS56vl9y5VeJ+rSLArLjPL6dTOGOKnjTCSB6lJQ4zEYWb8IDNukBEzyIzu
+ Z0b1siL62FGgXmZEDz28ixbeTYvuY8UMsGMHWbEDzOg+TkyvIK5fEDfAjxvgJY0JEkf4+4oGT8b1SgPo
+ Ia1HwdGD234M7TgQ1n4oFOjvlgrqlAuEAL0bFd6HDh9Ah/ejogawcSOE5AlIGpJTJwhpE4QMWA+axidP
+ YePH5WOHZBJH0WkTkGzhFj8UlUwD4kp1j5Vr58Rlj7gFD1l5DxlZU8S0CVTqpHTm9NGshwezH32fNff3
+ zCd/yZn/W/bsd3mPD+fM789+/l3mi+8ynx/OXpAtnMPVznDLmzlp0QT/s8c8NaV8eRg/FsmVoXCLLHMF
+ f/g08e/2Wn/29jxe1sbtWVCv7WNFeErdUv/HLcqx23i5mxRAHB6pcIeCdmHiXdlYFzbmPhvryyUGCamB
+ fLI/nxwIferENB8x1UeA3ITzEOR4C3BeQmywKjlMlRSqig/XpoUYcoLtNSI9bdy9re1v6lpd0dS34Yi1
+ FJ2uGoRF3Y2OvW93WofDlL54Ts/H/XSgr0Nk2CUfrzPqqlgXlzMFhVFRsS7uPmdcvazOXVG2O0dzvqea
+ l32pofJebpJNdf6lD2t1H/83/tvPE+9X+j6sDvz6fvrjP2H7xaN3y629Td7Tw/4/76SvbvnOr519uKUz
+ scMY3sUMv1MY2ZUb3ZEZ2ZUeeCcHzU0Ht/Bj2/TJNcHAQ9XKKs24YA1vJ1UXC1XnUwLvm/z4RJXKLt2u
+ Z9p9q4LudfjkI+Sjv7pW8b1v+XWDarFxStftOWc1RU7KJ86LdB0EMB8V3ORrFF+8+yaj/H1xy+D98AyT
+ s82X7q/GFz0PSC3Rc4giqJdrnH54OWDY/n46Gzb38CMImil0w1C06m1pzgU835atYs5TsRSoQU2vpVAF
+ tqJCGcwFzRM3dHW8LY0iz1nFOFnFOVklXbLOvG6f7+pYGHa6tMCirMMgd1gpdZSZMEJOHCElj1GSxmiJ
+ w9T4QVr8ID1hkJUwzI4fBLLB46nh3WSIbaL76bFD8Hg2KH6QkzDATxriJ40IQKnjopRR4b6SYbPEfrlQ
+ yCG2Hg1uO4hQ3nkgqudoeO/x0G6p4A6YgyKgh/WiwvpQoT2ocNiHMUhIHCXDj08ZI6WNE9OmCKlTeIhn
+ 4kZhiVQWQE+BGH2SWTAtKJ4WQYxe80hYNcsvmeUWPEJAT5vAJo7Kpk1JZT08kvXoR4A7a+7b7Cd/zVv4
+ W87jH3IeH86a/xFAz3nxY86zYyVPCfWPBTXdStnx9IhLir76x7zF0j5sBW822oUte4Mu5Ujaf07pH+43
+ jhTWMboWVHoWRY3jzJQoRRf9H26QDl3HHr9Glr5Olb1NUnAmoe/S0XeZaFcW1pWJ9eGRg0T0ICE9AMGa
+ 5i+m+4ppvkIqwr2IEiim+oqIPkJcqDIpXJUUooYP1qb6GXIjr50M83O87mKmZ8fRs+NanFfhqcmINRTO
+ OGpeuWpibMQ5aci+4qR39+bJAO8zEUGO7nfNdNWxVy8YlJdEFhcFJSZev+Osa2wqd/E6s6jkRkeLd3Xx
+ lcpcx9nR8J/Wan7799BvP4//a33kP1tj/4PddP+a/fi/Jz9vdPU1eT8aCfzl54Kf/h33Zuf6ow3dyV36
+ 2AfU0AfZoV3pYURSA7vS/ZvyIwD6BmP8laC5i58YzfS4zLlhwb55iul7h5edqdk6qD/wQntoTTy4w4SW
+ 013Qa3cD3bVG7FsStkxoJKer3D7PO6sldlLWO6ekc1osPisUu50wKrvr/jS7eKOqaSYxq9Mv/El60X9b
+ h95Xdgx6RFTZ35zyiv5fWftWRmXDOeckPesKhzt9t4KKza/eZ2mdp4ptuAjl1kLIwcKmOxUbsaa9ig5U
+ 1Dgb6QY5mMRet4q6bBRgr+VroxVibxB96WSKp2le+snSdsO8IdX0MW7KOB3BbJwGSh6jQzCTPMpMHWen
+ jnOSR1mJw3RAP26AGjdASxhiJo7ABcCKHwL7ZyUOciD4SR0TpI4J08eFcACObp7QrxDRLg85lojuY5E9
+ hyN6DsYPHo8ZlInolQkB0JGZqGIgPAB2SfcgXV+iYdfFINTlEpNHyRlTlPRJYjJUn4/KxwxJxw7JQi1X
+ 0gguY4xW9FBY8UStdkG1bl6p+rGw6BErZ5qWOUWExdT44eMpE0cyZw5mzEDQ8o/sx99lP/l77pNvsud+
+ yHp8MOfZgdznP+Q/O1z2DFs3C5SL89J4MdeIQYYofyV5b66MO+PofajF5R6/CEtF3B9czklnF9G6F0R9
+ K9zuZW7bnCAvm+hlfeA67YeruCPXKdI3qDK3SfK3CQq3KQrOTBSY+j0m2oNDCBDRA8V0PwHFh0/yEVD8
+ hTQ/IfBN9oMARkQJUCIHignhyqQodWqwBtFXhxZkoxF577TbPdsr96zOuZraXdezuaRhcpqrb8k4e0X7
+ 2m3TS1cMoSjX1kp4xUnX9Y6ph7Ox5x1T56uGNy7oet41c7IXWJpiLM0VLl2iJyRZl5Rey844Ex2s29vs
+ +nIuaett6W//6v/t35O//fTw44fp394/QED/7+x7cPRG74WJiI//rfrvr1kr7+8+3jZ48I429pPc8M/H
+ hz9IDb+XBUcf3j46uCE9vk2cWucMPOYWlZN83eRv2ZFu2tC979KzMlW6hk9MLUOfRx7Sq/onOrSohqZ5
+ 8AGlnWsEcPT2aZ30XLW7l3jndAUXVLUdVbRsxCJ7ZfFNXd1gG7vsm3cqPXwqvf2qfQM6o5MmM4sfZJX2
+ hCe1+IQ/TM5/39SzVds+HJFU7xk4kZq7WFLXGxAbYnz6PF/DGuJyoboNX82MIzaFzjBizbNq2ld0dFzM
+ T4RdNk2+b5Xobhx4Te2uDfe2OdfdTinojnpignZh84m8QdXMEU7GBDNjkpk+AeYIYqZNsEAZkxxQ+gQ7
+ bZyVOsZMGWWkjDLTxrlpY5yUEU7SECthgJU0wE4e5KQMc1OGeJkTgoxxwb6CPtO4bvnIDlR4l2xE3/Ho
+ /iOxA0eShqUSRuSSRjAJg1CDToiEDl6tChCsB3YA6/joPmLcADlhiJwyRsmYoqWOExKQAl3Z2CHpOPgq
+ cPQRfM4ko2pepeW1XueSQfuiduNz1aI5eIpwbaDih6Uh+548cTD9wY+ZD78D0DNnv8t+/G3e/Hc5T/Zn
+ LxzKfnYgf+FA2YJ89QN6VRM/E7z8Bt7XCO0P80I+wY+L8WHLe3DlrrGPOnJ+uGqyPyGW2Dwh7Fliwbyq
+ c4Xe80pUWccMvX7stvDHK8RDV8hHrxCP3STI3iJA7a4cgH4XkeJ9Lg5aLgLW3gIiNGr0EZDA2n1FFC8R
+ yQu8XESCrEuwEilSmQqt0321yf7mwmyfS1E+V+1O6xrbadhcNTx7y+iah4VLiL2z/+krLqbnr524dddS
+ W5vAYhw6oU00M6CcMqZcP6eek+wSFXguKtghyNfishP7khMtOso0IdH8vhs/LtJwoMPj6XT0aLfvyqui
+ j7+M/PbLzP/ezfxvZ+ojhC4/P/z470fvl9r6G/2m+wL/tVX408/pK7v3F3ZMpnapw++ODL7/cfinI8Mf
+ ZAd3jw/tHBzcPDq6iZraYPZBTVE52t9LyuM6IcCDmV3A7xjRnHyjPr3LGnuPGfsZM/4v2sh7zsB7Vtc7
+ UtcGqXeZ1zKlnpat5HKFfU6P46QKDRY1bJUhQ6Jkrwy7PNUua2pf19N3NjF1tbD0Ou0QfOFa9HXnmCu3
+ Yy/fyr7rWR0U3hod3xQZ2xAd05eVNZ5f0BASEWzj6CjWtBao2IjUrflq5lxlc4GqnZL6eXWtmyd0PG1O
+ RNw0Tg8yz0u0SI028HcROTsy7jjQ799mRcaJcps0ioaVCyd4eVOc7GlO1gN25iTAwwLiQYA+HGdOsLIn
+ 2NkTnKwJdtY4jJzMMW7GGCdjlJM+zEkdYqcNsVMH2SmDcC8vc4y3r6DXNKZTNqIdQId1TZm4EanEkWOp
+ o9Lp44rZk6SCGV7+tDh9hBPZhQ9oVvBtAneHLXbE+H5K0hA1EXbTDeJi+hSje2Uie6XihuWSxtEp47js
+ SUrZnKDuqVrzM62O17rtL7UaFsTg6OnjuIRBmfjBo4mjRySgf58z+0PO7HdpD7/NnvtH/vwPeQsHcp4d
+ znp2sODxkYpJbFEdJT2YGHpBweukjIuqrLsQ48mFJkQkfw7Zk0W4zDx+Qe0foR6y9V28vlfC7lV6xzKx
+ 8y2p6zWjdZifEUJw1zlylXrwEunwZfyRGzjpWwTYXKdwjwU9GSG5jnbn47zFJD9lqq+Y7MUn+PCJ/jyS
+ j5DoKSJ6KZF8xEhH6VAVSqQ6M0iT7mHETrvvUJMZ4uFy3tRGW8NEaHRGw9xR/fR1XddwJ/+EG9c9T111
+ MfMNuXTmrKa83F+OH/lSTupPZNzfr1/SLi3wDQ2wj4m60NQYXFfrXVZ2p6z8Rmq6TXnp5fYm16mBwKaK
+ qzkp1lPD0f/7z8jH3578d3fmX+tj4Oi//TTz8d8PP6x0DDT699TcXXwc99//FO78M2huzXRiiz767vjw
+ ++9Hfzow/P5o3/bB/t0f+ncODm1KjW3gBl6SGgcIBeWE3Fx2aRWv64HwwZLKzDZ38h1h/ANq4p/YiX9R
+ xn5iwkcP9EJz6k1qH4A+qZKWI3K9yjp/gnVeVXxOVcteVdNGDG23YKOnpOEWD2lgZMUX26ponFHXPauh
+ e17rxEUdg6snjO6cNHe3sfV3dAy5dDHq+tWYa9cCzjjc0DWwg35dfKVTQrVTADpE6iK1M8rqlzTUXUx0
+ A88axrga5ySaVZZbllSapGbphIQIvbyYfsGMuBxeboeocJRXNMkpnObkPeTlTHOzp1hZU6yMCXrGBC1z
+ kpE5xcieYEBmL3eSnTPBzhnn5I5zc8Z5OWO8rFFuxiAnbYANSh/kpA6ws0Z5mSMI6Cax4OXtqNBOuahB
+ ueRx+ZQxmewpxcKH5KIZZtkjpao5nYJJ1eQ+Vngb3q8J5dOICmmFxi8EYD2mlxjaLhvaLgXF69H9suDr
+ yWP4jEly8SN2/YJq84JG02O1jmeaoBb4TJzHvLxpctqoQtLw8eSxw+kPDmY9OpD/5ADE6GmPvs15/I+8
+ J9/lLuzPeX4k7/mx0kdyJe245ChUqK2sp/YxNw2puyqyLsoKHkrQJ5HizaK5UPHXhdIe9kdLCkkD87yB
+ ZRbSoXyJ0r0CFSCU/nmlsnROgJnsHc6xW0zYVidzhyTnTFGALrsubLQrB+PGxXoICD5KkECk+ojJ3nyC
+ Nxfvw4EmdQQvJbKvKjVQlR6sQgtVp4dqM/0MOH7nTiQGXg30vmRpq61mKGCqECkiBZqSLE8HY+CgfNXb
+ 9l6w470AB7+Iy2HRN41MOAf3f3Zg/2fm5ryiIv+cXI/snPtFpd7jD3KfPC/tH41vbPEpKLjU0eoz1BMc
+ FaB73lrG7SanrvzeTzt9H39d+OXdo39tTfwGM9H/PPrtnxPLCxV9db4tJTemhwL+90v5f35Lfb5xbmpV
+ NLGDGn1/aOTdj8O7+4d2f+zb/b5v98eB7cMj2zIj65jBV8SBeXr/HH9wgTu1ynywSZ/aok7tkh78RHrw
+ M3nqZ8rUz7TR97ShD7ThXcbomrDroVpOvtL968zz+qzzasoX1XXPq+rZK2nZCSH8ULXiKkOK0IwtQips
+ +ciKD9y04gH9kFERW3KF1gIxspgKTaWh9a6Sqp1QbCtQOiWAZAtsyFK24CvDNmoLoaqDitp1bQ1P8xPh
+ F0ySfE4W5JiWNxoWtWsUdGjn1mullqiklAuymnn5A+yCMXrxFKPwASsfWH/AyZ2CZXVGNhSSTFJzpiBD
+ TcudokPUkDvBzB5n5oyzcsYAdEHOiCB9kJvax07pZaX2cdL7eTAC93BmX1G/YWyPdChsi+6Sh80TyeOK
+ qWPyBQ+J5bOMqllR3ZxW7aMTJaPqmf38mHZKcAvOvxEd1IiKaMPG9hKhTYA/7NhoOR7WLh3fj4rrU4jr
+ Q2dN0mDNv2lBvffliYGX+mOLRhOLRiOv9dtfqdfO84pnyNmTChmTxzKmDmbPHsiDWOXJ99nz3+fOf5f7
+ +B+Qb8l5fiz/qXTJqGJmDirw0lFvzeOeIhlPFcX7yoruKoreahhvZcCRDF2K3IyP58ZQByfEEyucwTVK
+ 9xKpe4WC5BCg2ukpvzSDHmghc48n7cyWvwtrRlTUXZr8HYY8YucS1t2h/aKQCJT7CEnePLwvl+DLwXtx
+ 8eDofioUoDxQDD11Ke7qJD9b9ZSI2/Fxbnfunz1/08rC0UDHTKRIPXgU85djxL/9gPmT+CTFOejMbV9b
+ Z1/b+35nHC/qUBgHBEqy971OxSZdj066Ghl/KSX31sBkev9ESnLOFVcf3WvX+R5uGpHBJ/NTHCN8T+Qm
+ n337tOzX/0x//OXpx38/+/VnyC0++vjL7L93BqcH0uqL7jYUXH82nfDr/xp/+1i+/sHv4fLJyVXm5K7C
+ 2O7B8d0fx94fGHq3f2Dnh8Ht/cM7R0e3ZUY35ad2sDPvSNM7+Jkd3PQ2bmaX8vAdDdF7xsMP9Ec/06d/
+ Zkz+xJr+wJnZVhmY084vUvG4wUZAV1W5oKpzXlkXtkfYCTQsIepgI4lwc4Cbq2wN3swWWyALQCJzKC5n
+ CUyhiRcLtoQKzJg8CzhgwZZqZIeoJH0O1edCKN4y5ypZCVTPKave0dXyszSIuXoyI8S8qORkaYdOXj8/
+ Z4RfOKRSPKBSPCTMH2bkT1CKpqklMwA6M38aWGflP2DmPaDnTgHfNDgAwZn8SWbOGC17lJ41zMga5mSP
+ 8LKG+Km9nJQedjKMvdy0fl5aHz99kJfcy9pXMmgU13s8GNb5e1HRg6iEEcWkIbnscWzJFK1iUlD3SKd2
+ Wr9iTDd/SDW+nRHWTApuwgU3osJboKEXDvrA+DVJ+TUcC22TSehDR0Pw0yGXOkSsmlPqeXVi/K3pzJLF
+ 3Krl4xWr2RWLkRWDjteq9Quc0jli3kO5rOnD2Y8O5D4+gLj4s/05C//Iffz/aeotoNpOu7fd/nXmnZlO
+ FXeXBAhxwb20FChaSmmhSt3djVLaUtzd3d2dACEkgUCE4E4I7uR3Hub9znfOWs/KIim0a9E7O/ez5dqH
+ kwZEE7nSySz51HqVsB8KH1ylXxnIvyaqfNCHvSepf9BT+2yi9t5E6bWx0ptTimEvNFtqLJgTlt18dOu8
+ etOsRv0MrHJSpRponYPPikV8cpR+iJbcD+cI+afaSo8BFwCtAGjRz7GqQO5A6yCoA5sObqJvcRpA5R+w
+ AESq9hLcAfRhnwAxnQR/a6zzzFo39JVXUWFUbPqv97+ePvp44+5Lr5sP3WycSQgDOTnUkSNq/yOl8y9z
+ F6077xzvvXO4ct/ilLM2HH2IZCrjedPg4evTLz85+gV7ZZd+TC988+WX+7X7JAtbyRMnJV2dVFwdFF89
+ MM6Ov1uS8XykPxva7Nufht4aFG72QztMSMhcnq1vrQ6I+eWZHHpxdCAO2muAoIqN3TjuzC3KqAl1Dk4V
+ SFEXj3UtHe9YEm3lH21bOE5ZkupakukUyFCWpXrWJGkrMvQF5V6Bev8ynLmszVxC9C7q9i4h+ldRfWsY
+ xhqudxnXyzdu77dKTjV+fRd7zQZ32cT4irGlp6GFh74ZILGAsA3QRaC6CZR9Fqfvjjc8hwPoUAAkMtjX
+ Ona/JHQWb+iMNwB0AAeMngMYmcMbAHG7YA1AWdQFBXQPRA8QAMbexiavbE9+93AIve8S/8M1Lcc+s8ki
+ gYyK7IQldCHTqLgUKjKxB5bI0Ejp1U6loVOoIHeHSqWDR3Qy0Df1/zspQOgUJBB6Yicqvh0b04KObcHG
+ NOEj6tGhtdjwekJEAymyUQ8coHXwdF/owU1SXyplvtaDUQkwRaEU3CgfWq+Y1ITIJRtUMuzKe+yLqXYZ
+ HZaR9fhfVYhv5Ro+JQq+ZYrfq1V8qxR9KmQ+lUr4VsoEADJjpcKPKqVYsm4ew7CRZ0OfdGNNuXOnz/Om
+ 3Qdn3ajTDk2jVpWDegUc3Uww/L/f0CKRxBZP5IgkgRwL53By/+HkgeNJXIl0tmJarVrgR9k3ZyRf6Mm/
+ xqmCnPdnHMyHoPnVSO2juexba+mwu1o1WQa8QUcO/xR1AU8GSyDmQUpYtWJSqXpKo3EAmxGn9d5J7B5K
+ 5L6O7ENwB9VSeopUeoZRfopWAecJRvU5Xh0E9TdEOPD9r9Cqb9Cqb9FqL1DKL7CqH4gwH3A3BZdUY50X
+ tthE/4eFFQkxuSEfgl88eH/V87bdaSeCnYu+pQNWnSRxRO2/DyockNH5XwM7Ra8HRvfenrp4W8/xIsrG
+ DebgqX3jmem3kItZJe+Scp55PzCwcVK0c1U97aBsbi5uaiRibSLifELyspPKm7vGKRH3J7iVkHBUuDUk
+ 3GJBe0DrvYKZekpzRHTApfDvZ+ktATur9YB+IYQqpxe/MUad6JNoBl+JsShJXZTsFEi2zR/t4B+jLkt1
+ L8l0L0lTlsS6V0Roy1K9C6oDi5qsRThnRZezguxf0u1b1B1YxvStYBmrWMYSto9v1DlwIjXF+M1d3PXT
+ +EuAGW18AnBDz+ubuJGA8TBwxJMAtAgMPp/F6rkBWeP0XTD7Egcx2xG8TjAA3+NCNHTe514Y7tNgCIaO
+ BPAJoO8Kvg1JdEGBRgBDD6IpWA7zycku5Kpb7ONzqYHnsvLt0xuNYtrhoPIYS4HFUeDxVNj++HwvLBEM
+ XnYjk7pBRNdNoSNT6ciUHiQQ/f970Kl0DIjoiV2oxE5sHBkb2YwCdiW8HhNcg/pVgQqpxofX64WBHGst
+ Mbxh//FAVod9QJPExzJJn3q5Hy0KYNYzuEElsEw1rgJTTLZuYrg001xqehxyyJYxTYSQBuTPGs2vpXK+
+ ZfL71ADw9qgS86k6/rVa8nu18tcSZWBp4snYHKpB/cDJ3gkgcS/e5IWRafexOXfurDt1wr5h2Lh0GJsN
+ NuhyFBPY0ols0STOkbTBw+mcY6kDx5PZx5OHRHN4SnkApOEr98b52Btjmff6Kp+IGj54rR9GiG9m6p9O
+ SgdeVqmNMx/jOCysnB0W2FDnSO1zyKYpjdop1app5dpJ9WYWOiVR5ZXboVvIQ3fgUvfgAAqg8ASpAjLo
+ z3DqgKb7BKv2gqgBhP4Sr/4So/ISpfwKHLTKC7TKa6zGJwLcj6Tlb6T70VT3tS0+5vv9+OyArxEfH364
+ deH2aduzWMtTCBMLhJk1AmUkd1D+P/+UPSCP/BfBWsrGQ/X6M9Ljzyfe/XJ59cP59S8X36gLHwIdnn60
+ uuiNOuOqbntGze6MuqmplDbsvwnI386Yi9w9r/7YC/7urp7/O2dObx4EjQi3R/Y2OdDOvtBnRsorCr8G
+ fHUP++beXPpFMFEG7TIhiLGxXTgleMqZNh2YhfXxVWgLql182Y6FY52CI50LYpRF2Z5lmZ5FSdqiNGNR
+ mSmADSzA2QIEbwk9uIRlC9DMBWTfEpq2iKIt6/YtYQf4JhSmdUaK2Zu7+KunCV6mppeMTlzUM/MgmgBZ
+ 73eqACuCA84E2JL9+O2K1XdEEcGwsxPgdeH1HUHwJhr+I3QDJ3CIQOWG4MX9p+DH0fpOSD1XlIEnASzA
+ sPjoYhty7Wzcs3PZYecLis9kNAFImGZkp2pMlyYQOmjkTqIhk+ioZODCu3USuhH/CF03la6bTkemgy96
+ dFOp4KDSqci0LmQiGRnbho5sQUY2IyOaUGH16KBq9K9yTGAFPriKEFyND64hhNYRQmrwB3K6zoB84scK
+ Cd9GWf82ENFVgupU/EtUo0uxeU1WVR22jT1ONT1nsvdtOj6yGR3SpO1XCYQu/aUMNDZK/6iX/l4n/rVK
+ wqdE+kuxgn8VLLoFk9WpX8WwpnBdB0YvsscucoHip9wHZ9wZE/YtwybVo4QCMNrMAW2J0iBxHs/8O5l9
+ OJUtksoSSwX5Fp5oFle2EHTzJqj73BJ/d1LqvaniB0NVHwPYN1P4F0sFv/PyBb8Mx6hX1pYuLW64ja3Y
+ 0vmGHbPopilY3ZRazZxq/ZxGMxeVmqz8/Nyf11B/emuI3VCXugWXuYdQfIhUfYxVfYJTBVkXsO7iFVH9
+ BfgarfQUpfQMpfQcqfQcowrSjh8Imt/04P6myI/mup/PGmeFv0nOCw1I+PHc54HHjVMXrpmdv2h20gZn
+ 44CzOKMjofGbmMZ/GToqOXnr2l1ROu0l6/EAWHarxz4nn361ufPG2OW6+ik3GXt3lbMXtJxcNM1MxEm4
+ v62Mxb+8OlmYfKsi/WZp0tWytLttld/441XCnQHhHkAXDYDi/7qgqYccGx95x+etffj38xXZL8aY6Xur
+ PcLdfghqXdsOGuO7cOcwrH0Xrt4tUKQsi1NWRDoXJLsXFWnLCrRFGZpAniFQYy5qshe1BpeRw8uYoSUM
+ G6icrwOCOl2A6FnUZS7hWHxTCvNkRqrp6zv4qzZELxOLy0bWnkDoJHANNQSR23HfeQPUFngEhgTEacBp
+ Ie7zuv4tdODC9wFG+1+DL8AB4RzMy4HXHYCDRxNB0+JZrMFFkvFdS4sPbvbBN88lvruYG3uhoMI+s40U
+ 3wGL6lQDET2Bqg16/lIY2NReTBoDxGzUvsQZuml03TSaTgYdkUnTzWSgshiYLAYuh4bNoqKS2pExLcio
+ 5n+E3owKqUUGVaKCqvCBFYSgSkJQFeFXBfZXJSqgCgMWAdgGNIp8KBP1qZP60bI/FvQD9G/lKwYXIOPL
+ 9NMrjYtbrAvbT6Q0G0Q1Y2LaURFtWj8A5Khyv0kdTEz71YD5I9HP5aKfikQ+FkgGVMESWgnZnYbFneaV
+ ndYN3fbkPmcaz405do47da5/0r5t2Lh6BFM4rJ3BVUliSsQzQIYR3EGPJLNEU1mSacC3DEukc6Xy+tTz
+ 6nUiApW/XpV/Yyf11krqk6Wij7XCd1e5lI/IvsZzm4uPd3a8BZtuUxsO3GWLXgGhY067cVq9jq8OthC2
+ DWMz0tWee+wL/ZqG2BU1iUtqElc1ZLzhcre1wDip/L/vo+AAs/5AV+Gejtx98DbQVX6AVn2AUnkOyEd4
+ 1Y966m9MtV+dIUb53kvICg6I//n4450bT1zcLhna2OranMacu2jk6kVUw/6JsxK999nyyhuMwy2Z01fF
+ nG4qetzTcr+pc+GWrvsNuPNVVWs3ydNucm5eMHcPmJO93M1LiJyEm8tT2cKVgq25lNXppLW5nJ2V2p2V
+ tt3Nvr3dwa016gq/brA/pabCNz76jt8nJ//PTqkRV4B72VpoEW4B+07d2c2cXbrHnTVhLWgzlzV6llWB
+ KaesiVOWZLsEit0CeapAtge48wWN/iUN7qr28BpydA0zvILmLugOzGsPLOj0LugwljH9SwQQ0aks65ws
+ s7cPwGWUcMnE8pIBELrp+X2hGwGPvp88AYEcWBHQtQJsNyB1oYhAxP9/cf9b8fsHr/9vlZ/BEO2ReHtd
+ vBOKCDw92BFw94TVx/NOofc8Er545qR45NfaZoJsCRUW3wNPpIEshTaonWcM4DP7cVn/52Cy+zFZTFQm
+ UDkDkduLBpDNwn5C8QAJ1MvzevFpnZj4VuDO0RFNuiH1iIBK7cAKVHAVCOfE0Bq94Griz3KMfwXKvxwF
+ aLo2P2sPvy8+6lMr+b1FDkz+fymR+pwr/6sAFVmsF1esn1xulFprHN9IDGvQiWjXjiTDg5tUgpvVwtq1
+ AgFqtE7Jt0r2SzkYUBL9WiIT2YTM6jYp6DEv7DAtaDYtabWuptg19Tl2j7gNTLsyZ+xbx4zLeLp5XM0M
+ jhJoyk1gioCyfzz7WBJHLJUjlcaVTB+SACeLJZtH08iu0UmORga/VvG5IvHVXTL4imKeL55R6bQ0fWtn
+ ++7WztWVzfPzm45jq5bcRRJ1RrtlSrWBDxZ8apPHCNnZmi8v/n0VffCapsRVdZnLGjJXNeSuqslcURX1
+ 1pS8qwPErXRHW+mWlqK3pvxVddkr6nKXNBSvaClfhSve1JJ9jlF8jJS9hZa/Yqj5/fW19MKY4IRf9155
+ O3iY2DggrE9qOTgSLlw2druEMbYRufOWFFvq+SaS6PFGwuHuUdd7cm631S/e1rl6D3n5AcLthpqNh5TD
+ RXmPKxqXLmm8eIiLD3ai1DwfofsujoXuLMRDq1nQRvl+WXSHIdzlCoW83Q3q1Gg+tTOkvORTTOStHz7n
+ Pr+2DfSxbyh5uTpdBa337V9bhU3rOyHD8+f6ZzHMJa2eFY3OZemuVYnOZZmuRUWKQL57QZa+qNq3qNm/
+ pMZZhvOA1ld1h1aQg0sI9oIWOEy+du8SqncBw5zT6+GZF5aYfHyOvG6PvWxq6WVwwhOgufRMgMpBBzkQ
+ +r7hBpkT0KEFhP7viA4yKv/E7/8b1x2wAFW3H+b3YzmWBM4ZMAqNwoOczDmi0SUT80c2p30unAUQ3YRv
+ F/OzLpa0nsmnEdMY2qm9iPQ+VFovKnsAn8cl5bHBIeSx8HksXD4bm8/C5PWh8vuQhUxMETh92BImvqSf
+ WNhHyuzGJ5FxsS0gmYgKr0eF1aLDqnEhVYSQalJojX5IDSmkhhhaRwyswgKPbhfcIvGpTPRrvZRfsyxo
+ zf1UIu1TqPKjEB1apBdXYZJcZZZcZxJejflRphbcqBHVqRVB1oroQMR0o0LJCADu8qtS8SmT+1IsE9YA
+ z2EYlbNPVrJOllGtClrNC5otc5vMC8iWdSx7ysQZysypujH9fC4YoVBJ5ijFMqVimWIxAyLR7GPxXLFE
+ tngKkDtPNHVIJHNYOpenXNivU9pOKiolpkRrJX3XrIzW5zW5bYzfEm7c2t29urPjtb59XrDhML1uNbps
+ 0DuDaBlXrZ9WAdM3HaOEnGz4q4tHrmMOe8NlvGGKN3RUbumo3oQpXVGXuqwqcV1DzltT4aq6vKey3Dl5
+ GScpCTuxY/bSom4aCu4wuUta0k+JqvfQMhd0JG7ZEONDPuRXpHwN/uh+zemsl/l5LyMnF7zbOYPLN8y9
+ buIu3FKJzHFo4N5Ob7d5Favi8fLwuQeSZ28pX7qvc+sZ0uu+xtkbig6XZR0uyly9DXv0QOfTC0xeghOb
+ /Hii7+0s9+u2IH5nOVu4WSfcogn3RiDhFASNC3d7B/oSigtfpKc9jgy78+PrxddPT318YZafcm1uMOef
+ 7oABaJe+JyycX3/DmT3ZNw+yhHDqqjxlXaprRZaypEhdluteku1ZVO5dVO8VAJuuxlqC8dZ0h9dQvGUk
+ d1GXvaADhA7cC30B0Qt+fNywqtng2xetm67aly2MvMBl1MgcXEZBqsSNaALS5yCJvm9aQOIF2PT9rOL+
+ TRRo2uEfTe8HbzBGBGSNIQFK0b8V74DZH7lwwhDd8AYX9E2vm1s/t3fwv+QR+fhC0g+P4sKLlZ0O5Uyj
+ vH5kFlM3dwCTz8YXcojFXL1iLqmYSwCnhIsr4WJLONgyDq6cjavk4KvYhMp+XFkfpqyfWMQk5dKJqZ34
+ +FZMbBMqphEd00iIqidF1OiFVpFCqojhtaSIuv3ES0gNDkT0M2HtciAqg5RLYIeKX4PC53L5ryUaX/N0
+ f+YRoitMEmssYqqN/UuQvsVqYS06caD3oAtUYtExVCB07e91ar5lKj8q1EMaEIndpPw+k5rBU02jZxo5
+ Z6rotgXtVmkNxmktxrndpqX9JmWDBvk8dBpHI3ZAIXpALqxXMowhGsEUixwQieWIx3PEkgfFknkiqcOi
+ WWOymSOKWRz1kgFsLdOwvsuI3GI5TD+7Mn5ld+mqcMNzZ/Pczvb51Q3n2ZWTY4vGPAGRMaNDnlRvmlZr
+ mdcmDxNzcrTeeB0FXek3dGRu6ijcRinfQqrcRChd1Za+DJO8oil9Ha5wWUPeQ0n2jKSo+d9/6f/xv6ai
+ fzjBpVwRkucRxx8aKdwhSV01VAx5d7UwNzQuK+D2y2tOnjZnL1u4eJCcXHAuZ/W871rffGLw8huuuMOj
+ e/ZK+4xzVpf+uxgFrxciDtdEvR6o33uL9Lyv5Hxd0vW67AVvpYePETHhduTae9yup1N9T5eGP2/Ohws3
+ c4XbtXs7VGhvFBIKoL1FCJpbXe0qLnz11cfW39/j+/dLr185Pbpn/vqZcXrMxaGeqM25RtCqDu2yIGHb
+ ljB+ZvlO37QZdUGHvqrasyZHXZHvXlagrsh2L0t3L8nTllToC6o9c8r0eXX2CmJwFc1dwQyuYNhLKOYC
+ gr6IoC8h6AJd0CHTyCCFR2o9vq51zYZ0CWxdBHUfQ9PzIMNoYO5KNAF9tgA4ChyI2z9yB6IHYR6kXPb1
+ jSHaofC2KLw9inBmnz5H+PeL+7pHE10whHM4PVAl9TY78drBIfCye+Qjl/ifDkVlZ2vpjvVci0ousewf
+ NQOyQ8UgqWbEsHoIzIIQKwfxVYM4cGp4+LphPdDZ0Thq2DJq3DRkUM3CVbDwIK4XguG1Hr00UBNtQcc0
+ oGIacLENetF1+hGVxNAKXFSdXkyDflyzPkgYHshsdw5rVflWpfQdGJIuTVA2+lKm+LlY810m7FMG4kcO
+ 9nsm2jcL+aMEHVCNDGtGRu830KCiuhERXZohZMCVVvIpVfpephHWiEmmGObRTGs5dh0T5zpGz1XR7Uqp
+ 9mW9ToV99vn9p3L7jbPZhFSOLqBZRPTLhzNlg+mSgT2iIQyxiD4wYyEVx5FKGBRP4omnDEqmcGVTuEpp
+ bFjBAKaGa9gxYt03ZDc84jg/47q24ra15bq15bS16SpYsR3hG7PmCH1z6O4ZbfKUJsi9tM4iWodxWXmw
+ t1ePXSceuqItclVbzBstdQMjcwMj7U2QuEGQvIGRuo1TuI1R9kYqnlWVsBT90+DIfxlL/5cj+qiHoYiX
+ 4aEnpyQfWIvfd1bPSnxeUBIQnPjp1stL52862LnrO5zDO7riz3mY3H/mcuOJ4csfuHKqW++Cew/fijxl
+ UUoz8UvWvvFO5uZ7tfs+upefKbndlDh/Q+bxS2R2ltdgny8Q9+qoz+rY5/Wpn7vLqdB2LbRH2xMOC6EF
+ IbQBCdcgiL+y3JmVde/RY9zjJ6b3H1jduml2/57Jp/fWhZl3xwZidgRV0Cb1H6H3QlDVxk4Qb/Z8zwyu
+ dwnOWFWhrah0LykBp74vdBDXBSo9C2o9fDXagiZzBTmwimWtYdlruIEVbO8imiZAdi/odM3DO+cQLRxc
+ RgHu0yv0LWfCFUuTi8Ym5w3AqITJOcD5J5ruC/0f0wLCORD6fqkIpw8iN9A0eARCP62LBYBFB7QeEPd+
+ gEfhz6DAKDT+H6GT3Il6NyzNP7ieCbruGP7sVEKIVXGdfT3Lvp5nVs0hVA8CKZMaRvSbx4xaxo2aRvXr
+ eYR6Hq5hCA9O8wixddSgZcSwZcyobcy4dRQkrwm1HFw1m1QxYFjWb5JPN0wl45JaMMnNxKQm/fhavehq
+ UlQVIbqGEFNHSGgmxDXiDmSR3cPAjsVK0Iur9L1Z2Qe0K1apfS6GvcnQeJ2i+Toe9ipG/V0y/Fsxxr8S
+ HVSnGwWyOR2I8C5YRDeYklb4Xi/1pUzWp1g5sAaR1GGQSzGvYzlQJi+2cFyr6I7NQ1dap25Uj3kVDzvn
+ 8ixT2fgElnZUv3IIQzqEJh1Clw7qFg/tkYjqlY0ZUIjnKSQMySbywPicAhiCTuPopPWj8miEqn6zlkGr
+ Do4ZbdBsePrkwoqdYO3k8sbJlXW7aYFV/zShe0qXPA1vndJomdJsmFBvnkM0DqMzi+Ef7opcMzp4Sffw
+ Zd0j1/DHrxFEvEkid8xEH1lJPTCVfGwm98xM5ZGxymW0lJPKYWuF/z2l/R8eZodvOog9dhP9fFnmzQWJ
+ Tw908wsfZ1V8Dkl5f/u116UHLm7XLM644Z3OkryuWN996nzpnoHHfbnQHDxr2a1v0Ygxb8CYt61k2ITm
+ El6GaD0JRN/z1b71Wu3VF1xJ+Z3VpQxoK2tXELY75789/WtrLhraLId2qXvCkV2Ivwet70Fb4BGC5tfX
+ qVnZd69eV3M9p3z2HOLmbdO3b0798DuTm/WA1x++Op+9u94A7fYLhWwI6oCEuQurz/tnTelzIDZr0pZg
+ Pcsa1GWFfwudIlCmLqjTBHD6MoKxgmauE/rXicw1fO8aFvR4dc9jO8Fa6hmNzjntzgliVbt5eJDpk0tG
+ 3qfNL5mbexiZnDMwddMzc8ED97If0cE11BVDArdSQMcFqRjgT/6/iI7E/Z+IjtqP6KBc6rgf0XFA6O44
+ vfP6pJunTD+62wTdson+YJmebFnafqqWa13DM6rnEZuHiO3jBuRx47YRw7ZRg7YRUtsQoX0E3zqKbxnG
+ twwR2kb0W4fBHxm1jBi0jOq3juo1DOLruXq1bKPKAeNihlEGGZvSgk5tJiTXE+NriLHgVBNiawhxtbiE
+ RmxcPRZEdLegRmWfcmXQi/seVH+qFb7Vwb+Uwj/mab3PgL9N0vTNRIVVmgZXGfiVIH5VaUW2ISP2e9AB
+ 71wpolv6e73op1KpN3myviXqkQ24jHazOo5r5+TlSoZjLet8x/TdxokbZcMXi4ZcsrgW8X2YqF4Y2GoU
+ SJUK7JYK7pYJ6pQK7JQK61GIYirHcVTiuIpJQ8opPFXAlY6nA14AOouiV8owL+sxqWYYtrGNacPGnCmj
+ Eb7+1JLR1KI5e4bUNqJVPaRaOapSPaFeP61dN6NVO6NdMaiTXq7x6YWo98k/rxgd9rYUvW593NP0jxun
+ /n5+QfTdFamX58VenpUAHIE7ViKepL/OYv5wxv2Ph/V/P7oi8u6+5K83CnG+GmEflJLCjCqrH8RmPQhJ
+ efvmx/1L913Oe5+4cN383EVTzyvWN+7bnr2MNHc++PqXxsia5+SW9fiG4ciKFXPWsa7PLqJI72Mc5sUv
+ 3UcfteNSzwoEadBWIZ8X1FPr3dd8a4H3U7heKNxqFgo5QmhmD1regzaFwi2hEAh9mr9Ym5l/7fINGTvn
+ Yx6XYU9fWn74aPv1q31C3LX66rc9ZL+JobSt9U4IGhAKu/drpXu/hhddaFO4nnkt+rJW7zqCvgbrWVGi
+ ArMuUO7ha/YsaNMWEbRFJH0JQ1vCdi+iu5dQFD66axZNnkc0zmk2zWu2TGOraObRsZbPb5hdtzW9bGlx
+ wRSsFzU7a2DuQjB1JZiAdYr7SxXx+mdB9hBYlP37qD4ohe6j09EgqAOvQnJAkc4gcWdQ+9P+oPgPRkWB
+ p/cg6IOt6ndszT9csA57dioz4nRFjW01w6qKY1g5TGoa1ydPGJDH9NuGSe3DJPIIsWP0nzNGII8DuRPa
+ ePhWHrF1iEQeNmgeJDUPkVqGiQ0cXDPPoJFrVNVvXNZnlNOJT2lGpzThk+sIibWk5Hq9lAa9pHpiYi0u
+ qQGTUI8+kN5xJrAJJMXlP1cpvS+V+QAQLgAjWof4Xo74VqgVWILJJJ+q414q7XeNbzEIrFIPbgEYaLVY
+ ukICTTKy87hv1ZH3xZIv8+ReZyt8K4IltpoW9DqWc9yLel0rBz3rJq6V8jwK2GfzOU7pLNNYOiaCBg+i
+ Kv6kyP7skv7VIePfJve9STawUzGCoR7dr7Evd7YKIC3GMjViehExoD+zg5DQTkpuIeV1GVb3mjX167ey
+ 0V3D2rQxbeqoTusIrISrmN4vkcGRLhhTKxrXKJ6AFY5qFgzAkytUPn86dsv99xvux+7fkLl187iX5389
+ uPvHJ5/jfj/EfX1EfN6LvH8qcvvibxcd/uOiw39e9/jtyZODfj/FY2MVC3IQDRXE+jIiucmpo+VRaITn
+ z6iHH389uvHY49Yj10s3T7pfNPO4ZHnjnvXFa2ijE78FROqt7Dxc2TmzuGWwsGU0tWrLnnVrYTnlNlvH
+ 5p3yC7ZoankLQbULY7G9TW87y2+0Fl9itL7d3QBZ826hcHgP4guhFQjEcuE2tLsICfv5S1m17d5v/NQ8
+ bx+9clf2wXPU01dmHz+fiYq6lJF6vyT3RQ85cHY8b2u1RbhDhqCaHSh2avVm77QpnY/oWdLoWdGirSBB
+ CKcvqtAESowFOEOgwxAgaAu61HlE1zzYua7TNodom9ECH4PNsxrVcyrls6DWhsqj6PkG4a6cR7qZYd0N
+ 9d0MjV0NwD4ji7Ogs5wA2gFM3PX392G46QGDDkRMcgGlIlAbAnLHGpxB69uj9PfdCwprj0bbYwF6ztiJ
+ YH4Wb3pJz+SOmdXTMzafrtlE+dqVlzi39Tk2AdPCAygOXMOEQduEUdsoEDqRPIwnD2M7RnDto7jWMSx5
+ FN8BXuHhWjiYdh6+a0SvlUvYP4PEFi54Vxg3cw3rBwwrew2yydjUJmxaMzGxHptQi0+t00ttIKXU45Nr
+ sakNqOQ63QMZnacDmsQ+lUp/rlT4UCb7qkD8Y5l8cAsmqA4TVIlJI5u3DF/om7vdM+tdzXOMbNL+WScb
+ RlGIp8sndItHtBz9VnbkfZHkmwLFF5myb7MVgqqw4bX6kQ0miWTbApZHyeDFApZbAcctj+2Q0msY24OO
+ 6UOE09V+dEj7tkn4tUl9bwZwL4XgLrUIOtiNBgunK4OFR0k87bRhbOqgXkKffkQ7MagGHVymCz6Dctr1
+ yumE2gHd2gHlaqZ0zYBMOUc2my2R0HcMpCnTuLJpHDnwmMqWS6Mrx1XIfwk48ujJn89eib3xk3v1U/y5
+ 718+YUdDs6USypRSy1SSCpWj0xR9A0Wefzr40uewb6h0eLpyWplmfSexl2PNG7IbH3GanrjJY72Njrnw
+ 5tuFl1+u3X527tp9e09vkHgxvuxtfcnb5Pod0mmXYzEplpt7z9d2nJY2DJd3TGc3To8uuXEFl1j828zx
+ tyGRdj9+nOyj+i+MpSzw4oX8lFnur+byJysLpTvb3cC37EFLwLQIoc19g743KtxrnFsMHhi9V9J80i9C
+ 8+5LsWv3pW491H71zvy7v2NkuFdO+sPmuk8sWujwQOrKQhUkrBdCqQsbL1lztvR5VPeCWhdfjbKg07N/
+ 0dTsWdz3LYwFLTpfi8rX6ZrT6prTbp2GN03CmibVmiaU6qeUy8flSydUysF2gFrUg5cqNpayVih1W6Qu
+ APvb4vUcwE2UAFptzcFilvNGpu6GYN+isTuA7hqTzhnhXA0wznpYJ+J+f4stWv80GhDncLZYzGk8wY5o
+ 7Gxodd4QjHFY3La0fOxo9cHbMi7YtrndnTbq0j5hUTtKrB4l1o8bto0bkceN2kdAzAbK/rfQ8a1jOBDO
+ wSttPFwzFwvieseIQdsgsZ23H9rJI8YdY6atw8YNbP2KXhAHsZltQOj4hDpsYi3QNzGlnphaT0iuxaU1
+ oveFDjjZoW2iPuWSH8uk3pVJv8wXe50vEdCICK7HRtTiM9qN2gZdeYu32Ys36wbto5u1/SokA5rEI9pE
+ o1uOpXbLJ3Vpfq9UfJMj8yZP/mWWzIs0mVdpiq/TNb+V6EW2nE6hOmf1uWYxHVJolmDWFQxqxDFRkQy4
+ P1nep0nCt1n2B9hBAJrJyGpBVM2IXq2IXgBhVM+YABhvg8xRkzSORWKPZXi9vn+x7s98eEQFLK1dq4Cu
+ VsSUKWeLlw2K5oAWduaR+L5jcX3HQYtYElMsBeieKRbXLRVRK+WfIe4TJ/49Xd6/SMm/RDqgSCSqWiyB
+ LJ5Bky3qVynqU82lqCTUy4VWSAdVycQ2q2Z2wEoYOh2j+pzZE2Pzp6dnHfhz11cE38srn7z8cvbRuwt3
+ Xpy9dMfK85bp+StGnlctPK+ZXLyGuXJbPafYeXX76dKG48qO5eKOxeSq1dDiGa7g3Mz209F5n0fPEEaG
+ v3ueU4oPvdRZ78vu/jEzGDk9lLI4V7q327OzMwrsyp5QuAdt7EGzwIrs7CRP85+N8C+xpt0aek7G5aJ9
+ gtXf+iLe+Rp89LX8FeSalHStpPBhU82b5qov49x0aLcOEmZv7PmNCNx7ptDdc6oUvkr3Apy2pNO9oN4x
+ p0yZ06DyYdR5GE2gQ11AUOZ12mdg4NbeNKPWPKvcOKVUPixfzlMvYiCD0uCXvGXNDWUNtVRM1TWMNWAG
+ GjpmOngblJE9WKGIM3EkGrvombobmV06YXzL2ej+ef277sTLZ/AuplhbAsEaTTyBJpzCEU7h8dY4gjVY
+ 9qJv7mpg7m5gAtZGAwzBy2t6SdGn2qlnaRNn2ifN6sf068YMGieM2oHKR4F2/6/QsW0j+OZhXMswrnkI
+ 0zSIbuSgmwexwMC08Ihtw3odo4aUKQvKlCV5zKRxkFQ9QKgA85ZUfFozNrmJkFRPSKjBJtUBiRNTGvCp
+ TZjEOsSBUuaZRLrMj1rp90Vi74uk3hRIvM4T96/X+lWtG1qtm95KbOg7wZ69wOF71bCsQkBDS6mYf+2x
+ sIajuQw12uIp5qpbQZ+JT77Cx0LFt/nyT9PEn6XJPM9Q/ViE/w6cfcOJ2E6bpG7rGLJBeKtuZIduAhMb
+ 34+OoGvtQ3ebQb8k/Fu92rcWlZ8Ae0SDRTA0ADE9ZxKfO2WQMWyYxjFLZ9rEdloEVGP9imG/ylWjW1RS
+ aXI5A2JFQyL5g4fSmH/E0f6I7v47rvtoCk0svU88pV8kue94AkMqmiwV06YY0aYc1qkUDnad0qQiKSJx
+ PccT+45msI+XTshWTiqUDMvnchRT+xXi+mTi++TSmapFPKAAdO+80YjAem7ZaW39FiQMY/T6v/p8/t7L
+ 83dfnL9408rD28Tjqtk5T+DUjV3OI67eUiupOr+y/Wxx03Vx5+TMuvnYiiVvyYYjcJjdfjgj+HHthipC
+ +4CZ4dHXj09FBl5Lir5KafZbmSlfX6wX7tJ2tof3dtcg4S4ELQoh9q6wdGX949TCxbElO57gFHf+TCfP
+ przTqrDBIaPsfGi8y49gR/+AM6HBdjHhTrkptwYZsdA26PQq2hUGz65eZ04Te2aVuvmgLKpCW4RRF9Qo
+ 86o9ApAv1+nhawHr0s3X7piBk2fgLTMwQIMhCzRaZjUqB5WLaOpptTqfAtQveCqfMIUZ6MAN1DRwSmpI
+ RTWsqraeBtpQA2cCJ5rrGliijECY9zil9/SGyc9P1qE/7d4/NXOx0TJGq+tr6Rhqoc0QODNdnIE2xhCB
+ M0frnUIDP6PnSiRdssa9uaOXlnKyjX6mZ/xkx4Rxy7g+yLG0jhu3jukD2902TCCPEsEdFJyWYWz9IKZ+
+ EF3HRdZydOs4qIZBTOMgtnmI0DpMAlrvnDTtmDAHd9P6QXw1B2QbcflUdHobOKQUoPU6TGoDPr2ZmNyI
+ TWpCxtVpH6gccErrU/CvlflYLPWuQPpltsTLHPHARu3vFbAfxWpJDcjCdnw1xaBlwKqSaRpUIf8p73BA
+ 7fEsuhpVcIIPXV+FHvcveEZUaH7Ok/lULPsmX+J5lviTTPkPJdivFfrfKg3DWs2TqCeSuk0TqHoJNGIC
+ A580QIyk6X5vVP7eoPazSftLrcqXevkfAKZOUY+ggU0v8IJJbOGUXgaPmMrSTx+wTKSbhbbgghoQwc2a
+ UV2KCQyJlIFDmUN/Zg/+kdb/RyL1r3jykYJe5eYJbPOkbhlPoYArk8mWS2bKJferxLFUIthyoSzxMNbx
+ cOaxWObRVNbRwhGxxgVF8opG6yK8ZgZWOKaRATbNshRSOeq5w/DKCWTXvNHYmsPq3uXdvWcQFMfsD3n4
+ yvnW4/PX77udu2rldevkhWunznpYXbhk7XmFdMb1SEK6xbLwpWD3/OyOzX6ZVmDZy7dgzFmPrt6YFPh5
+ XlfD4f/l7oa8dsn03ctzfr4XUuLuzQ+V7K627m50QbvDwp2Ff6w5Twg1bexG81duTCzaji6ZDgqMeEuW
+ nKXTfbPOwAWx5z/Udj/9GWn38Dn6yXPdN69xUSFnGR1B0HotJCzdE0YINu5y541p04qMBdneRSX6ggp7
+ HcHdwHDXSdw10sASjikAExgIIPT2ac22GVj7PLydDwwMvIShnFGvEpuj8/Yz/JInws4KY4xE6Gtp6+lo
+ Y+CaMCVFLSVlhJImUlkHrYZFq6NJWlp2llrPHhFj42yyc51/+hu7OMoZEqSI2ip4VU19DYQ+DEmCofS0
+ MUYIrAWAo6OIrkSDKzakD4+MMjNOtjNsKWPmHRP6beN67WOGLSP6DcOEpiFi8wihZQTfNkZoHsHVDaKr
+ OchKFqJsQLuCjajiIGu4qFoepmEEJBz3047t4H0yZlQ/TKgbwtUMYkr7ASwfk0PFpbVj0lpxqa24tBZc
+ Wis+qQmV2KQT2wA/UEJzTOiS/VEp97lYEXiPu3FHHiYdBdt0/WvgH7OkwipUM5p181sxJV16hT2gzqTy
+ Me9v/6ojBQOI7oXTMztXN6EnvJkLeU3o6BqNn2Uyb3IPPc86/Cxb8lOlzucqjG8VIaLNNJlqmUazyGSa
+ Z7IsEuj6sTR8NBWwCuB+9Zq+NbD3pUrvAYOgVh5sEAjvVEzuVy+eQJdMEnJ4YM8OIYNjnNxvGNODi+xG
+ RgLyEVM5gSWWNHgwY+SPLO7vKbR/FbJk28aJQysO87vnJ7fP9C3oN43DS4dVC0bUc0ZhqSPqMYOyof1H
+ g/sORfQfix84nsMWqZ2S716BMzdQA+vEniX95jm96mlS9iAyqV87m6dbMoprm7YcWXNb3vXeFb6CoEQK
+ xe/uU/s7Ty/cfHTxgret5007Zw8LeyfjU7YYI1MxktGBmBT9DejZ/J7r6JZVv8CEOmvUMWPUNWPOXLgw
+ tvI1q/S2lzfaxELK0lLN1ZXo7W356b0roz12dri4ozGUP9kI7Q5BewMQ1Ly1l8xffz27dn5i7QRvSY+7
+ qMdZMhpYMusTnGAK3NnLj2sZd3zDza/cU716R+neA5jvF8vCzAcTnIS97UIIStjYezOycGpgVou7rM5b
+ hY9uoCZ3iBObhlNbVjO7Jye3LEfWjVmLhJ5ZXfIUDAT1znntdkCuG4MXd6sll6sGx8CevYRfuYB0sESf
+ IKIsiSgbC5yFiTYSIaEDE0PC5BDqqnAVuKaSBlpb4dQJ+RdvdPPK7erIrlGJerfuyJ89q3jSVFUfrkxS
+ VyJoqOprwwzROsYYHXO0zikM+qwB6ao94cNTvYwsi9beE+3Dhq0jhMYhTCMPU81Bl3NQVVxMFRdVztau
+ 4OiUs3TKBhBFTK2CXlh+r1bxAKKcjawE0ueiKtg6Zf1a5WyduiFM/Qi+ZhhTwwM/i6rg7BebipnEjA5k
+ OhmZ0YFJa0cntejGN2jFNmpG16sdyG2zj2iQ8i2U8ylU/1qCeJ2l/CDx2LcKlbAW5Ndi+Z/F8omN2sDm
+ F/YY5dP1Yls0flaJ/6oTSelWL2EQKBybmdnLXK4TjXWSOnoqs0PLp/jQp5Ij+6vtGmA+NVq+VboRbQap
+ PRZZvdYZvZaZA1apfWaRHdjoLlxIG+prtcaHMtV3xcqfy5W/1agGNqpFtCnFUwAvV7N4BFU8Qiwc1c/i
+ EVPYuCQWLpGNje3XjmMpx7JE4zh/JnJ/S+3/PYt5tHkSO7Fzfgt6sAs92ISuzW+7MPkGILTXTCLLJpEF
+ E1qpPPmYvmORjEOx/ccS+48VcEVb55QH1nXHt0nTu5bj27asVXvqol3VmHlKLzaThS8e1GsZPzm0cn5+
+ /cq28DUExbe0vPG+b3r1rv2lW45eN+0ueNucdjU4aY/FEkVUYQfMTv5HaaONALo2sm05sErqmMM2TuEb
+ pvUap4xoAmfO6iva6M9vYa5wzH/iDcT0DOUMTeRcXBGBvp6FWe9uXyXmZb6BIKpQWL22FTu38mZ8+crU
+ luPkjhVv1YC9qN+/ZNC/ot8jIJGnTTqnXXNand4HE7zuK1+8KX/lhtLzF6Rgf6eqoierixkQlL0L+U8v
+ XhieNxpfIsyukxZ3zPgbFkvbp9cg11Whq2DXfmbr5Oiq2YCAQJ3T7ZrV6ZyDt02pNw6pl/ZoJJWpfwtW
+ ffRY9fJ5uIO55ik9DScL7WsX9S556jo5y7m7q3t6YJ3tiUQsXENFCoMSdXGR+xVGIDPODoy5Vzebh0Yg
+ XrxU83CVNydK6CNEidri+hhZUyMVc1NVCwPlEyQ1VwuEtxvy/WvdxGxcNV0flHuquEig17IBeEm/dtEA
+ ohAoe0CrsB+Wx9TM6dXI64Pn9MKy6LBsOrygX7eEg67gYSuHMCUD8CKmeiFTrYwNqxlC1A6jKrn7b4Oq
+ IWLNiH45h5jbg8ihIvLpILqjk9uBytVjGlWiGpUOgC0fgWWSX/MUf5ajk8GARb8TGCP6XCQf1AD/WaX6
+ uUAysFItrgmVTTXKZxql94BxD614mmpcp0p6q05dp/EY79zskNv8zDnBzmXqrHVyt0pgk1hQq7w/oB01
+ gdW7OkEgi9lhlNN7MrnbLJFqmkI3C2zQDWgAg9VY/waEbzX8SyXMt1Lbv0YnuA4e3aIW1SwVT5bO7lMv
+ 4mLyuLgUlk7igE4SC5XIQceztaMGFCKZxyOZ/4pi/k/OiGjzEoq9ZS+AbkMQUOQ7CHq0Dl2e2LDrAbzZ
+ GUCiIlVPY8qntHIHZRLA/DXjYGLfwfzBw218Bdaa1tQ2aQk6LYDOjm64sdfc22ccc0GNjW1cNmjSOmrD
+ E7jPrnourt+HoIiC4iv65oc9rhvdeOR49cEZrzsn3a6YuFzE65kewxn9x8efcPqYJ2vJmraMoKwAtKd2
+ 5f57DFM+gWudP8lYvl/LfPI57KSe9UGimQhM9y9Ftf82MpO8epng6YFyc9FsafaHoMaljWjO1P2BaQ+u
+ wGFo3WpwzZC9QurlE3vm8MxVPHlOq35Mq3HcLKXB7FUQ0uuR6sXbKp7Xle89xHz+ZBkd4TLMC4egMvCe
+ XN16PrfsOr98cnn95NqG7ca6496uuxDy2hZ6rO+dXdw6Ay7KvBXwKUGi8tGgY6JlVKGGJV9IUU4sU/se
+ rv7iJczLTcXWSNLWQPq6K/zbB6vgIKuIKPP0DJfM9Mv+39zPOevhUJLGBqIPHyGKK+0GJy9M8F36uTbV
+ tUaRYfC7NySszQ9aGB4yMzpmYipqbSdrf1bZzlHJ5oSc82nF614q77+ox+TB8ik6hf3wvAF4Th8sp08j
+ p1czu1czk6EB/tOzgcpZ8EymRipDLZWmlk6DZfXqFrIwxSDq87DVo7hyHqKUAysaUCseUC3jqFcOalUO
+ IisGMTXDejXDpFIWuqBPu5CJKB7AFPZhMijaMc0qkQ3yUY1yB9LrnL9kinzKUPpehAmqMQZJksgmfb9S
+ Vd8K+c+lsu/zJH9WakY14+Ja8VkMowKWccGgYQoTFdUKS6lDVzeZj3M9l8eurMxeXN+5NrbuWjkE6ALy
+ YWRZv0bJL7UgqQL/Wasb2oBLp1lHtxqFNhCi20jhoNeMjI2m4EPb0CC986Nex7cM7lcKi2hApnZiE1s0
+ I2tkEtuUMnpgyVS1BIZaMgueykEARHpsv1p0r3QU/WgE/fdoxm8lE7LUTaPBHdfxtev8pXtrq0+3t+5v
+ 711d2HblCix6Z0275y3IcyZdi8YdAlLdlFbFmHIhT6Z0UIw8p8hcVR/a0p2DzGeF9sObzqw11/YZu1Ku
+ RfmQdc3IqdZR297pM0MLjuwx56HRu9V1164/1Pa6Rbj52Mbr7gm3G0bnb5rYnYMRTP/75mNZ5vCN0VVP
+ 2hy+U6AKWFkAk1I+iSgaRxaPoeumzdpnL+R1nXsdgne/rXj5LsraXlEX9S93D/ibdza/fl1sbw8Da0TX
+ NjJ50y8ZE+69fLu+RYv+ddP+NQPmMomxQKDMosiz8NoxxbpxzYZxk+RGsxfB2peeKl97CLtyS+PuA+Sb
+ 10a/fpyidH4T7lVDUO7O9s+VpXsrS17raxe31j13Ny7tbl/eE16FoCt70MX1XefpFcvhZSPWMpE8CW/g
+ yTVwJauZEkUdMmmVKtEZut9/YS55SNqa/O11RubLE0xuolNTnVc3xbuf8ay77WVOyvXn94ztrSQ8XMUj
+ Igx7es9Nz5+bX7AfGT5B6TTKToe9fnX8nNvvzk4H7Z2OnnQ85OApcumB8pV7aq7npV1cJG7ekfsaqhZT
+ oZrSpZzGUE4Dau7VSOtVT2eop9PBo2b6/lONtP79k0hTTaJpZPQhQLNXAQdXyMGUAqGP46vGUKWDmoVs
+ lUKWStGAeglbq5KHqx7Sr+DqlbBwRUxkGQdVxgWBHwWe5vdhEtrVI5rkIppkDuS3ef7Ik/2YpvwpS+tV
+ knpojVFkg+HPMtjnMplX+aDFXO57OSy8ARtWh4prI+SzrPM41gk0/dBq1K8kzaRkHKXJhd5iPz7otr19
+ a37Do3XCNIepE0OR+1Z/7G3psW/1YNk0yrcUHliN8StFfi/VDalDRbYCeDQ+loIKbdUKaoEHNmt/K1X/
+ Xgz6t2zIUxeq++2z2/UyO1DJHWox7TKJdOX0/Y0XsES2ahygS/eIx1KPhHb8bxTl9+Ih+Y5FfQr/VBPb
+ prTBoKaOxOacWl4/t7LpNDpnzpm14C7aDq47je6cm4Y8xoUu7A1rKl+PPArvmlalr6j0bqqyd1GsTQPm
+ mlUX36p21KhiyLScd6Jy8FTLhC119lT3uGkH06yl9dTg4LOxudBn74yMToidOqtu6qSIPyGprfff9u5/
+ 1DQ78te9p9Yd2CsE2rJGmwDwgMB1FlE2gSofx9ZOmTVN2Rb32n5N1L3wQNzrtrL3be1LXqpv35mEx3iQ
+ OwL3dluhvfrllYSRuefMORfKvCFliUhdM6KvGtAExO55ZPssrHZCuZSnWDUGrxs1S2ywfBagceGBxOV7
+ StfuaNx/iHz1Uu/Te+Pc7Nvz89kQBBoKEnZXv2+vvN3bfgPtvtrbfgL6mXf2vIV713Z3PFY27KaWjHlL
+ hD4+aAdSbOBKtwxJtQ7K1PbIVrRrljcZRCdhrl3728P5rw9PtdKjrbrqL42wHk8Mvh5jfRns+dpc/jD2
+ 18kPj9UCvqnX1pgPDTssLNgtzVlP8czpXaSiIrUfv449ePyX9+1DF24edbz6l/uDw/e/yD/xUb3yQPL8
+ 5aMPnssEJGskNGkk9Mgn9MmD1eHxdKX4HoVEcKjKiT1q/xyV+G75eKpCbJdSUg8sg4nK6kfngk5dNqZ4
+ CFc1pVc5iSnkaeSylXIHVIG9ye/TLWERytiABkfIpaFBH2/FIBFIv4SFLOXgCweIaVREHFk5vEX2QBHZ
+ C9w4P2cqvk1VepGk5FcMxjGw38EQdIXyhyKZwDrEz3JYUJVWYIVGQLlaOtU4hWoc2UoMLMN+DFd9/k7s
+ 0wfxkADZ4hIMk2vHnnZoHDYp6AdkArCDV+xjudinMgDS0P5RpuNfjvQp1PpWph1Qgwit106n66fRAUUS
+ GBjAh0H+LNYIKtGmzlwXCH34m58nV1/SZi4W0oiJZKVEhkIa2MbIUo7pl4+mS0dRxKKpIgFt/wrtPJg+
+ IFs+hqoeMc7pIAQmiYfGHW7vwcwt28ysWnH5BtxFq/Et56ldj9kdrxUI/M0XR9ftWAIT+iyia1YJzA6T
+ BdLkecWOaa3OKVLLmGHNoEH1kFk527So17CGY9I2YkodNWdwbZhMV/7cy10opqTy2inHg1izA9iTB7RN
+ D1x5olza5Dy9fGtuzX164/TwGom9odu3ju5axjTOI6umdSqndGumAW3Gon7cNqP1xItfmudvSly8Ku/p
+ Je91RenyDdWIqAtzs0XAtwiFORtC/4nN67R5064FfOciiTyPbZ3VaZ7RqJtWLpuQLRlTqJqC1YKI3mT6
+ +Kfi2VtHLt6WvX5H4+59nQcPdJ49QX7xMe2ift93L3vJwp0w4Q6wQz8h6Ack/ArtvRPuPRDu3Njevriw
+ ajMkIPULkLQ5DfKYQueYHH1GuXdalcpT6+FiKQNmOSWYtx+Of/4on51s1VJ9Yaj/4fz4G/7Ep7kRv9nB
+ n0PU95Rq76YSR3LjaTbLbnzSep5vtsQ3mxo2onfjKqo0ouIlP347/uS9iPfLY+cf/Xn5zaEXIQrvQlUf
+ fpa+9uDvpx9FQ7NVUsiqSQzpWIZENEM6hi4bA/ibPdLxNPl4muL+6ZGL7ZaK7hSP7ZZPpmum0hFpfahs
+ Fi6XjSsawldPGVZO4AuGtHLZatlMtX0Hz9DJZ+LzGficHiwQenE/vopHAm6+hIMqZeML+vHpNFR8h2pY
+ i9yBiu6rsfUaPjmyr9Ok3mWr/KzC/KrDfavS8avQ+FamHtEMyAFIvxKFj1kiX/MlIxu1w+oQvyrQfrmo
+ 91Ga996L3Xj8+6svhyLTVau6jOr6zHIpmCwaKp2hG94GdgfIvi+U/lioFFSNCa0n+pXr/qpF+lfCkjoJ
+ 7QtnG6cdMhkGUW3I2Hbsr0LNyGLM6BLw2dEgnQdBUYu7nzvHzhX3EXO4YG+6CmDqhvdJB1NEwyhiUXSp
+ n60HgzqPRNEkwjulASsmqV0nIOt4ZPbf5H7tEYE+m48BtCrqvB5TcGJoyXl69cLq7tXlLY/JFduhFRPG
+ knbrrHTjvFgTX6J2TLx+WJk8QeiYMOuYPNE5Y9cxfbpxxLSWRWrlGvWOn+aMOk9OXtzaAElGv/Utv4rG
+ s1FZZiFZRr4JKMqIN3/v4diiMyj4z2xYTWzoj2/heZt45hqBsoZtXoJXz6lVTmvVzeLaFiybJhwjS0hX
+ nom6XDns4nnc8dyRW/fVq2peb23VQVCdUJi5CwWtCp9NrJ/tE5i2zeKbJpENE5p1s8pVc3IlM9KlM4qV
+ 8/C6aYPMDoOXoXKut/865y1x8Zri5evK3jeV79xVufdAPb/42tZ2CiSMgYShQmHgHhQohIIgcIDWtx/t
+ rl9b3zg3tWI5sIACbYxUvnLPnEL/gurQMmx0UXtyETu1aDQ2a9kFQJllWuWV+J5OVx7z9tz4c8H069X5
+ r+vzAatT/osjPnzuq1nenYnh8+NjpyemjGbm9RaXjKenDei9qOpGjZR8Of9Y8XdBIve+/n310++3f/71
+ Nk7sY7zUi2CROx/+9/WvPyPLpNN65JKY4jHM41GgPbtPLIx+LKLnWAxdHBBn/zli0d3HIjqPxlKlkuhK
+ gAgAuFfZLEIeh1Q4pF85YVwxpl86gisYBMN16kmdaikUeAYNlU1D5/Sg/xE6rm7EoGoIV8zSLWHji1mk
+ zB50HFktokn+QCPjZgmDGF6l8TFbxqdYI6xVL7LDADAtwpsB6whMQyNCG7XeZR95mfrbtyLRiAZ1/1KV
+ 9+nK75I1PiZpvwpXfeIv/SFSJixPI6+dVEw1TG7FRDVp7QOW6ADgq+FbpfI8XTygGhXZZhTYiA9uQH8r
+ VspjGA9BN3vXPSu4J1IppMQ2QkwltrTDbnnzBwQlQVAaBCULofBB/oNqllXhMDJlUDW6Xz6IKva16a8f
+ bYeCuo9/afzta8vvfu1/fmk4GNwhF9WuFFl7PK3xSA1dqp2n2Daq3DipWT+m0zSEZ0ydHJp3nl9x5687
+ T22cGtk2Afz8wsGDpZNHqxckKiYla8fVKHxD+sIp5qITa+XcwKpz79IpyrRp97gVY9SOO+4qWLyyu3dn
+ T3h3R/hgRfhgdO0mY86TuXSlX+DBW3IZX7MfW7OY2jKb3TWe3zUa3ySywFz9JpK8rl4jkKuel29c0mxf
+ wZEXLUqZpu8iZD3u/+589bfL98XTc8+vreeCNhXhXikEpUNQyB70YXXvFk9g3z6q3zSMbRiG104qV83L
+ lS/IVvCVqgHnf55YxjIMKoBff3fcxfuoo6fI2UuSF69IXr8p+/qdVmEZaFr2E+79EAq/QJCPEPoKCX2h
+ 3W/Q5ivh8pV1gcvcoiUH5NHBGPWCHBg+Ygjkh1Y15jbRixv4zW2T7W2L1Q3Lab7p8KjxEM90ctiFP3Zr
+ afrx6uzrDb7f1kLwjiB8eyFwa953Y+Hl7JQnb9h6eMJgdBY/No/jTuM7WToVnaqp1TKh+eJ+6cdfx/z9
+ JPqPZ/G/v884+DX7yMeUQ69j/sc3/Y/YZtEUulgSSySOcyyKdTyk93AA/e9g2t+h1EORPUej6ceie49G
+ Uv8Kp/wV1S0S1SUVSVZKooKIrlfIMykdNS8dNSnk6RcMEnJYANOlHktWS+iAJXVpp/cgcxiYXDq6qB9b
+ P2ZYN0Ks5uLqeYa1PLMqjnFhHza9U+tAE927nm2U2oz2y1faxxp16Ee3E32KlOM79eI69AJqNL+Xy77K
+ /P1L0aHwFqWMXkJgjfqrFLkP6Zo+2chPGTpfMrX8C7SCSjQTGkHyUg+ANX5WwMOadVNZxuGdup8rFF/k
+ SP6oQUaQTcBb6Hul1qvk42ntOM7W1d4lj5rBk3m9JsnthJwOM9rEjbkVn9WVkLWFyL3NRJAVnlh6Vc08
+ lcvBZvJ0Y5lq/p1i39uOBFOPh/eKhPYeD+47+q7xPz81/SugSyyCIpnSI1Y8IFrJFqlkiZazpSpG1KpH
+ EbU8LHXqxOjK2ZkNt9F1G+6GKYD1lI9KJ/X+Dsi9WWPHCiakamY12xaIrdOGNIEtd819YPUMdcGMMmfa
+ M2NDHbFljTuubV3dhS6vbDksbNrObtt1TxiA9v/OeRvmmhNn2WFo1XZ4w3Jq12oeslqAzCa3iexVBH0d
+ 1rGu3Lgs07om17Wt0rGu1rEM+qgMSnv0gkDz8y/piDQ9Fg/kiLIgKAeCQGYwFoJ+CaFPmztPJvie/ZMO
+ AzO21Am95jH12mnZqnmZqnmFmln1VgGudc6kpN/kZybi+lupM5cPOnkdcvb86+od8ZRs65Gpuxs7L3f3
+ Hu1Bd/f27gih+9DeA2jrgXD16ta83TLfbJwPhv+VugQSbYvHyQsiPQKpkQ3Y0iZhY9NQuGUObZrvbpps
+ rBuvrZisCsxX5+zXprxWJ2+uTj9bm/HdEoRB64nCjfi9jUjh9s/1tacTU659g/oUjlb3qHbHqE4tR6OI
+ rpjeLh3bIB5Sfdyv5IhPyaGPRX/4lP75o/LvnxUHAyoPhjYfius5ltR/PGVQFOA1IwaO/qL//ZN2MJB+
+ MKT7j+DuP8Jof0bQ/4yg/RFG+TOk/XBg87HABpnodp30Pr2iYfPiEctCnmkuWy9rACzGAjAjrYRu7aRu
+ nZRuxL7Q6SDfgitkYisH9ZtGjGizp3rnHemzzn38s8wF565JmwN1lCuVvXpxNTo/C1QT2w2imvAxbfjg
+ OlhUCzq0QTuwFuCKJN/nH/atEI3q0MhmGYY3677NkH+brvwmTeV1qpJfkfbPUs1fZWpB5WphNYiQWuKP
+ cvT3Kt0oil7agGlAs9bbAtkvFfBoikVYq75PkeqbFNEMMr5nzq1z0rGCaZZHN4xu1E1v1+8c8+zheTe1
+ exUVOdN7n67thvIEz0rpJ/LZ+gVjBtG9moFdUvFspbQh5WSebPqEQvyw1Lu6Az4tfwVQxCLp4hks0SLe
+ kdKhIyWDogVsqcw+0NelkD+g3TF7Ymj73OiuW/ecUc2IVuWoej5XLoryt1/Lf/t3/Z7IEauYh9XPo8qH
+ dVpmjegCu65Zi6ZxYs0QoXHYFKDWKUPWM2vnFredxlf0mbMw2qxGw5hq9TiyVWDas3aaseRAm7ftXbYa
+ 3DoxsWs1um0wtIVhbQLgm1rbqmLzsmz7uhx1V4GyLt29okhf1mXwTVvYFuUdJi29ztOCV1u7Pzd3fmzu
+ +G7tft6DPu4K32xtvdjcerm283J999Hi9oWhFWMqH3SkyNTPyDbOqrUvojuXDVrmTuTRzL+lIa59EPV6
+ fujqi79ffpdrpjuv7Hkv71xY3nRe2bKfX7RcXjm9s+a0t+y4y7ddnTJdWdCfWtTqF0h1LB5q4P/ROHew
+ gy/KXladXcWsrhntbVhCGxZ7q/pbK4SNZdLmouEW/8TWnMvatBd/+AZ/7PWGIES4mSTcSRHuJQNjuQd9
+ XVm7xx492TqAqB/UrOCpFXKVsvtlM3plUmhS8d0SoWSRgNZjfk2H/Br+9G/6I4T8d3QPwD0AlYuCA/qR
+ kthS0UzxgO5DPykHAyl/hVD+COr6PZT6r32h0/8MpfwV0HrYv0kEjOpHk5HJdL08rnke1zSbZZA1QARo
+ gGQa2AMHYF3IZCoirQeZSUdl0pC5DExeL6Z4AExpWPTPOQ7Mn2PPewwJLo0uew0tnD9Q2nYhqx0ZWKT+
+ LVctoAwZUoOMaUMnUTDfimU+5Ij7lsq/zhb5WAhWp8v4lMtFtKCC6pAf85RfZyg8jBG9EXzwVYrUp2zZ
+ kCpYYKmGfwk8ttUqpNbkWzHqVxU6m3Minop/ny/7IksuGKQsq1DvsxX88pQqmJbdk851LPMsMiaZjPEv
+ UwooU44BcxvVpNgC0s8YnaRCs67hmy2889mdBoWDxmXT5vFMeBhVNm1IPWNYNYkjnT2plMST/VD7L58m
+ kbAeJVC9T2Mfyx78V+Ho36C8n8WRBtedmG7pJLpq5bgRdcWxd9WZPGNZyNDKpqnn9KnHdcm+Lv7tWcl/
+ BHYfyh1XrAYLaSdhZTx4BRezf2HnAuKkTjYdVUDDVvRi65hoyii+l49sn5RqmBCpmpQpn4HVLhKbl82b
+ pk81jts2T1u3zZp2zBOpoDN2A07bUCWvKjYK5Bv48u1LQOWylGXR3lVpzjocVGqY0xbcORdA/hid8xyZ
+ 8RgYcWBwTw5OOArWrm5sP9jdBd28fiC6QxCwH/dWheATA0WZl2mflyUvqJEFOuQVQuuiWe3EiRy6aWQN
+ KrxCO7oCltOGo0xYc1Ys+vmE/lkd+phKz6DS4ITO3Ize6qTx6qjxwrDhyqzB3DKCtSTTNv9X3dzvdfMH
+ G2eOkqel+uc1JpYIq6vmOxvmexuG2+v49WXcmoC0sWC6wbdZmXMVTF8VzDzdXvOH9mKApYSgFCB0IeS7
+ B72aWznPmDZuHNPN5SpmceUyWNIZbOkUlnTigFRsv1QoXfRHx2Hf5j++t/we3PVXVN/h2P6jgLaZ2CuV
+ yVZO61OMp0pFkEWDWo+Edx0L6/orjPJXRM9fUfSDEbQ/QygHA9pEAJ85olM7tgsX0wny0chYQOSiIOMo
+ OrFdsMQe7RQaMo0G1gcBlaNz+nFA6+mALQegnxzD1jEAz3Kmj7vRR90Gxj1G+V7gV30gs8EpvEr5Z5Hm
+ lxyND9lqwXUoQG2M74T5FIu9SD30PPXo4+Sjb/NlQGB+nScd1ID2LdP6Uqb5pVTjWbLEg6jDr5JFP6RL
+ BFVoRtWjA0oR8S3W4bUWAWWkgHJUYhdxP7XSjnqTKRNYi4voMHqXLf89X7F6wJI+51zHNslo1wmqVPhW
+ IuVbIuFfLhdYqhJejoipJEWWETLbzTM7jdIpuOx+bEqfVnSPUkSPDGi9SmHJgl9Wzrgi2Ob1qfL4zxaV
+ aLp2xjCseEa2ZOZQyfTfBVOiOWNyWcPKGUPqKRyNfND2ueLAWDnLXr1In3Upo5OyOrUS2tX8aiTflv3x
+ tfG31EGJ0mn54lGZHKZUUpdkcrdiKgOWRNdK6tHO6kOUczFNI4TOaSJIZrfMS9QtHCubl8ifVSxd1q3b
+ MKqds66bdKyetC3m6RcOagIAAXlDsXVNqmlJumZOtmpCrmUOzG7KM9YlBrdlpiGtuV2DuU2QFHKdFDhN
+ LNoMz5sxhtFtfSq0IcTEovXa9pV/al4gYRLxT87k8S7kxN/FDG0rM7cV6VsqXeuwzk1C64pB7Zxh5ZRx
+ 5YRB06xB+wKJzMeQBcjaKeXGGfnWKcnGoaN1/YfoI8p8gcHu8gkQ0bfmnbaXbBZWcGyBbPvcX/Vzv9Xx
+ /6ydO9Q0J9a9qDy0iuGvG21vm0GQKSQkbW5gNleJW2umK6tma2sOezu3drae721/3lz7srX2HdoNgaBg
+ CPoEQc+3oMsj61bgXy8cVcocks4akswalkoblk4dkU4Zl0sekw8bEP/RdfB7x+/+lN9C6L9H0v9M7BMr
+ HdchL5p2CMyap/RqR7GVw+jiEVg8QySqByz2AdSTY8DYJLBATkYhsgcWS0Mn0QhxFHRQo5p/rWJQk3J4
+ m0osRSO1X2c/89iHyWZicgYweWxcLqBi9KKyerGFHNAAY9055tA9eraL69I34jGzdG1i1uNAVrObf5HC
+ +1TlT5mw95nq30o0I1p1AmoVPhdJPE87/DT5yMsMcdCT+CJT5mWW4utMtceJCh8KYN/KEa8z5D9lK7xL
+ k3yTLPo9TzGqBhVUovstE/M1Ax9cZuxXgPhWoprWq5/LNv+UB4ZKAYbO7mO+8rsU8YxWbB3bIp+CSm6F
+ BVXL+ZVLgdbfoHqFgCp5/xKliFpsRA0hsh4X24EDNXlw7/5Wefhn4/HIHvmYHtlUllwiUxz0nQe1ir3O
+ FfWtAts1dDIHkVUL8BqBeOH47wXjoqUz8NIZdP4EGmzijWeo5XEAa9dxcvvFCvRtRPCIMnq2mG4c1aj+
+ vea4T/X/hHcezGJLgD6wnH7phC7xJKpiVr8uqFMkUtSA82meIA2snBzasqXOY+onpavmjhfOHk8bE8sY
+ Vyzl4xuX7Jv4Z8vGbHI5hAyQ1h2WLpuVqOCLlc9KlE7KFA3JVI/Ldy6q9q4qD22q8iH0OmS6Bzlv751f
+ 3nBa2LCYXiWyZjQoI9Kdw/LsedzitssedAeCQEeADyj0CqHr25DtGkSYgTSYmzLUNTnKBqx1BV09iyqe
+ QJVNYeoFuI41dNeGduOSXMmUSPbokeKpY2Xjh0rZv1f1/UUbUeEvWUC7HtD2LWjnCbR3c23Tjjuv1T52
+ tGb09/LR30pH/6qbkeha0GAv4WfXTLd3rCHIQrhDWltFrazg1zeNljdNBGs2S+sXBEs3Bgc9u9rPdLc7
+ jbG8N5efQ8JHQujmmtCdu2rSPAMvHJVLAVgetnjaoFTqsEzaqEzKiEzqmHIMV96/+9i39j/82n8LoPwe
+ 0f0XYN5TViwnII9Z6NLk7oXhddeRnXN9m6dKRjSTesWTB8QzhqSzxxTTeCopLHhCLyqOhomjYMLbdb7X
+ KvlUSvk3A/q+WlSHShwF1E11s3sJQOuZTGQaA5ECEF9dyOgWncQOTCHNpIxqnt1IyqkjNFKt2GMunCGH
+ /wd6M4989rNkxAAAAABJRU5ErkJggg==
+</value>
+ </data>
+ <data name="panelHeader.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+ <value>
+ iVBORw0KGgoAAAANSUhEUgAAAAcAAABWCAIAAACM4hvWAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8
+ YQUAAAAgY0hSTQAAeiYAAICEAAD6AAAAgOgAAHUwAADqYAAAOpgAABdwnLpRPAAAAAlwSFlzAAAOxAAA
+ DsQBlSsOGwAABnlJREFUSEstlFlvG9cVx6/0Bbxq4b7NcBZKceA4TRAkafoBnMiy5TayaCMFWqDoQ4H2
+ KZZkiZydiyQuWl03iR2L2i2JywxnOCuHHJKS7TZp2iBAP0Qf8hQgyJUT4GCA+5v//M85c++5fdI/+f//
+ +L/vv/8B9Pf/2Af6+vr6+/vBhnoz136NUUhKiyW1Udq4wlivg8/NKb6Ozxyg94+wT8vYp8f4dIUAX1jx
+ VIN8cITNVLDpCna/gs3WCPC4GU9r5FwZe1DFZ2v4TI2Yk2Lgy9bdrDGSqOJJiZyXCPhiTiLBUye+aMWo
+ GpaUiESdmKvjCZkEm87dJYukqmhCxOYlbF7GEgoBaXzRJF9p8VcUp9QY2OzE861RViZhyUmFSMLCId3q
+ 3i06r3EKwSokrZK0PsLoMVDqxgttkqlHmQbO6gRrxjhzBDpMFZwYK2OMgv1CrRFQ6kwVOzFeI1kVp3Wc
+ NUjOgtm6Hxe6uHC2xhgTp034xECpN5nvYIIeZU2MbeKMFaVNFOy8iBd7BKdhtIbCNQymGQU7L+8Vu9AX
+ pzSUMaMQcTYKtp7fyzkEp2K0jlEGCh14Owq2T+8uOTir4ZxF0DqUo1wTBaXTqWwLpRswG84YEGFCCwel
+ F1MLDkopEVpDGCPKW3imRYLS8zuQ0o0IpaO0DimRsSE9mVxqR9kGRjWwJHQ3CMHEwdbJZB5SBaMUPAmb
+ 1jBWQ8HuyZ3lNibAH6aRjE5Ayr2i8RWHSKkEr8d4IwYdeAP21ouvOmS6gfMqwekwMN7AfqGCAj/EeB3n
+ 9WjqLFtnarlFpOQo38B4DRO0aAZqt7rxYgsXIFVQXkXSWjT7My3YOC+hvIzwCpJRo4s6Dna68WUb48Ww
+ IIfTDWRRw4vmKNjrTK3ZWEoKpeRw6mcKd3O/G3/YIjL1SFqJpBtoVo0uGTjYd+48tPFsHcmc0UhaQ7IG
+ CunkmhXNSJG0DK2DvOJPa0Fw0L2z3kQyUkiQQ7wS4GQvr/og/Xi9GUqLfl4KMJKfkTyc7AYHvd+u28GU
+ 6OVEHyN6GdHFSsOwslsrlo+vupmKm666WdHNiMNgq3k9rwxyZQ8Do+aBiJWGQMm8npMHmWcu+hiq3Iw0
+ xNaHwHZzLN9w04du6niYqg7S4mVWvAy2W+M5xUMfuqjycLJ6mREvsrULkN7MNbyQ0pWhZO0SK13k6+fB
+ Vmsip/qZYw9VHpgvn2PEc2d0s3VzSfMyxy6qMpCsnufkc4J8ATy1byyqbqYMUw2w9csZfSCjDUDteE73
+ CqKXl4fZ+qWUNpBRB0CpfbNg+SEVZDfU8sqlFKTbnYkVO5Sq+3nZzSuDEKWhQ6l9e9mOZBvhlOrLGp4F
+ czhrDoKnzVsFI7ygohk9mG16l2xXvuUCT61bOS2YbUAaXrADha5vpeuBdDwPVQ00pYUW26G10/D6SQCU
+ 7I/ypj8F91EPLzmRtVNk4yQMtttjBcvLK6GMhS610eUusuqEzmix6WFlH2+Esna46ESKVhDs9m7kmi5K
+ dHNGINsKFdpIEc7x/uk4LIWquwTLv+hALVrQ4XT3PspZA0lxkNM92VYwf9ZRCJ7J6wva+UTlIqe5M61A
+ xvCxVQ/0HSu2XLQ4yBuetO0XNC9d9ULt+GonICh+WEC+h6bNECMFwU7n9mobTSmRTBMtnpKwF1YMg93u
+ 1GqLFGREMCIZG+FUeAbhZHUmCxbCSQjXQBJ1H90IpfQRsNUbzze9rBhklEii5kvWA4JGgO2TD3PNIbrm
+ ZeRQUvTPHA1TYhDsv/xwxRnkJDcleuZF7/Th0OyhC9T+M/74K19W8ybKQ4my58GRa/bZMJC/nSh9E1rU
+ fFTFM3/knd5zTe8Pg+q/bn1x6s/KAaYSnt71/eWzC397chEcODfWTY9wHOCOcb76+uwu8tfHl0BJH8uL
+ bv5ZeEG69uXLse3/TqSlUbCl32Z3BujtSKb8ZkH9ze43v9tovgcOnU+yz/xUCaF3R2ee4CvqBxvmr0G5
+ +/tVeH3vhOc2I/efRNKVN3LKr0D9xR//YRDcvn+25JnfQxeUN5f0t4D19Z+rX7+zphDUno+rEKvtdzd6
+ 74PmV38yvvtg076WPoxsNN961Hvv7513gP78D/K/3/1MvbJwFH3cef9h8+1Hztug5tzb7VzNl/HUAZYT
+ ry6rVx8518COObGmIAtlkt0nkntYUX/j89NrYNeeXCyHEpsIvRNL7OCpKrnevvITI0lytudA8kUAAAAA
+ SUVORK5CYII=
+</value>
+ </data>
+ <metadata name="saveFileDialogExport.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
+ <value>17, 17</value>
+ </metadata>
+ <metadata name="menuStrip1.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
+ <value>171, 17</value>
+ </metadata>
+</root>
\ No newline at end of file diff --git a/tools/stats_viewer/win32.cs b/tools/stats_viewer/win32.cs new file mode 100644 index 0000000..f09afa48 --- /dev/null +++ b/tools/stats_viewer/win32.cs @@ -0,0 +1,75 @@ +// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+using System;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace StatsViewer {
+ /// <summary>
+ /// Win32 API constants, structs, and wrappers for access via C#.
+ /// </summary>
+ class Win32 {
+ #region Constants
+ public enum MapAccess {
+ FILE_MAP_COPY = 0x0001,
+ FILE_MAP_WRITE = 0x0002,
+ FILE_MAP_READ = 0x0004,
+ FILE_MAP_ALL_ACCESS = 0x001f,
+ }
+
+ public const int GENERIC_READ = unchecked((int)0x80000000);
+ public const int GENERIC_WRITE = unchecked((int)0x40000000);
+ public const int OPEN_ALWAYS = 4;
+ public static readonly IntPtr INVALID_HANDLE_VALUE = new IntPtr(-1);
+ #endregion
+
+ [DllImport("kernel32", SetLastError=true, CharSet=CharSet.Auto)]
+ public static extern IntPtr CreateFile (
+ String lpFileName, int dwDesiredAccess, int dwShareMode,
+ IntPtr lpSecurityAttributes, int dwCreationDisposition,
+ int dwFlagsAndAttributes, IntPtr hTemplateFile);
+
+ [DllImport("kernel32", SetLastError=true)]
+ public static extern IntPtr MapViewOfFile (
+ IntPtr hFileMappingObject, int dwDesiredAccess, int dwFileOffsetHigh,
+ int dwFileOffsetLow, int dwNumBytesToMap);
+
+ [DllImport("kernel32", SetLastError=true, CharSet=CharSet.Auto)]
+ public static extern IntPtr OpenFileMapping (
+ int dwDesiredAccess, bool bInheritHandle, String lpName);
+
+ [DllImport("kernel32", SetLastError=true)]
+ public static extern bool UnmapViewOfFile (IntPtr lpBaseAddress);
+
+ [DllImport("kernel32", SetLastError = true)]
+ public static extern bool CloseHandle(IntPtr handle);
+ }
+}
|