1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/renderer/safe_browsing/phishing_url_feature_extractor.h"
#include <algorithm>
#include <string>
#include <vector>
#include "base/logging.h"
#include "base/perftimer.h"
#include "base/metrics/histogram.h"
#include "base/string_split.h"
#include "base/string_util.h"
#include "chrome/renderer/safe_browsing/features.h"
#include "googleurl/src/gurl.h"
#include "net/base/registry_controlled_domain.h"
namespace safe_browsing {
PhishingUrlFeatureExtractor::PhishingUrlFeatureExtractor() {}
PhishingUrlFeatureExtractor::~PhishingUrlFeatureExtractor() {}
bool PhishingUrlFeatureExtractor::ExtractFeatures(const GURL& url,
FeatureMap* features) {
PerfTimer timer;
if (url.HostIsIPAddress()) {
if (!features->AddBooleanFeature(features::kUrlHostIsIpAddress))
return false;
} else {
std::string host;
TrimString(url.host(), ".", &host); // Remove any leading/trailing dots.
// TODO(bryner): Ensure that the url encoding is consistent with
// the features in the model.
// Disallow unknown registries so that we don't classify
// partial hostnames (e.g. "www.subdomain").
size_t registry_length =
net::RegistryControlledDomainService::GetRegistryLength(host, false);
if (registry_length == 0 || registry_length == std::string::npos) {
DVLOG(1) << "Could not find TLD for host: " << host;
return false;
}
DCHECK_LT(registry_length, host.size()) << "Non-zero registry length, but "
"host is only a TLD: " << host;
size_t tld_start = host.size() - registry_length;
if (!features->AddBooleanFeature(features::kUrlTldToken +
host.substr(tld_start)))
return false;
// Pull off the TLD and the preceeding dot.
host.erase(tld_start - 1);
std::vector<std::string> host_tokens;
base::SplitStringDontTrim(host, '.', &host_tokens);
// Get rid of any empty components.
std::vector<std::string>::iterator new_end =
std::remove(host_tokens.begin(), host_tokens.end(), "");
host_tokens.erase(new_end, host_tokens.end());
if (host_tokens.empty()) {
DVLOG(1) << "Could not find domain for host: " << host;
return false;
}
if (!features->AddBooleanFeature(features::kUrlDomainToken +
host_tokens.back()))
return false;
host_tokens.pop_back();
// Now we're just left with the "other" host tokens.
for (std::vector<std::string>::iterator it = host_tokens.begin();
it != host_tokens.end(); ++it) {
if (!features->AddBooleanFeature(features::kUrlOtherHostToken + *it))
return false;
}
if (host_tokens.size() > 1) {
if (!features->AddBooleanFeature(features::kUrlNumOtherHostTokensGTOne))
return false;
if (host_tokens.size() > 3) {
if (!features->AddBooleanFeature(
features::kUrlNumOtherHostTokensGTThree))
return false;
}
}
}
std::vector<std::string> long_tokens;
SplitStringIntoLongAlphanumTokens(url.path(), &long_tokens);
for (std::vector<std::string>::iterator it = long_tokens.begin();
it != long_tokens.end(); ++it) {
if (!features->AddBooleanFeature(features::kUrlPathToken + *it))
return false;
}
UMA_HISTOGRAM_TIMES("SBClientPhishing.URLFeatureTime", timer.Elapsed());
return true;
}
// static
void PhishingUrlFeatureExtractor::SplitStringIntoLongAlphanumTokens(
const std::string& full,
std::vector<std::string>* tokens) {
// Split on common non-alphanumerics.
// TODO(bryner): Split on all(?) non-alphanumerics and handle %XX properly.
static const char kTokenSeparators[] = ".,\\/_-|=%:!&";
std::vector<std::string> raw_splits;
Tokenize(full, kTokenSeparators, &raw_splits);
// Copy over only the splits that are 3 or more chars long.
// TODO(bryner): Determine a meaningful min size.
for (std::vector<std::string>::iterator it = raw_splits.begin();
it != raw_splits.end(); ++it) {
if (it->length() >= kMinPathComponentLength)
tokens->push_back(*it);
}
}
} // namespace safe_browsing
|
