1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/renderer/safe_browsing/phishing_url_feature_extractor.h"
#include <algorithm>
#include <string>
#include <vector>
#include "base/logging.h"
#include "base/metrics/histogram.h"
#include "base/strings/string_split.h"
#include "base/strings/string_util.h"
#include "base/timer/elapsed_timer.h"
#include "chrome/renderer/safe_browsing/features.h"
#include "net/base/registry_controlled_domains/registry_controlled_domain.h"
#include "url/gurl.h"
namespace safe_browsing {
PhishingUrlFeatureExtractor::PhishingUrlFeatureExtractor() {}
PhishingUrlFeatureExtractor::~PhishingUrlFeatureExtractor() {}
bool PhishingUrlFeatureExtractor::ExtractFeatures(const GURL& url,
FeatureMap* features) {
base::ElapsedTimer timer;
if (url.HostIsIPAddress()) {
if (!features->AddBooleanFeature(features::kUrlHostIsIpAddress))
return false;
} else {
// Remove any leading/trailing dots.
std::string host;
base::TrimString(url.host(), ".", &host);
// TODO(bryner): Ensure that the url encoding is consistent with
// the features in the model.
// Disallow unknown registries so that we don't classify
// partial hostnames (e.g. "www.subdomain").
size_t registry_length =
net::registry_controlled_domains::GetRegistryLength(
host,
net::registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES,
net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES);
if (registry_length == 0 || registry_length == std::string::npos) {
DVLOG(1) << "Could not find TLD for host: " << host;
return false;
}
DCHECK_LT(registry_length, host.size()) << "Non-zero registry length, but "
"host is only a TLD: " << host;
size_t tld_start = host.size() - registry_length;
if (!features->AddBooleanFeature(features::kUrlTldToken +
host.substr(tld_start)))
return false;
// Pull off the TLD and the preceeding dot.
host.erase(tld_start - 1);
std::vector<std::string> host_tokens = base::SplitString(
host, ".", base::KEEP_WHITESPACE, base::SPLIT_WANT_NONEMPTY);
if (host_tokens.empty()) {
DVLOG(1) << "Could not find domain for host: " << host;
return false;
}
if (!features->AddBooleanFeature(features::kUrlDomainToken +
host_tokens.back()))
return false;
host_tokens.pop_back();
// Now we're just left with the "other" host tokens.
for (std::vector<std::string>::iterator it = host_tokens.begin();
it != host_tokens.end(); ++it) {
if (!features->AddBooleanFeature(features::kUrlOtherHostToken + *it))
return false;
}
if (host_tokens.size() > 1) {
if (!features->AddBooleanFeature(features::kUrlNumOtherHostTokensGTOne))
return false;
if (host_tokens.size() > 3) {
if (!features->AddBooleanFeature(
features::kUrlNumOtherHostTokensGTThree))
return false;
}
}
}
std::vector<std::string> long_tokens;
SplitStringIntoLongAlphanumTokens(url.path(), &long_tokens);
for (const std::string& token : long_tokens) {
if (!features->AddBooleanFeature(features::kUrlPathToken + token))
return false;
}
UMA_HISTOGRAM_TIMES("SBClientPhishing.URLFeatureTime", timer.Elapsed());
return true;
}
// static
void PhishingUrlFeatureExtractor::SplitStringIntoLongAlphanumTokens(
const std::string& full,
std::vector<std::string>* tokens) {
// Split on common non-alphanumerics.
// TODO(bryner): Split on all(?) non-alphanumerics and handle %XX properly.
static const char kTokenSeparators[] = ".,\\/_-|=%:!&";
for (const base::StringPiece& token :
base::SplitStringPiece(full, kTokenSeparators, base::KEEP_WHITESPACE,
base::SPLIT_WANT_NONEMPTY)) {
// Copy over only the splits that are 3 or more chars long.
// TODO(bryner): Determine a meaningful min size.
if (token.length() >= kMinPathComponentLength)
tokens->push_back(token.as_string());
}
}
} // namespace safe_browsing
|
