xref: /webtrees/app/Http/Middleware/BadBotBlocker.php (revision edf6ae5b0d35e10e7e82da63bf2168c85a8df026) 
1<?php
2
3/**
4 * webtrees: online genealogy
5 * Copyright (C) 2021 webtrees development team
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 */
17
18declare(strict_types=1);
19
20namespace Fisharebest\Webtrees\Http\Middleware;
21
22use Fig\Http\Message\StatusCodeInterface;
23use Fisharebest\Webtrees\Registry;
24use Iodev\Whois\Loaders\CurlLoader;
25use Iodev\Whois\Modules\Asn\AsnRouteInfo;
26use Iodev\Whois\Whois;
27use IPLib\Address\AddressInterface;
28use IPLib\Factory as IPFactory;
29use IPLib\Range\RangeInterface;
30use Psr\Http\Message\ResponseInterface;
31use Psr\Http\Message\ServerRequestInterface;
32use Psr\Http\Server\MiddlewareInterface;
33use Psr\Http\Server\RequestHandlerInterface;
34use Throwable;
35
36use function array_map;
37use function assert;
38use function gethostbyaddr;
39use function gethostbyname;
40use function response;
41use function str_contains;
42use function str_ends_with;
43
44/**
45 * Middleware to block bad robots before they waste our valuable CPU cycles.
46 */
47class BadBotBlocker implements MiddlewareInterface
48{
49    // Cache whois requests.  Try to avoid all caches expiring at the same time.
50    private const WHOIS_TTL_MIN = 28 * 86400;
51    private const WHOIS_TTL_MAX = 35 * 86400;
52    private const WHOIS_TIMEOUT = 5;
53
54    // Bad robots - SEO optimisers, advertisers, etc.  This list is shared with robots.txt.
55    public const BAD_ROBOTS = [
56        'admantx',
57        'Adsbot',
58        'AhrefsBot',
59        'AspiegelBot',
60        'Barkrowler',
61        'DotBot',
62        'Grapeshot',
63        'ia_archiver',
64        'Linguee',
65        'MJ12bot',
66        'panscient',
67        'PetalBot',
68        'proximic',
69        'SemrushBot',
70        'Turnitin',
71        'XoviBot',
72        'zoominfobot',
73    ];
74
75    /**
76     * Some search engines use reverse/forward DNS to verify the IP address.
77     *
78     * @see https://support.google.com/webmasters/answer/80553?hl=en
79     * @see https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0
80     * @see https://www.bing.com/webmaster/help/how-to-verify-bingbot-3905dc26
81     * @see https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
82     */
83    private const ROBOT_REV_FWD_DNS = [
84        'bingbot'     => ['.search.msn.com'],
85        'BingPreview' => ['.search.msn.com'],
86        'Google'      => ['.google.com', '.googlebot.com'],
87        'Mail.RU_Bot' => ['mail.ru'],
88        'msnbot'      => ['.search.msn.com'],
89        'Qwantify'    => ['.search.qwant.com'],
90        'Sogou'       => ['.crawl.sogou.com'],
91        'Yahoo'       => ['.crawl.yahoo.net'],
92        'Yandex'      => ['.yandex.ru', '.yandex.net', '.yandex.com'],
93    ];
94
95    /**
96     * Some search engines only use reverse DNS to verify the IP address.
97     *
98     * @see https://help.baidu.com/question?prod_id=99&class=0&id=3001
99     */
100    private const ROBOT_REV_ONLY_DNS = [
101        'Baiduspider' => ['.baidu.com', '.baidu.jp'],
102    ];
103
104    /**
105     * Some search engines operate from designated IP addresses.
106     *
107     * @see http://www.apple.com/go/applebot
108     * @see https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
109     */
110    private const ROBOT_IPS = [
111        'AppleBot'    => [
112            '17.0.0.0/8',
113        ],
114        'Ask Jeeves'  => [
115            '65.214.45.143',
116            '65.214.45.148',
117            '66.235.124.192',
118            '66.235.124.7',
119            '66.235.124.101',
120            '66.235.124.193',
121            '66.235.124.73',
122            '66.235.124.196',
123            '66.235.124.74',
124            '63.123.238.8',
125            '202.143.148.61',
126        ],
127        'DuckDuckBot' => [
128            '23.21.227.69',
129            '50.16.241.113',
130            '50.16.241.114',
131            '50.16.241.117',
132            '50.16.247.234',
133            '52.204.97.54',
134            '52.5.190.19',
135            '54.197.234.188',
136            '54.208.100.253',
137            '54.208.102.37',
138            '107.21.1.8',
139        ],
140    ];
141
142    /**
143     * Some search engines operate from within a designated autonomous system.
144     *
145     * @see https://developers.facebook.com/docs/sharing/webmasters/crawler
146     * @see https://www.facebook.com/peering/
147     */
148    private const ROBOT_ASNS = [
149        'facebook' => ['AS32934', 'AS63293'],
150        'twitter'  => ['AS13414'],
151    ];
152
153    /**
154     * @param ServerRequestInterface  $request
155     * @param RequestHandlerInterface $handler
156     *
157     * @return ResponseInterface
158     */
159    public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
160    {
161        $ua      = $request->getServerParams()['HTTP_USER_AGENT'] ?? '';
162        $ip      = $request->getAttribute('client-ip');
163        $address = IPFactory::addressFromString($ip);
164        assert($address instanceof AddressInterface);
165
166        foreach (self::BAD_ROBOTS as $robot) {
167            if (str_contains($ua, $robot)) {
168                return $this->response();
169            }
170        }
171
172        foreach (self::ROBOT_REV_FWD_DNS as $robot => $valid_domains) {
173            if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, false)) {
174                return $this->response();
175            }
176        }
177
178        foreach (self::ROBOT_REV_ONLY_DNS as $robot => $valid_domains) {
179            if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, true)) {
180                return $this->response();
181            }
182        }
183
184        foreach (self::ROBOT_IPS as $robot => $valid_ips) {
185            if (str_contains($ua, $robot)) {
186                foreach ($valid_ips as $ip) {
187                    $range = IPFactory::rangeFromString($ip);
188
189                    if ($range instanceof RangeInterface && $range->contains($address)) {
190                        continue 2;
191                    }
192                }
193
194                return $this->response();
195            }
196        }
197
198        foreach (self::ROBOT_ASNS as $robot => $asns) {
199            foreach ($asns as $asn) {
200                if (str_contains($ua, $robot)) {
201                    foreach ($this->fetchIpRangesForAsn($asn) as $range) {
202                        if ($range->contains($address)) {
203                            continue 2;
204                        }
205                    }
206
207                    return $this->response();
208                }
209            }
210        }
211
212        // Allow sites to block access from entire networks.
213        preg_match_all('/(AS\d+)/', $request->getAttribute('block_asn', ''), $matches);
214        foreach ($matches[1] as $asn) {
215            foreach ($this->fetchIpRangesForAsn($asn) as $range) {
216                if ($range->contains($address)) {
217                    return $this->response();
218                }
219            }
220        }
221
222        return $handler->handle($request);
223    }
224
225    /**
226     * Check that an IP address belongs to a robot operator using a forward/reverse DNS lookup.
227     *
228     * @param string        $ip
229     * @param array<string> $valid_domains
230     * @param bool          $reverse_only
231     *
232     * @return bool
233     */
234    private function checkRobotDNS(string $ip, array $valid_domains, bool $reverse_only): bool
235    {
236        $host = gethostbyaddr($ip);
237
238        if ($host === false) {
239            return false;
240        }
241
242        foreach ($valid_domains as $domain) {
243            if (str_ends_with($host, $domain)) {
244                return $reverse_only || $ip === gethostbyname($host);
245            }
246        }
247
248        return false;
249    }
250
251    /**
252     * Perform a whois search for an ASN.
253     *
254     * @param string $asn - The autonomous system number to query
255     *
256     * @return array<RangeInterface>
257     */
258    private function fetchIpRangesForAsn(string $asn): array
259    {
260        return Registry::cache()->file()->remember('whois-asn-' . $asn, static function () use ($asn): array {
261            try {
262                $loader = new CurlLoader(self::WHOIS_TIMEOUT);
263                $whois  = new Whois($loader);
264                $info   = $whois->loadAsnInfo($asn);
265                $routes = $info->getRoutes();
266                $ranges = array_map(static function (AsnRouteInfo $route_info): ?RangeInterface {
267                    return IPFactory::rangeFromString($route_info->getRoute() ?: $route_info->getRoute6());
268                }, $routes);
269
270                return array_filter($ranges);
271            } catch (Throwable $ex) {
272                return [];
273            }
274        }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX));
275    }
276
277    /**
278     * @return ResponseInterface
279     */
280    private function response(): ResponseInterface
281    {
282        return response('Not acceptable', StatusCodeInterface::STATUS_NOT_ACCEPTABLE);
283    }
284}
285