xref: /webtrees/app/Http/Middleware/BadBotBlocker.php (revision 67497a53f705a1c2998fed00c3bba0fd1c7a86ed) 
1<?php
2
3/**
4 * webtrees: online genealogy
5 * Copyright (C) 2023 webtrees development team
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 */
17
18declare(strict_types=1);
19
20namespace Fisharebest\Webtrees\Http\Middleware;
21
22use Fig\Http\Message\StatusCodeInterface;
23use Fisharebest\Webtrees\Registry;
24use Fisharebest\Webtrees\Validator;
25use GuzzleHttp\Client;
26use GuzzleHttp\Exception\GuzzleException;
27use Iodev\Whois\Loaders\CurlLoader;
28use Iodev\Whois\Modules\Asn\AsnRouteInfo;
29use Iodev\Whois\Whois;
30use IPLib\Address\AddressInterface;
31use IPLib\Factory as IPFactory;
32use IPLib\Range\RangeInterface;
33use Psr\Http\Message\ResponseInterface;
34use Psr\Http\Message\ServerRequestInterface;
35use Psr\Http\Server\MiddlewareInterface;
36use Psr\Http\Server\RequestHandlerInterface;
37use Throwable;
38
39use function array_filter;
40use function array_map;
41use function assert;
42use function gethostbyaddr;
43use function gethostbyname;
44use function preg_match_all;
45use function random_int;
46use function response;
47use function str_contains;
48use function str_ends_with;
49
50/**
51 * Middleware to block bad robots before they waste our valuable CPU cycles.
52 */
53class BadBotBlocker implements MiddlewareInterface
54{
55    private const REGEX_OCTET = '(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)';
56    private const REGEX_IPV4  = '/\\b' . self::REGEX_OCTET . '(?:\\.' . self::REGEX_OCTET . '){3}\\b/';
57
58    // Cache whois requests.  Try to avoid all caches expiring at the same time.
59    private const WHOIS_TTL_MIN = 28 * 86400;
60    private const WHOIS_TTL_MAX = 35 * 86400;
61    private const WHOIS_TIMEOUT = 5;
62
63    // Bad robots - SEO optimisers, advertisers, etc.  This list is shared with robots.txt.
64    public const BAD_ROBOTS = [
65        'admantx',
66        'Adsbot',
67        'AhrefsBot',
68        'Amazonbot', // Until it understands crawl-delay and noindex / nofollow
69        'AspiegelBot',
70        'Awario', // Brand management
71        'Barkrowler',
72        'BLEXBot',
73        'Bytespider',
74        'CCBot', // Used to train a number of LLMs
75        'ChatGPT-User', // Used by ChatGPT during operation
76        'DataForSEO',
77        'DataForSeoBot', // https://dataforseo.com/dataforseo-bot
78        'DotBot',
79        'FacebookBot', // Collects training data for Facebook's LLM translator.
80        'Google-Extended', // Collects training data for Google Bard
81        'GPTBot', // Collects training data for ChatGPT
82        'Grapeshot',
83        'Honolulu-bot', // Aggressive crawer, no info available
84        'ia_archiver',
85        'linabot', // Aggressive crawer, no info available
86        'Linguee',
87        'MegaIndex.ru',
88        'MJ12bot',
89        'netEstate NE',
90        'Omgilibot', // Collects training data for LLMs
91        'panscient',
92        'PetalBot',
93        'proximic',
94        'SeekportBot', // Pretends to be a search engine - but isn't
95        'SemrushBot',
96        'serpstatbot',
97        'SEOkicks',
98        'SiteKiosk',
99        'TinyTestBot',
100        'Turnitin',
101        'wp_is_mobile', // Nothing to do with wordpress
102        'XoviBot',
103        'YisouSpider',
104        'ZoominfoBot',
105    ];
106
107    /**
108     * Some search engines use reverse/forward DNS to verify the IP address.
109     *
110     * @see https://developer.amazon.com/support/amazonbot
111     * @see https://support.google.com/webmasters/answer/80553?hl=en
112     * @see https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0
113     * @see https://www.bing.com/webmaster/help/how-to-verify-bingbot-3905dc26
114     * @see https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
115     * @see https://www.mojeek.com/bot.html
116     * @see https://support.apple.com/en-gb/HT204683
117     */
118    private const ROBOT_REV_FWD_DNS = [
119        'Amazonbot'        => ['.crawl.amazon.com'],
120        'Applebot'         => ['.applebot.apple.com'],
121        'BingPreview'      => ['.search.msn.com'],
122        'Google'           => ['.google.com', '.googlebot.com'],
123        'Mail.RU_Bot'      => ['.mail.ru'],
124        'MicrosoftPreview' => ['.search.msn.com'],
125        'MojeekBot'        => ['.mojeek.com'],
126        'Qwantify'         => ['.search.qwant.com'],
127        'Sogou'            => ['.crawl.sogou.com'],
128        'Yahoo'            => ['.crawl.yahoo.net'],
129        'Yandex'           => ['.yandex.ru', '.yandex.net', '.yandex.com'],
130        'bingbot'          => ['.search.msn.com'],
131        'msnbot'           => ['.search.msn.com'],
132    ];
133
134    /**
135     * Some search engines only use reverse DNS to verify the IP address.
136     *
137     * @see https://help.baidu.com/question?prod_id=99&class=0&id=3001
138     * @see https://napoveda.seznam.cz/en/full-text-search/seznambot-crawler
139     * @see https://www.ionos.de/terms-gtc/faq-crawler
140     */
141    private const ROBOT_REV_ONLY_DNS = [
142        'Baiduspider' => ['.baidu.com', '.baidu.jp'],
143        'FreshBot'    => ['.seznam.cz'],
144        'IonCrawl'    => ['.1und1.org'],
145        'Neevabot'    => ['.neeva.com'],
146        'SeznamBot'   => ['.seznam.cz'],
147    ];
148
149    /**
150     * Some search engines operate from designated IP addresses.
151     *
152     * @see https://www.apple.com/go/applebot
153     * @see https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
154     */
155    private const ROBOT_IPS = [
156        'AppleBot'    => [
157            '17.0.0.0/8',
158        ],
159        'Ask Jeeves'  => [
160            '65.214.45.143',
161            '65.214.45.148',
162            '66.235.124.192',
163            '66.235.124.7',
164            '66.235.124.101',
165            '66.235.124.193',
166            '66.235.124.73',
167            '66.235.124.196',
168            '66.235.124.74',
169            '63.123.238.8',
170            '202.143.148.61',
171        ],
172        'DuckDuckBot' => [
173            '23.21.227.69',
174            '50.16.241.113',
175            '50.16.241.114',
176            '50.16.241.117',
177            '50.16.247.234',
178            '52.204.97.54',
179            '52.5.190.19',
180            '54.197.234.188',
181            '54.208.100.253',
182            '54.208.102.37',
183            '107.21.1.8',
184        ],
185    ];
186
187    /**
188     * Some search engines operate from designated IP addresses.
189     *
190     * @see https://bot.seekport.com/
191     */
192    private const ROBOT_IP_FILES = [
193        'SeekportBot' => 'https://bot.seekport.com/seekportbot_ips.txt',
194    ];
195
196    /**
197     * Some search engines operate from within a designated autonomous system.
198     *
199     * @see https://developers.facebook.com/docs/sharing/webmasters/crawler
200     * @see https://www.facebook.com/peering/
201     */
202    private const ROBOT_ASNS = [
203        'facebook' => ['AS32934', 'AS63293'],
204        'twitter'  => ['AS13414'],
205    ];
206
207    /**
208     * @param ServerRequestInterface  $request
209     * @param RequestHandlerInterface $handler
210     *
211     * @return ResponseInterface
212     */
213    public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
214    {
215        $ua      = Validator::serverParams($request)->string('HTTP_USER_AGENT', '');
216        $ip      = Validator::attributes($request)->string('client-ip');
217        $address = IPFactory::parseAddressString($ip);
218        assert($address instanceof AddressInterface);
219
220        foreach (self::BAD_ROBOTS as $robot) {
221            if (str_contains($ua, $robot)) {
222                return $this->response();
223            }
224        }
225
226        foreach (self::ROBOT_REV_FWD_DNS as $robot => $valid_domains) {
227            if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, false)) {
228                return $this->response();
229            }
230        }
231
232        foreach (self::ROBOT_REV_ONLY_DNS as $robot => $valid_domains) {
233            if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, true)) {
234                return $this->response();
235            }
236        }
237
238        foreach (self::ROBOT_IPS as $robot => $valid_ip_ranges) {
239            if (str_contains($ua, $robot)) {
240                foreach ($valid_ip_ranges as $ip_range) {
241                    $range = IPFactory::parseRangeString($ip_range);
242
243                    if ($range instanceof RangeInterface && $range->contains($address)) {
244                        continue 2;
245                    }
246                }
247
248                return $this->response();
249            }
250        }
251
252        foreach (self::ROBOT_IP_FILES as $robot => $url) {
253            if (str_contains($ua, $robot)) {
254                $valid_ip_ranges = $this->fetchIpRangesForUrl($robot, $url);
255
256                foreach ($valid_ip_ranges as $ip_range) {
257                    $range = IPFactory::parseRangeString($ip_range);
258
259                    if ($range instanceof RangeInterface && $range->contains($address)) {
260                        continue 2;
261                    }
262                }
263
264                return $this->response();
265            }
266        }
267
268        foreach (self::ROBOT_ASNS as $robot => $asns) {
269            foreach ($asns as $asn) {
270                if (str_contains($ua, $robot)) {
271                    foreach ($this->fetchIpRangesForAsn($asn) as $range) {
272                        if ($range->contains($address)) {
273                            continue 2;
274                        }
275                    }
276
277                    return $this->response();
278                }
279            }
280        }
281
282        // Allow sites to block access from entire networks.
283        $block_asn = Validator::attributes($request)->string('block_asn', '');
284        preg_match_all('/(AS\d+)/', $block_asn, $matches);
285
286        foreach ($matches[1] as $asn) {
287            foreach ($this->fetchIpRangesForAsn($asn) as $range) {
288                if ($range->contains($address)) {
289                    return $this->response();
290                }
291            }
292        }
293
294        return $handler->handle($request);
295    }
296
297    /**
298     * Check that an IP address belongs to a robot operator using a forward/reverse DNS lookup.
299     *
300     * @param string        $ip
301     * @param array<string> $valid_domains
302     * @param bool          $reverse_only
303     *
304     * @return bool
305     */
306    private function checkRobotDNS(string $ip, array $valid_domains, bool $reverse_only): bool
307    {
308        $host = gethostbyaddr($ip);
309
310        if ($host === false) {
311            return false;
312        }
313
314        foreach ($valid_domains as $domain) {
315            if (str_ends_with($host, $domain)) {
316                return $reverse_only || $ip === gethostbyname($host);
317            }
318        }
319
320        return false;
321    }
322
323    /**
324     * Perform a whois search for an ASN.
325     *
326     * @param string $asn The autonomous system number to query
327     *
328     * @return array<RangeInterface>
329     */
330    private function fetchIpRangesForAsn(string $asn): array
331    {
332        return Registry::cache()->file()->remember('whois-asn-' . $asn, static function () use ($asn): array {
333            $mapper = static fn (AsnRouteInfo $route_info): ?RangeInterface => IPFactory::parseRangeString($route_info->route ?: $route_info->route6);
334
335            try {
336                $loader = new CurlLoader(self::WHOIS_TIMEOUT);
337                $whois  = new Whois($loader);
338                $info   = $whois->loadAsnInfo($asn);
339                $routes = $info->routes;
340                $ranges = array_map($mapper, $routes);
341
342                return array_filter($ranges);
343            } catch (Throwable) {
344                return [];
345            }
346        }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX));
347    }
348
349    /**
350     * Fetch a list of IP addresses from a remote file.
351     *
352     * @param string $ua
353     * @param string $url
354     *
355     * @return array<string>
356     */
357    private function fetchIpRangesForUrl(string $ua, string $url): array
358    {
359        return Registry::cache()->file()->remember('url-ip-list-' . $ua, static function () use ($url): array {
360            try {
361                $client   = new Client();
362                $response = $client->get($url, ['timeout' => 5]);
363                $contents = $response->getBody()->getContents();
364
365                preg_match_all(self::REGEX_IPV4, $contents, $matches);
366
367                return $matches[0];
368            } catch (GuzzleException) {
369                return [];
370            }
371        }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX));
372    }
373
374    /**
375     * @return ResponseInterface
376     */
377    private function response(): ResponseInterface
378    {
379        return response('Not acceptable', StatusCodeInterface::STATUS_NOT_ACCEPTABLE);
380    }
381}
382