xref: /webtrees/app/Http/Middleware/BadBotBlocker.php (revision ac71572d8462e396ed5a307f05b29381e49f9e6e) 
1<?php
2
3/**
4 * webtrees: online genealogy
5 * Copyright (C) 2022 webtrees development team
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 */
17
18declare(strict_types=1);
19
20namespace Fisharebest\Webtrees\Http\Middleware;
21
22use Fig\Http\Message\StatusCodeInterface;
23use Fisharebest\Webtrees\Registry;
24use Fisharebest\Webtrees\Validator;
25use Iodev\Whois\Loaders\CurlLoader;
26use Iodev\Whois\Modules\Asn\AsnRouteInfo;
27use Iodev\Whois\Whois;
28use IPLib\Address\AddressInterface;
29use IPLib\Factory as IPFactory;
30use IPLib\Range\RangeInterface;
31use Psr\Http\Message\ResponseInterface;
32use Psr\Http\Message\ServerRequestInterface;
33use Psr\Http\Server\MiddlewareInterface;
34use Psr\Http\Server\RequestHandlerInterface;
35use Throwable;
36
37use function array_filter;
38use function array_map;
39use function assert;
40use function gethostbyaddr;
41use function gethostbyname;
42use function preg_match_all;
43use function random_int;
44use function response;
45use function str_contains;
46use function str_ends_with;
47
48/**
49 * Middleware to block bad robots before they waste our valuable CPU cycles.
50 */
51class BadBotBlocker implements MiddlewareInterface
52{
53    // Cache whois requests.  Try to avoid all caches expiring at the same time.
54    private const WHOIS_TTL_MIN = 28 * 86400;
55    private const WHOIS_TTL_MAX = 35 * 86400;
56    private const WHOIS_TIMEOUT = 5;
57
58    // Bad robots - SEO optimisers, advertisers, etc.  This list is shared with robots.txt.
59    public const BAD_ROBOTS = [
60        'admantx',
61        'Adsbot',
62        'AhrefsBot',
63        'Amazonbot', // Until it understands crawl-delay and noindex / nofollow
64        'AspiegelBot',
65        'Barkrowler',
66        'BLEXBot',
67        'DataForSEO',
68        'DotBot',
69        'Grapeshot',
70        'ia_archiver',
71        'Linguee',
72        'MJ12bot',
73        'netEstate NE',
74        'panscient',
75        'PetalBot',
76        'proximic',
77        'SemrushBot',
78        'SEOkicks',
79        'SiteKiosk',
80        'Turnitin',
81        'XoviBot',
82        'ZoominfoBot',
83    ];
84
85    /**
86     * Some search engines use reverse/forward DNS to verify the IP address.
87     *
88     * @see https://developer.amazon.com/support/amazonbot
89     * @see https://support.google.com/webmasters/answer/80553?hl=en
90     * @see https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0
91     * @see https://www.bing.com/webmaster/help/how-to-verify-bingbot-3905dc26
92     * @see https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
93     * @see https://www.mojeek.com/bot.html
94     * @see https://support.apple.com/en-gb/HT204683
95     */
96    private const ROBOT_REV_FWD_DNS = [
97        'Amazonbot'   => ['.crawl.amazon.com'],
98        'Applebot'    => ['.applebot.apple.com'],
99        'bingbot'     => ['.search.msn.com'],
100        'BingPreview' => ['.search.msn.com'],
101        'Google'      => ['.google.com', '.googlebot.com'],
102        'MojeekBot'   => ['.mojeek.com'],
103        'Mail.RU_Bot' => ['.mail.ru'],
104        'msnbot'      => ['.search.msn.com'],
105        'Qwantify'    => ['.search.qwant.com'],
106        'Sogou'       => ['.crawl.sogou.com'],
107        'Yahoo'       => ['.crawl.yahoo.net'],
108        'Yandex'      => ['.yandex.ru', '.yandex.net', '.yandex.com'],
109    ];
110
111    /**
112     * Some search engines only use reverse DNS to verify the IP address.
113     *
114     * @see https://help.baidu.com/question?prod_id=99&class=0&id=3001
115     * @see https://napoveda.seznam.cz/en/full-text-search/seznambot-crawler
116     * @see https://www.ionos.de/terms-gtc/faq-crawler
117     */
118    private const ROBOT_REV_ONLY_DNS = [
119        'Baiduspider' => ['.baidu.com', '.baidu.jp'],
120        'FreshBot'    => ['.seznam.cz'],
121        'IonCrawl'    => ['.1und1.org'],
122        'Neevabot'    => ['.neeva.com'],
123    ];
124
125    /**
126     * Some search engines operate from designated IP addresses.
127     *
128     * @see https://www.apple.com/go/applebot
129     * @see https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
130     */
131    private const ROBOT_IPS = [
132        'AppleBot'    => [
133            '17.0.0.0/8',
134        ],
135        'Ask Jeeves'  => [
136            '65.214.45.143',
137            '65.214.45.148',
138            '66.235.124.192',
139            '66.235.124.7',
140            '66.235.124.101',
141            '66.235.124.193',
142            '66.235.124.73',
143            '66.235.124.196',
144            '66.235.124.74',
145            '63.123.238.8',
146            '202.143.148.61',
147        ],
148        'DuckDuckBot' => [
149            '23.21.227.69',
150            '50.16.241.113',
151            '50.16.241.114',
152            '50.16.241.117',
153            '50.16.247.234',
154            '52.204.97.54',
155            '52.5.190.19',
156            '54.197.234.188',
157            '54.208.100.253',
158            '54.208.102.37',
159            '107.21.1.8',
160        ],
161    ];
162
163    /**
164     * Some search engines operate from within a designated autonomous system.
165     *
166     * @see https://developers.facebook.com/docs/sharing/webmasters/crawler
167     * @see https://www.facebook.com/peering/
168     */
169    private const ROBOT_ASNS = [
170        'facebook' => ['AS32934', 'AS63293'],
171        'twitter'  => ['AS13414'],
172    ];
173
174    /**
175     * @param ServerRequestInterface  $request
176     * @param RequestHandlerInterface $handler
177     *
178     * @return ResponseInterface
179     */
180    public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
181    {
182        $ua      = Validator::serverParams($request)->string('HTTP_USER_AGENT', '');
183        $ip      = Validator::attributes($request)->string('client-ip');
184        $address = IPFactory::parseAddressString($ip);
185        assert($address instanceof AddressInterface);
186
187        foreach (self::BAD_ROBOTS as $robot) {
188            if (str_contains($ua, $robot)) {
189                return $this->response();
190            }
191        }
192
193        foreach (self::ROBOT_REV_FWD_DNS as $robot => $valid_domains) {
194            if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, false)) {
195                return $this->response();
196            }
197        }
198
199        foreach (self::ROBOT_REV_ONLY_DNS as $robot => $valid_domains) {
200            if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, true)) {
201                return $this->response();
202            }
203        }
204
205        foreach (self::ROBOT_IPS as $robot => $valid_ips) {
206            if (str_contains($ua, $robot)) {
207                foreach ($valid_ips as $ip) {
208                    $range = IPFactory::parseRangeString($ip);
209
210                    if ($range instanceof RangeInterface && $range->contains($address)) {
211                        continue 2;
212                    }
213                }
214
215                return $this->response();
216            }
217        }
218
219        foreach (self::ROBOT_ASNS as $robot => $asns) {
220            foreach ($asns as $asn) {
221                if (str_contains($ua, $robot)) {
222                    foreach ($this->fetchIpRangesForAsn($asn) as $range) {
223                        if ($range->contains($address)) {
224                            continue 2;
225                        }
226                    }
227
228                    return $this->response();
229                }
230            }
231        }
232
233        // Allow sites to block access from entire networks.
234        $block_asn = Validator::attributes($request)->string('block_asn', '');
235        preg_match_all('/(AS\d+)/', $block_asn, $matches);
236
237        foreach ($matches[1] as $asn) {
238            foreach ($this->fetchIpRangesForAsn($asn) as $range) {
239                if ($range->contains($address)) {
240                    return $this->response();
241                }
242            }
243        }
244
245        return $handler->handle($request);
246    }
247
248    /**
249     * Check that an IP address belongs to a robot operator using a forward/reverse DNS lookup.
250     *
251     * @param string        $ip
252     * @param array<string> $valid_domains
253     * @param bool          $reverse_only
254     *
255     * @return bool
256     */
257    private function checkRobotDNS(string $ip, array $valid_domains, bool $reverse_only): bool
258    {
259        $host = gethostbyaddr($ip);
260
261        if ($host === false) {
262            return false;
263        }
264
265        foreach ($valid_domains as $domain) {
266            if (str_ends_with($host, $domain)) {
267                return $reverse_only || $ip === gethostbyname($host);
268            }
269        }
270
271        return false;
272    }
273
274    /**
275     * Perform a whois search for an ASN.
276     *
277     * @param string $asn - The autonomous system number to query
278     *
279     * @return array<RangeInterface>
280     */
281    private function fetchIpRangesForAsn(string $asn): array
282    {
283        return Registry::cache()->file()->remember('whois-asn-' . $asn, static function () use ($asn): array {
284            $mapper = static fn (AsnRouteInfo $route_info): ?RangeInterface => IPFactory::parseRangeString($route_info->route ?: $route_info->route6);
285
286            try {
287                $loader = new CurlLoader(self::WHOIS_TIMEOUT);
288                $whois  = new Whois($loader);
289                $info   = $whois->loadAsnInfo($asn);
290                $routes = $info->routes;
291                $ranges = array_map($mapper, $routes);
292
293                return array_filter($ranges);
294            } catch (Throwable $ex) {
295                return [];
296            }
297        }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX));
298    }
299
300    /**
301     * @return ResponseInterface
302     */
303    private function response(): ResponseInterface
304    {
305        return response('Not acceptable', StatusCodeInterface::STATUS_NOT_ACCEPTABLE);
306    }
307}
308