xref: /webtrees/app/Http/Middleware/BadBotBlocker.php (revision 62dc46a69c19b130c54cf06c2246a10e684f6faa) 
1<?php
2
3/**
4 * webtrees: online genealogy
5 * Copyright (C) 2021 webtrees development team
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 */
17
18declare(strict_types=1);
19
20namespace Fisharebest\Webtrees\Http\Middleware;
21
22use Fig\Http\Message\StatusCodeInterface;
23use Fisharebest\Webtrees\Registry;
24use Iodev\Whois\Loaders\CurlLoader;
25use Iodev\Whois\Modules\Asn\AsnRouteInfo;
26use Iodev\Whois\Whois;
27use IPLib\Address\AddressInterface;
28use IPLib\Factory as IPFactory;
29use IPLib\Range\RangeInterface;
30use Psr\Http\Message\ResponseInterface;
31use Psr\Http\Message\ServerRequestInterface;
32use Psr\Http\Server\MiddlewareInterface;
33use Psr\Http\Server\RequestHandlerInterface;
34use Throwable;
35
36use function array_filter;
37use function array_map;
38use function assert;
39use function gethostbyaddr;
40use function gethostbyname;
41use function preg_match_all;
42use function random_int;
43use function response;
44use function str_contains;
45use function str_ends_with;
46
47/**
48 * Middleware to block bad robots before they waste our valuable CPU cycles.
49 */
50class BadBotBlocker implements MiddlewareInterface
51{
52    // Cache whois requests.  Try to avoid all caches expiring at the same time.
53    private const WHOIS_TTL_MIN = 28 * 86400;
54    private const WHOIS_TTL_MAX = 35 * 86400;
55    private const WHOIS_TIMEOUT = 5;
56
57    // Bad robots - SEO optimisers, advertisers, etc.  This list is shared with robots.txt.
58    public const BAD_ROBOTS = [
59        'admantx',
60        'Adsbot',
61        'AhrefsBot',
62        'AspiegelBot',
63        'Barkrowler',
64        'BLEXBot',
65        'DotBot',
66        'Grapeshot',
67        'ia_archiver',
68        'Linguee',
69        'MJ12bot',
70        'panscient',
71        'PetalBot',
72        'proximic',
73        'SemrushBot',
74        'Turnitin',
75        'XoviBot',
76        'ZoominfoBot',
77    ];
78
79    /**
80     * Some search engines use reverse/forward DNS to verify the IP address.
81     *
82     * @see https://support.google.com/webmasters/answer/80553?hl=en
83     * @see https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0
84     * @see https://www.bing.com/webmaster/help/how-to-verify-bingbot-3905dc26
85     * @see https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
86     */
87    private const ROBOT_REV_FWD_DNS = [
88        'bingbot'     => ['.search.msn.com'],
89        'BingPreview' => ['.search.msn.com'],
90        'Google'      => ['.google.com', '.googlebot.com'],
91        'Mail.RU_Bot' => ['mail.ru'],
92        'msnbot'      => ['.search.msn.com'],
93        'Qwantify'    => ['.search.qwant.com'],
94        'Sogou'       => ['.crawl.sogou.com'],
95        'Yahoo'       => ['.crawl.yahoo.net'],
96        'Yandex'      => ['.yandex.ru', '.yandex.net', '.yandex.com'],
97    ];
98
99    /**
100     * Some search engines only use reverse DNS to verify the IP address.
101     *
102     * @see https://help.baidu.com/question?prod_id=99&class=0&id=3001
103     */
104    private const ROBOT_REV_ONLY_DNS = [
105        'Baiduspider' => ['.baidu.com', '.baidu.jp'],
106    ];
107
108    /**
109     * Some search engines operate from designated IP addresses.
110     *
111     * @see https://www.apple.com/go/applebot
112     * @see https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
113     */
114    private const ROBOT_IPS = [
115        'AppleBot'    => [
116            '17.0.0.0/8',
117        ],
118        'Ask Jeeves'  => [
119            '65.214.45.143',
120            '65.214.45.148',
121            '66.235.124.192',
122            '66.235.124.7',
123            '66.235.124.101',
124            '66.235.124.193',
125            '66.235.124.73',
126            '66.235.124.196',
127            '66.235.124.74',
128            '63.123.238.8',
129            '202.143.148.61',
130        ],
131        'DuckDuckBot' => [
132            '23.21.227.69',
133            '50.16.241.113',
134            '50.16.241.114',
135            '50.16.241.117',
136            '50.16.247.234',
137            '52.204.97.54',
138            '52.5.190.19',
139            '54.197.234.188',
140            '54.208.100.253',
141            '54.208.102.37',
142            '107.21.1.8',
143        ],
144    ];
145
146    /**
147     * Some search engines operate from within a designated autonomous system.
148     *
149     * @see https://developers.facebook.com/docs/sharing/webmasters/crawler
150     * @see https://www.facebook.com/peering/
151     */
152    private const ROBOT_ASNS = [
153        'facebook' => ['AS32934', 'AS63293'],
154        'twitter'  => ['AS13414'],
155    ];
156
157    /**
158     * @param ServerRequestInterface  $request
159     * @param RequestHandlerInterface $handler
160     *
161     * @return ResponseInterface
162     */
163    public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
164    {
165        $ua      = $request->getServerParams()['HTTP_USER_AGENT'] ?? '';
166        $ip      = $request->getAttribute('client-ip');
167        $address = IPFactory::parseAddressString($ip);
168        assert($address instanceof AddressInterface);
169
170        foreach (self::BAD_ROBOTS as $robot) {
171            if (str_contains($ua, $robot)) {
172                return $this->response();
173            }
174        }
175
176        foreach (self::ROBOT_REV_FWD_DNS as $robot => $valid_domains) {
177            if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, false)) {
178                return $this->response();
179            }
180        }
181
182        foreach (self::ROBOT_REV_ONLY_DNS as $robot => $valid_domains) {
183            if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, true)) {
184                return $this->response();
185            }
186        }
187
188        foreach (self::ROBOT_IPS as $robot => $valid_ips) {
189            if (str_contains($ua, $robot)) {
190                foreach ($valid_ips as $ip) {
191                    $range = IPFactory::parseRangeString($ip);
192
193                    if ($range instanceof RangeInterface && $range->contains($address)) {
194                        continue 2;
195                    }
196                }
197
198                return $this->response();
199            }
200        }
201
202        foreach (self::ROBOT_ASNS as $robot => $asns) {
203            foreach ($asns as $asn) {
204                if (str_contains($ua, $robot)) {
205                    foreach ($this->fetchIpRangesForAsn($asn) as $range) {
206                        if ($range->contains($address)) {
207                            continue 2;
208                        }
209                    }
210
211                    return $this->response();
212                }
213            }
214        }
215
216        // Allow sites to block access from entire networks.
217        preg_match_all('/(AS\d+)/', $request->getAttribute('block_asn', ''), $matches);
218        foreach ($matches[1] as $asn) {
219            foreach ($this->fetchIpRangesForAsn($asn) as $range) {
220                if ($range->contains($address)) {
221                    return $this->response();
222                }
223            }
224        }
225
226        return $handler->handle($request);
227    }
228
229    /**
230     * Check that an IP address belongs to a robot operator using a forward/reverse DNS lookup.
231     *
232     * @param string        $ip
233     * @param array<string> $valid_domains
234     * @param bool          $reverse_only
235     *
236     * @return bool
237     */
238    private function checkRobotDNS(string $ip, array $valid_domains, bool $reverse_only): bool
239    {
240        $host = gethostbyaddr($ip);
241
242        if ($host === false) {
243            return false;
244        }
245
246        foreach ($valid_domains as $domain) {
247            if (str_ends_with($host, $domain)) {
248                return $reverse_only || $ip === gethostbyname($host);
249            }
250        }
251
252        return false;
253    }
254
255    /**
256     * Perform a whois search for an ASN.
257     *
258     * @param string $asn - The autonomous system number to query
259     *
260     * @return array<RangeInterface>
261     */
262    private function fetchIpRangesForAsn(string $asn): array
263    {
264        return Registry::cache()->file()->remember('whois-asn-' . $asn, static function () use ($asn): array {
265            $mapper = static fn (AsnRouteInfo $route_info): ?RangeInterface => IPFactory::parseRangeString($route_info->route ?: $route_info->route6);
266
267            try {
268                $loader = new CurlLoader(self::WHOIS_TIMEOUT);
269                $whois  = new Whois($loader);
270                $info   = $whois->loadAsnInfo($asn);
271                $routes = $info->routes;
272                $ranges = array_map($mapper, $routes);
273
274                return array_filter($ranges);
275            } catch (Throwable $ex) {
276                return [];
277            }
278        }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX));
279    }
280
281    /**
282     * @return ResponseInterface
283     */
284    private function response(): ResponseInterface
285    {
286        return response('Not acceptable', StatusCodeInterface::STATUS_NOT_ACCEPTABLE);
287    }
288}
289