xref: /webtrees/app/Http/Middleware/BadBotBlocker.php (revision b4a2f8851f56e7ca6fae19d0a51ff909996d7a30) 
1<?php
2
3/**
4 * webtrees: online genealogy
5 * Copyright (C) 2020 webtrees development team
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18declare(strict_types=1);
19
20namespace Fisharebest\Webtrees\Http\Middleware;
21
22use Fig\Http\Message\StatusCodeInterface;
23use Fisharebest\Webtrees\Cache;
24use Iodev\Whois\Loaders\CurlLoader;
25use Iodev\Whois\Modules\Asn\AsnRouteInfo;
26use Iodev\Whois\Whois;
27use IPLib\Address\AddressInterface;
28use IPLib\Factory;
29use IPLib\Range\RangeInterface;
30use Psr\Http\Message\ResponseInterface;
31use Psr\Http\Message\ServerRequestInterface;
32use Psr\Http\Server\MiddlewareInterface;
33use Psr\Http\Server\RequestHandlerInterface;
34use Throwable;
35
36use function app;
37use function array_map;
38use function assert;
39use function gethostbyaddr;
40use function gethostbyname;
41use function response;
42use function str_contains;
43use function str_ends_with;
44
45/**
46 * Middleware to block bad robots before they waste our valuable CPU cycles.
47 */
48class BadBotBlocker implements MiddlewareInterface
49{
50    // Cache whois requests.  Try to avoid all caches expiring at the same time.
51    private const WHOIS_TTL_MIN = 28 * 86400;
52    private const WHOIS_TTL_MAX = 35 * 86400;
53    private const WHOIS_TIMEOUT = 5;
54
55    // Bad robots - SEO optimisers, advertisers, etc
56    private const BAD_ROBOTS = [
57        'admantx',
58        'AhrefsBot',
59        'AspiegelBot',
60        'DotBot',
61        'Grapeshot',
62        'ia_archiver',
63        'MJ12bot',
64        'panscient',
65        'proximic',
66        'SemrushBot',
67        'XoviBot',
68    ];
69
70    /**
71     * Some search engines use reverse/forward DNS to verify the IP address.
72     *
73     * @see https://support.google.com/webmasters/answer/80553?hl=en
74     * @see https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0
75     * @see https://www.bing.com/webmaster/help/how-to-verify-bingbot-3905dc26
76     * @see https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
77     */
78    private const ROBOT_REV_FWD_DNS = [
79        'bingbot'     => ['.search.msn.com'],
80        'BingPreview' => ['.search.msn.com'],
81        'Google'      => ['.google.com', '.googlebot.com'],
82        'msnbot'      => ['.search.msn.com'],
83        'Qwantify'    => ['.search.qwant.com'],
84        'Sogou'       => ['.crawl.sogou.com'],
85        'Yahoo'       => ['.crawl.yahoo.net'],
86        'Yandex'      => ['.yandex.ru', '.yandex.net', '.yandex.com'],
87    ];
88
89    /**
90     * Some search engines only use reverse DNS to verify the IP address.
91     *
92     * @see https://help.baidu.com/question?prod_id=99&class=0&id=3001
93     */
94    private const ROBOT_REV_ONLY_DNS = [
95        'Baiduspider' => ['.baidu.com', '.baidu.jp'],
96    ];
97
98    /**
99     * Some search engines operate from designated IP addresses.
100     *
101     * @see http://www.apple.com/go/applebot
102     * @see https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
103     */
104    private const ROBOT_IPS = [
105        'AppleBot'    => [
106            '17.0.0.0/8',
107        ],
108        'Ask Jeeves'  => [
109            '65.214.45.143',
110            '65.214.45.148',
111            '66.235.124.192',
112            '66.235.124.7',
113            '66.235.124.101',
114            '66.235.124.193',
115            '66.235.124.73',
116            '66.235.124.196',
117            '66.235.124.74',
118            '63.123.238.8',
119            '202.143.148.61',
120        ],
121        'DuckDuckBot' => [
122            '23.21.227.69',
123            '50.16.241.113',
124            '50.16.241.114',
125            '50.16.241.117',
126            '50.16.247.234',
127            '52.204.97.54',
128            '52.5.190.19',
129            '54.197.234.188',
130            '54.208.100.253',
131            '54.208.102.37',
132            '107.21.1.8',
133        ],
134    ];
135
136    /**
137     * Some search engines operate from within a designated autonomous system.
138     *
139     * @see https://developers.facebook.com/docs/sharing/webmasters/crawler
140     */
141    private const ROBOT_ASN = [
142        'facebook' => 'AS32934',
143        'twitter'  => 'AS13414',
144    ];
145
146    /**
147     * @param ServerRequestInterface  $request
148     * @param RequestHandlerInterface $handler
149     *
150     * @return ResponseInterface
151     */
152    public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
153    {
154        $ua      = $request->getServerParams()['HTTP_USER_AGENT'] ?? '';
155        $ip      = $request->getAttribute('client-ip');
156        $address = Factory::addressFromString($ip);
157        assert($address instanceof AddressInterface);
158
159        foreach (self::BAD_ROBOTS as $robot) {
160            if (str_contains($ua, $robot)) {
161                return $this->response();
162            }
163        }
164
165        foreach (self::ROBOT_REV_FWD_DNS as $robot => $valid_domains) {
166            if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, false)) {
167                return $this->response();
168            }
169        }
170
171        foreach (self::ROBOT_REV_ONLY_DNS as $robot => $valid_domains) {
172            if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, true)) {
173                return $this->response();
174            }
175        }
176
177        foreach (self::ROBOT_IPS as $robot => $valid_ips) {
178            if (str_contains($ua, $robot)) {
179                foreach ($valid_ips as $ip) {
180                    $range = Factory::rangeFromString($ip);
181
182                    if ($range instanceof RangeInterface && $range->contains($address)) {
183                        continue 2;
184                    }
185                }
186
187                return $this->response();
188            }
189        }
190
191        foreach (self::ROBOT_ASN as $robot => $asn) {
192            if (str_contains($ua, $robot)) {
193                foreach ($this->fetchIpRangesForAsn($asn) as $range) {
194                    if ($range->contains($address)) {
195                        continue 2;
196                    }
197                }
198
199                return $this->response();
200            }
201        }
202
203        // Allow sites to block access from entire networks.
204        preg_match_all('/(AS\d+)/', $request->getAttribute('block_asn', ''), $matches);
205        foreach ($matches[1] as $asn) {
206            foreach ($this->fetchIpRangesForAsn($asn) as $range) {
207                if ($range->contains($address)) {
208                    return $this->response();
209                }
210            }
211        }
212
213        return $handler->handle($request);
214    }
215
216    /**
217     * Check that an IP address belongs to a robot operator using a forward/reverse DNS lookup.
218     *
219     * @param string        $ip
220     * @param array<string> $valid_domains
221     * @param bool          $reverse_only
222     *
223     * @return bool
224     */
225    private function checkRobotDNS(string $ip, array $valid_domains, bool $reverse_only): bool
226    {
227        $host = gethostbyaddr($ip);
228
229        if ($host === false) {
230            return false;
231        }
232
233        foreach ($valid_domains as $domain) {
234            if (str_ends_with($host, $domain)) {
235                return $reverse_only || $ip === gethostbyname($host);
236            }
237        }
238
239        return false;
240    }
241
242    /**
243     * Perform a whois search for an ASN.
244     *
245     * @param string $asn - The autonomous system number to query
246     *
247     * @return array<RangeInterface>
248     */
249    private function fetchIpRangesForAsn(string $asn): array
250    {
251        $cache = app('cache.files');
252        assert($cache instanceof Cache);
253
254        return $cache->remember('whois-asn-' . $asn, static function () use ($asn): array {
255            try {
256                $loader = new CurlLoader(self::WHOIS_TIMEOUT);
257                $whois  = new Whois($loader);
258                $info   = $whois->loadAsnInfo($asn);
259                $routes = $info->getRoutes();
260                $ranges = array_map(static function (AsnRouteInfo $route_info): ?RangeInterface {
261                    return Factory::rangeFromString($route_info->getRoute() ?: $route_info->getRoute6());
262                }, $routes);
263
264                return array_filter($ranges);
265            } catch (Throwable $ex) {
266                return [];
267            }
268        }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX));
269    }
270
271    /**
272     * @return ResponseInterface
273     */
274    private function response(): ResponseInterface
275    {
276        return response('Not acceptable', StatusCodeInterface::STATUS_NOT_ACCEPTABLE);
277    }
278}
279