xref: /webtrees/app/Http/Middleware/BadBotBlocker.php (revision eec99f1a5c97e2ce7795a2e8d6b23f86e359534f) 
1<?php
2
3/**
4 * webtrees: online genealogy
5 * Copyright (C) 2019 webtrees development team
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18declare(strict_types=1);
19
20namespace Fisharebest\Webtrees\Http\Middleware;
21
22use Fig\Http\Message\StatusCodeInterface;
23use Fisharebest\Webtrees\Cache;
24use Illuminate\Support\Str;
25use Iodev\Whois\Loaders\CurlLoader;
26use Iodev\Whois\Modules\Asn\AsnRouteInfo;
27use Iodev\Whois\Whois;
28use IPLib\Address\AddressInterface;
29use IPLib\Factory;
30use IPLib\Range\RangeInterface;
31use Psr\Http\Message\ResponseInterface;
32use Psr\Http\Message\ServerRequestInterface;
33use Psr\Http\Server\MiddlewareInterface;
34use Psr\Http\Server\RequestHandlerInterface;
35use Throwable;
36
37use function app;
38use function array_map;
39use function assert;
40use function gethostbyaddr;
41use function gethostbyname;
42use function in_array;
43use function response;
44
45/**
46 * Middleware to block bad robots before they waste our valuable CPU cycles.
47 */
48class BadBotBlocker implements MiddlewareInterface
49{
50    // Cache whois requests.  Try to avoid all caches expiring at the same time.
51    private const WHOIS_TTL_MIN = 28 * 86400;
52    private const WHOIS_TTL_MAX = 35 * 86400;
53    private const WHOIS_TIMEOUT = 5;
54
55    // Bad robots - SEO optimisers, advertisers, etc
56    private const BAD_ROBOTS = [
57        'admantx',
58        'AhrefsBot',
59        'AspiegelBot',
60        'DotBot',
61        'Grapeshot',
62        'ia_archiver',
63        'MJ12bot',
64        'panscient',
65        'proximic',
66        'SemrushBot',
67        'XoviBot',
68    ];
69
70    /**
71     * Some search engines provide reverse DNS to verify the IP address.
72     *
73     * @see https://support.google.com/webmasters/answer/80553?hl=en
74     * @see https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0
75     * @see https://www.bing.com/webmaster/help/how-to-verify-bingbot-3905dc26
76     * @see https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
77     * @see https://help.baidu.com/question?prod_id=99&class=0&id=3001
78     */
79    private const ROBOT_DNS = [
80        'Baidu'       => ['.baidu.com', '.baidu.jp'],
81        'bingbot'     => ['.search.msn.com'],
82        'BingPreview' => ['.search.msn.com'],
83        'Google'      => ['.google.com', '.googlebot.com'],
84        'msnbot'      => ['.search.msn.com'],
85        'Qwantify'    => ['.search.qwant.com'],
86        'Sogou'       => ['.crawl.sogou.com'],
87        'Yahoo'       => ['.crawl.yahoo.net'],
88        'Yandex'      => ['.yandex.ru', '.yandex.net', '.yandex.com'],
89    ];
90
91    /**
92     * Some search engines operate from designated IP addresses.
93     *
94     * @see http://www.apple.com/go/applebot
95     * @see https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
96     */
97    private const ROBOT_IPS = [
98        'AppleBot'    => [
99            '17.0.0.0/8',
100        ],
101        'Ask Jeeves'  => [
102            '65.214.45.143',
103            '65.214.45.148',
104            '66.235.124.192',
105            '66.235.124.7',
106            '66.235.124.101',
107            '66.235.124.193',
108            '66.235.124.73',
109            '66.235.124.196',
110            '66.235.124.74',
111            '63.123.238.8',
112            '202.143.148.61',
113        ],
114        'DuckDuckBot' => [
115            '23.21.227.69',
116            '50.16.241.113',
117            '50.16.241.114',
118            '50.16.241.117',
119            '50.16.247.234',
120            '52.204.97.54',
121            '52.5.190.19',
122            '54.197.234.188',
123            '54.208.100.253',
124            '54.208.102.37',
125            '107.21.1.8',
126        ],
127    ];
128
129    /**
130     * Some search engines operate from within a designated autonomous system.
131     *
132     * @see https://developers.facebook.com/docs/sharing/webmasters/crawler
133     */
134    private const ROBOT_ASN = [
135        'facebook' => 'AS32934',
136        'twitter'  => 'AS13414',
137    ];
138
139    /**
140     * These ASNs belong to server farms.
141     */
142    private const BLOCK_ASN = [
143        'hetzner'   => 'AS24920',
144        'hostdime'  => 'AS33182',
145        'linode'    => 'AS63949',
146        'ovh'       => 'AS16276',
147        'rackspace' => 'AS15395',
148    ];
149
150    /**
151     * @param ServerRequestInterface  $request
152     * @param RequestHandlerInterface $handler
153     *
154     * @return ResponseInterface
155     */
156    public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
157    {
158        $ua      = $request->getServerParams()['HTTP_USER_AGENT'] ?? '';
159        $ip      = $request->getAttribute('client-ip');
160        $address = Factory::addressFromString($ip);
161        assert($address instanceof AddressInterface);
162
163        if (Str::contains($ua, self::BAD_ROBOTS)) {
164            return $this->response();
165        }
166
167        foreach (self::ROBOT_DNS as $robot => $valid_domains) {
168            if (Str::contains($ua, $robot) && !$this->checkReverseDNS($ip, $valid_domains)) {
169                return $this->response();
170            }
171        }
172
173        foreach (self::ROBOT_IPS as $robot => $valid_ips) {
174            if (Str::contains($ua, $robot)) {
175                foreach ($valid_ips as $ip) {
176                    $range = Factory::rangeFromString($ip);
177
178                    if ($range instanceof RangeInterface && $range->contains($address)) {
179                        continue 2;
180                    }
181                }
182
183                return $this->response();
184            }
185        }
186
187        foreach (self::ROBOT_ASN as $robot => $asn) {
188            if (Str::contains($ua, $robot)) {
189                foreach ($this->fetchIpRangesForAsn($asn) as $range) {
190                    if ($range->contains($address)) {
191                        continue 2;
192                    }
193                }
194
195                return $this->response();
196            }
197        }
198
199        // This is potentially controversial, and whois lookups may be slow.
200        //foreach (self::BLOCK_ASN as $host => $asn) {
201        //    foreach ($this->fetchIpRangesForAsn($asn) as $range) {
202        //        if ($range->contains($address)) {
203        //            return $this->response();
204        //        }
205        //    }
206        //}
207
208        return $handler->handle($request);
209    }
210
211    /**
212     * Check that an IP address belongs to a robot operator using a forward/reverse DNS lookup.
213     *
214     * @param string        $ip
215     * @param array<string> $valid_domains
216     *
217     * @return bool
218     */
219    private function checkReverseDNS(string $ip, array $valid_domains): bool
220    {
221        $host = gethostbyaddr($ip);
222
223        if ($host === false || !Str::endsWith($host, $valid_domains)) {
224            return false;
225        }
226
227        return $ip === gethostbyname($host);
228    }
229
230    /**
231     * Perform a whois search for an ASN.
232     *
233     * @param string $asn - The autonomous system number to query
234     *
235     * @return array<RangeInterface>
236     */
237    private function fetchIpRangesForAsn(string $asn): array
238    {
239        $cache = app('cache.files');
240        assert($cache instanceof Cache);
241
242        return $cache->remember('whois-asn-' . $asn, static function () use ($asn): array {
243            try {
244                $loader = new CurlLoader(self::WHOIS_TIMEOUT);
245                $whois  = new Whois($loader);
246                $info   = $whois->loadAsnInfo($asn);
247                $routes = $info->getRoutes();
248                $ranges = array_map(static function (AsnRouteInfo $route_info): ?RangeInterface {
249                    return Factory::rangeFromString($route_info->getRoute() ?: $route_info->getRoute6());
250                }, $routes);
251
252                return array_filter($ranges);
253            } catch (Throwable $ex) {
254                return [];
255            }
256        }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX));
257    }
258
259    /**
260     * @return ResponseInterface
261     */
262    private function response(): ResponseInterface
263    {
264        return response('Not acceptable', StatusCodeInterface::STATUS_NOT_ACCEPTABLE);
265    }
266}
267