xref: /webtrees/app/Http/Middleware/BadBotBlocker.php (revision 6e9b2106ef8de9279faf02c6e02326f2ecbbf5a5) 
1<?php
2
3/**
4 * webtrees: online genealogy
5 * Copyright (C) 2020 webtrees development team
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18declare(strict_types=1);
19
20namespace Fisharebest\Webtrees\Http\Middleware;
21
22use Fig\Http\Message\StatusCodeInterface;
23use Fisharebest\Webtrees\Cache;
24use Illuminate\Support\Str;
25use Iodev\Whois\Loaders\CurlLoader;
26use Iodev\Whois\Modules\Asn\AsnRouteInfo;
27use Iodev\Whois\Whois;
28use IPLib\Address\AddressInterface;
29use IPLib\Factory;
30use IPLib\Range\RangeInterface;
31use Psr\Http\Message\ResponseInterface;
32use Psr\Http\Message\ServerRequestInterface;
33use Psr\Http\Server\MiddlewareInterface;
34use Psr\Http\Server\RequestHandlerInterface;
35use Throwable;
36
37use function app;
38use function array_map;
39use function assert;
40use function gethostbyaddr;
41use function gethostbyname;
42use function response;
43
44/**
45 * Middleware to block bad robots before they waste our valuable CPU cycles.
46 */
47class BadBotBlocker implements MiddlewareInterface
48{
49    // Cache whois requests.  Try to avoid all caches expiring at the same time.
50    private const WHOIS_TTL_MIN = 28 * 86400;
51    private const WHOIS_TTL_MAX = 35 * 86400;
52    private const WHOIS_TIMEOUT = 5;
53
54    // Bad robots - SEO optimisers, advertisers, etc
55    private const BAD_ROBOTS = [
56        'admantx',
57        'AhrefsBot',
58        'AspiegelBot',
59        'DotBot',
60        'Grapeshot',
61        'ia_archiver',
62        'MJ12bot',
63        'panscient',
64        'proximic',
65        'SemrushBot',
66        'XoviBot',
67    ];
68
69    /**
70     * Some search engines use reverse/forward DNS to verify the IP address.
71     *
72     * @see https://support.google.com/webmasters/answer/80553?hl=en
73     * @see https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0
74     * @see https://www.bing.com/webmaster/help/how-to-verify-bingbot-3905dc26
75     * @see https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
76     */
77    private const ROBOT_REV_FWD_DNS = [
78        'bingbot'     => ['.search.msn.com'],
79        'BingPreview' => ['.search.msn.com'],
80        'Google'      => ['.google.com', '.googlebot.com'],
81        'msnbot'      => ['.search.msn.com'],
82        'Qwantify'    => ['.search.qwant.com'],
83        'Sogou'       => ['.crawl.sogou.com'],
84        'Yahoo'       => ['.crawl.yahoo.net'],
85        'Yandex'      => ['.yandex.ru', '.yandex.net', '.yandex.com'],
86    ];
87
88    /**
89     * Some search engines only use reverse DNS to verify the IP address.
90     *
91     * @see https://help.baidu.com/question?prod_id=99&class=0&id=3001
92     */
93    private const ROBOT_REV_ONLY_DNS = [
94        'Baiduspider' => ['.baidu.com', '.baidu.jp'],
95    ];
96
97    /**
98     * Some search engines operate from designated IP addresses.
99     *
100     * @see http://www.apple.com/go/applebot
101     * @see https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
102     */
103    private const ROBOT_IPS = [
104        'AppleBot'    => [
105            '17.0.0.0/8',
106        ],
107        'Ask Jeeves'  => [
108            '65.214.45.143',
109            '65.214.45.148',
110            '66.235.124.192',
111            '66.235.124.7',
112            '66.235.124.101',
113            '66.235.124.193',
114            '66.235.124.73',
115            '66.235.124.196',
116            '66.235.124.74',
117            '63.123.238.8',
118            '202.143.148.61',
119        ],
120        'DuckDuckBot' => [
121            '23.21.227.69',
122            '50.16.241.113',
123            '50.16.241.114',
124            '50.16.241.117',
125            '50.16.247.234',
126            '52.204.97.54',
127            '52.5.190.19',
128            '54.197.234.188',
129            '54.208.100.253',
130            '54.208.102.37',
131            '107.21.1.8',
132        ],
133    ];
134
135    /**
136     * Some search engines operate from within a designated autonomous system.
137     *
138     * @see https://developers.facebook.com/docs/sharing/webmasters/crawler
139     */
140    private const ROBOT_ASN = [
141        'facebook' => 'AS32934',
142        'twitter'  => 'AS13414',
143    ];
144
145    /**
146     * @param ServerRequestInterface  $request
147     * @param RequestHandlerInterface $handler
148     *
149     * @return ResponseInterface
150     */
151    public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
152    {
153        $ua      = $request->getServerParams()['HTTP_USER_AGENT'] ?? '';
154        $ip      = $request->getAttribute('client-ip');
155        $address = Factory::addressFromString($ip);
156        assert($address instanceof AddressInterface);
157
158        if (Str::contains($ua, self::BAD_ROBOTS)) {
159            return $this->response();
160        }
161
162        foreach (self::ROBOT_REV_FWD_DNS as $robot => $valid_domains) {
163            if (Str::contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, false)) {
164                return $this->response();
165            }
166        }
167
168        foreach (self::ROBOT_REV_ONLY_DNS as $robot => $valid_domains) {
169            if (Str::contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, true)) {
170                return $this->response();
171            }
172        }
173
174        foreach (self::ROBOT_IPS as $robot => $valid_ips) {
175            if (Str::contains($ua, $robot)) {
176                foreach ($valid_ips as $ip) {
177                    $range = Factory::rangeFromString($ip);
178
179                    if ($range instanceof RangeInterface && $range->contains($address)) {
180                        continue 2;
181                    }
182                }
183
184                return $this->response();
185            }
186        }
187
188        foreach (self::ROBOT_ASN as $robot => $asn) {
189            if (Str::contains($ua, $robot)) {
190                foreach ($this->fetchIpRangesForAsn($asn) as $range) {
191                    if ($range->contains($address)) {
192                        continue 2;
193                    }
194                }
195
196                return $this->response();
197            }
198        }
199
200        // Allow sites to block access from entire networks.
201        preg_match_all('/(AS\d+)/', $request->getAttribute('block_asn', ''), $matches);
202        foreach ($matches[1] as $asn) {
203            foreach ($this->fetchIpRangesForAsn($asn) as $range) {
204                if ($range->contains($address)) {
205                    return $this->response();
206                }
207            }
208        }
209
210        return $handler->handle($request);
211    }
212
213    /**
214     * Check that an IP address belongs to a robot operator using a forward/reverse DNS lookup.
215     *
216     * @param string        $ip
217     * @param array<string> $valid_domains
218     * @param bool          $reverse_only
219     *
220     * @return bool
221     */
222    private function checkRobotDNS(string $ip, array $valid_domains, bool $reverse_only): bool
223    {
224        $host = gethostbyaddr($ip);
225
226        if ($host === false || !Str::endsWith($host, $valid_domains)) {
227            return false;
228        }
229
230        return $reverse_only || $ip === gethostbyname($host);
231    }
232
233    /**
234     * Perform a whois search for an ASN.
235     *
236     * @param string $asn - The autonomous system number to query
237     *
238     * @return array<RangeInterface>
239     */
240    private function fetchIpRangesForAsn(string $asn): array
241    {
242        $cache = app('cache.files');
243        assert($cache instanceof Cache);
244
245        return $cache->remember('whois-asn-' . $asn, static function () use ($asn): array {
246            try {
247                $loader = new CurlLoader(self::WHOIS_TIMEOUT);
248                $whois  = new Whois($loader);
249                $info   = $whois->loadAsnInfo($asn);
250                $routes = $info->getRoutes();
251                $ranges = array_map(static function (AsnRouteInfo $route_info): ?RangeInterface {
252                    return Factory::rangeFromString($route_info->getRoute() ?: $route_info->getRoute6());
253                }, $routes);
254
255                return array_filter($ranges);
256            } catch (Throwable $ex) {
257                return [];
258            }
259        }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX));
260    }
261
262    /**
263     * @return ResponseInterface
264     */
265    private function response(): ResponseInterface
266    {
267        return response('Not acceptable', StatusCodeInterface::STATUS_NOT_ACCEPTABLE);
268    }
269}
270