1<?php 2 3/** 4 * webtrees: online genealogy 5 * Copyright (C) 2023 webtrees development team 6 * This program is free software: you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation, either version 3 of the License, or 9 * (at your option) any later version. 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * You should have received a copy of the GNU General Public License 15 * along with this program. If not, see <https://www.gnu.org/licenses/>. 16 */ 17 18declare(strict_types=1); 19 20namespace Fisharebest\Webtrees\Http\Middleware; 21 22use Fig\Http\Message\StatusCodeInterface; 23use Fisharebest\Webtrees\Registry; 24use Fisharebest\Webtrees\Validator; 25use GuzzleHttp\Client; 26use GuzzleHttp\Exception\GuzzleException; 27use Iodev\Whois\Loaders\CurlLoader; 28use Iodev\Whois\Modules\Asn\AsnRouteInfo; 29use Iodev\Whois\Whois; 30use IPLib\Address\AddressInterface; 31use IPLib\Factory as IPFactory; 32use IPLib\Range\RangeInterface; 33use Psr\Http\Message\ResponseInterface; 34use Psr\Http\Message\ServerRequestInterface; 35use Psr\Http\Server\MiddlewareInterface; 36use Psr\Http\Server\RequestHandlerInterface; 37use Throwable; 38 39use function array_filter; 40use function array_map; 41use function assert; 42use function gethostbyaddr; 43use function gethostbyname; 44use function preg_match_all; 45use function random_int; 46use function response; 47use function str_contains; 48use function str_ends_with; 49 50/** 51 * Middleware to block bad robots before they waste our valuable CPU cycles. 52 */ 53class BadBotBlocker implements MiddlewareInterface 54{ 55 private const REGEX_OCTET = '(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'; 56 private const REGEX_IPV4 = '/\\b' . self::REGEX_OCTET . '(?:\\.' . self::REGEX_OCTET . '){3}\\b/'; 57 58 // Cache whois requests. Try to avoid all caches expiring at the same time. 59 private const WHOIS_TTL_MIN = 28 * 86400; 60 private const WHOIS_TTL_MAX = 35 * 86400; 61 private const WHOIS_TIMEOUT = 5; 62 63 // Bad robots - SEO optimisers, advertisers, etc. This list is shared with robots.txt. 64 public const BAD_ROBOTS = [ 65 'admantx', 66 'Adsbot', 67 'AhrefsBot', 68 'Amazonbot', // Until it understands crawl-delay and noindex / nofollow 69 'AntBot', // Aggressive crawler 70 'AspiegelBot', 71 'Awario', // Brand management 72 'Barkrowler', 73 'BLEXBot', 74 'Bytespider', 75 'CCBot', // Used to train a number of LLMs 76 'ChatGPT-User', // Used by ChatGPT during operation 77 'ClaudeBot', // Collects training data for LLMs 78 'DataForSeoBot', // https://dataforseo.com/dataforseo-bot 79 'DotBot', 80 'FacebookBot', // Collects training data for Facebook's LLM translator. 81 'fidget-spinner-bot', // Agressive crawler 82 'Google-Extended', // Collects training data for Google Bard 83 'GPTBot', // Collects training data for ChatGPT 84 'Grapeshot', 85 'Honolulu-bot', // Aggressive crawer, no info available 86 'ia_archiver', 87 'linabot', // Aggressive crawer, no info available 88 'Linguee', 89 'MegaIndex.ru', 90 'MJ12bot', 91 'netEstate NE', 92 'Omgilibot', // Collects training data for LLMs 93 'panscient', 94 'PetalBot', 95 'proximic', 96 'SeekportBot', // Pretends to be a search engine - but isn't 97 'SemrushBot', 98 'serpstatbot', 99 'SEOkicks', 100 'SiteKiosk', 101 'test-bot', // Agressive crawler 102 'TinyTestBot', 103 'Turnitin', 104 'wp_is_mobile', // Nothing to do with wordpress 105 'XoviBot', 106 'YisouSpider', 107 'ZoominfoBot', 108 ]; 109 110 /** 111 * Some search engines use reverse/forward DNS to verify the IP address. 112 * 113 * @see https://developer.amazon.com/support/amazonbot 114 * @see https://support.google.com/webmasters/answer/80553?hl=en 115 * @see https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0 116 * @see https://www.bing.com/webmaster/help/how-to-verify-bingbot-3905dc26 117 * @see https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html 118 * @see https://www.mojeek.com/bot.html 119 * @see https://support.apple.com/en-gb/HT204683 120 */ 121 private const ROBOT_REV_FWD_DNS = [ 122 'Amazonbot' => ['.crawl.amazon.com'], 123 'Applebot' => ['.applebot.apple.com'], 124 'BingPreview' => ['.search.msn.com'], 125 'Google' => ['.google.com', '.googlebot.com'], 126 'Mail.RU_Bot' => ['.mail.ru'], 127 'MicrosoftPreview' => ['.search.msn.com'], 128 'MojeekBot' => ['.mojeek.com'], 129 'Qwantify' => ['.qwant.com'], 130 'Sogou' => ['.crawl.sogou.com'], 131 'Yahoo' => ['.crawl.yahoo.net'], 132 'Yandex' => ['.yandex.ru', '.yandex.net', '.yandex.com'], 133 'bingbot' => ['.search.msn.com'], 134 'msnbot' => ['.search.msn.com'], 135 ]; 136 137 /** 138 * Some search engines only use reverse DNS to verify the IP address. 139 * 140 * @see https://help.baidu.com/question?prod_id=99&class=0&id=3001 141 * @see https://napoveda.seznam.cz/en/full-text-search/seznambot-crawler 142 * @see https://www.ionos.de/terms-gtc/faq-crawler 143 */ 144 private const ROBOT_REV_ONLY_DNS = [ 145 'Baiduspider' => ['.baidu.com', '.baidu.jp'], 146 'FreshBot' => ['.seznam.cz'], 147 'IonCrawl' => ['.1und1.org'], 148 'Neevabot' => ['.neeva.com'], 149 'SeznamBot' => ['.seznam.cz'], 150 ]; 151 152 /** 153 * Some search engines operate from designated IP addresses. 154 * 155 * @see https://www.apple.com/go/applebot 156 * @see https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot 157 */ 158 private const ROBOT_IPS = [ 159 'AppleBot' => [ 160 '17.0.0.0/8', 161 ], 162 'Ask Jeeves' => [ 163 '65.214.45.143', 164 '65.214.45.148', 165 '66.235.124.192', 166 '66.235.124.7', 167 '66.235.124.101', 168 '66.235.124.193', 169 '66.235.124.73', 170 '66.235.124.196', 171 '66.235.124.74', 172 '63.123.238.8', 173 '202.143.148.61', 174 ], 175 'DuckDuckBot' => [ 176 '23.21.227.69', 177 '50.16.241.113', 178 '50.16.241.114', 179 '50.16.241.117', 180 '50.16.247.234', 181 '52.204.97.54', 182 '52.5.190.19', 183 '54.197.234.188', 184 '54.208.100.253', 185 '54.208.102.37', 186 '107.21.1.8', 187 ], 188 ]; 189 190 /** 191 * Some search engines operate from designated IP addresses. 192 * 193 * @see https://bot.seekport.com/ 194 */ 195 private const ROBOT_IP_FILES = [ 196 'SeekportBot' => 'https://bot.seekport.com/seekportbot_ips.txt', 197 ]; 198 199 /** 200 * Some search engines operate from within a designated autonomous system. 201 * 202 * @see https://developers.facebook.com/docs/sharing/webmasters/crawler 203 * @see https://www.facebook.com/peering/ 204 */ 205 private const ROBOT_ASNS = [ 206 'facebook' => ['AS32934', 'AS63293'], 207 'twitter' => ['AS13414'], 208 ]; 209 210 /** 211 * @param ServerRequestInterface $request 212 * @param RequestHandlerInterface $handler 213 * 214 * @return ResponseInterface 215 */ 216 public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface 217 { 218 $ua = Validator::serverParams($request)->string('HTTP_USER_AGENT', ''); 219 $ip = Validator::attributes($request)->string('client-ip'); 220 $address = IPFactory::parseAddressString($ip); 221 assert($address instanceof AddressInterface); 222 223 foreach (self::BAD_ROBOTS as $robot) { 224 if (str_contains($ua, $robot)) { 225 return $this->response(); 226 } 227 } 228 229 foreach (self::ROBOT_REV_FWD_DNS as $robot => $valid_domains) { 230 if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, false)) { 231 return $this->response(); 232 } 233 } 234 235 foreach (self::ROBOT_REV_ONLY_DNS as $robot => $valid_domains) { 236 if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, true)) { 237 return $this->response(); 238 } 239 } 240 241 foreach (self::ROBOT_IPS as $robot => $valid_ip_ranges) { 242 if (str_contains($ua, $robot)) { 243 foreach ($valid_ip_ranges as $ip_range) { 244 $range = IPFactory::parseRangeString($ip_range); 245 246 if ($range instanceof RangeInterface && $range->contains($address)) { 247 continue 2; 248 } 249 } 250 251 return $this->response(); 252 } 253 } 254 255 foreach (self::ROBOT_IP_FILES as $robot => $url) { 256 if (str_contains($ua, $robot)) { 257 $valid_ip_ranges = $this->fetchIpRangesForUrl($robot, $url); 258 259 foreach ($valid_ip_ranges as $ip_range) { 260 $range = IPFactory::parseRangeString($ip_range); 261 262 if ($range instanceof RangeInterface && $range->contains($address)) { 263 continue 2; 264 } 265 } 266 267 return $this->response(); 268 } 269 } 270 271 foreach (self::ROBOT_ASNS as $robot => $asns) { 272 foreach ($asns as $asn) { 273 if (str_contains($ua, $robot)) { 274 foreach ($this->fetchIpRangesForAsn($asn) as $range) { 275 if ($range->contains($address)) { 276 continue 2; 277 } 278 } 279 280 return $this->response(); 281 } 282 } 283 } 284 285 // Allow sites to block access from entire networks. 286 $block_asn = Validator::attributes($request)->string('block_asn', ''); 287 preg_match_all('/(AS\d+)/', $block_asn, $matches); 288 289 foreach ($matches[1] as $asn) { 290 foreach ($this->fetchIpRangesForAsn($asn) as $range) { 291 if ($range->contains($address)) { 292 return $this->response(); 293 } 294 } 295 } 296 297 return $handler->handle($request); 298 } 299 300 /** 301 * Check that an IP address belongs to a robot operator using a forward/reverse DNS lookup. 302 * 303 * @param string $ip 304 * @param array<string> $valid_domains 305 * @param bool $reverse_only 306 * 307 * @return bool 308 */ 309 private function checkRobotDNS(string $ip, array $valid_domains, bool $reverse_only): bool 310 { 311 $host = gethostbyaddr($ip); 312 313 if ($host === false) { 314 return false; 315 } 316 317 foreach ($valid_domains as $domain) { 318 if (str_ends_with($host, $domain)) { 319 return $reverse_only || $ip === gethostbyname($host); 320 } 321 } 322 323 return false; 324 } 325 326 /** 327 * Perform a whois search for an ASN. 328 * 329 * @param string $asn The autonomous system number to query 330 * 331 * @return array<RangeInterface> 332 */ 333 private function fetchIpRangesForAsn(string $asn): array 334 { 335 return Registry::cache()->file()->remember('whois-asn-' . $asn, static function () use ($asn): array { 336 $mapper = static fn (AsnRouteInfo $route_info): ?RangeInterface => IPFactory::parseRangeString($route_info->route ?: $route_info->route6); 337 338 try { 339 $loader = new CurlLoader(self::WHOIS_TIMEOUT); 340 $whois = new Whois($loader); 341 $info = $whois->loadAsnInfo($asn); 342 $routes = $info->routes; 343 $ranges = array_map($mapper, $routes); 344 345 return array_filter($ranges); 346 } catch (Throwable) { 347 return []; 348 } 349 }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX)); 350 } 351 352 /** 353 * Fetch a list of IP addresses from a remote file. 354 * 355 * @param string $ua 356 * @param string $url 357 * 358 * @return array<string> 359 */ 360 private function fetchIpRangesForUrl(string $ua, string $url): array 361 { 362 return Registry::cache()->file()->remember('url-ip-list-' . $ua, static function () use ($url): array { 363 try { 364 $client = new Client(); 365 $response = $client->get($url, ['timeout' => 5]); 366 $contents = $response->getBody()->getContents(); 367 368 preg_match_all(self::REGEX_IPV4, $contents, $matches); 369 370 return $matches[0]; 371 } catch (GuzzleException) { 372 return []; 373 } 374 }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX)); 375 } 376 377 /** 378 * @return ResponseInterface 379 */ 380 private function response(): ResponseInterface 381 { 382 return response('Not acceptable', StatusCodeInterface::STATUS_NOT_ACCEPTABLE); 383 } 384} 385