1<?php 2 3/** 4 * webtrees: online genealogy 5 * Copyright (C) 2023 webtrees development team 6 * This program is free software: you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation, either version 3 of the License, or 9 * (at your option) any later version. 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * You should have received a copy of the GNU General Public License 15 * along with this program. If not, see <https://www.gnu.org/licenses/>. 16 */ 17 18declare(strict_types=1); 19 20namespace Fisharebest\Webtrees\Http\Middleware; 21 22use Fig\Http\Message\StatusCodeInterface; 23use Fisharebest\Webtrees\Registry; 24use Fisharebest\Webtrees\Validator; 25use GuzzleHttp\Client; 26use GuzzleHttp\Exception\GuzzleException; 27use Iodev\Whois\Loaders\CurlLoader; 28use Iodev\Whois\Modules\Asn\AsnRouteInfo; 29use Iodev\Whois\Whois; 30use IPLib\Address\AddressInterface; 31use IPLib\Factory as IPFactory; 32use IPLib\Range\RangeInterface; 33use Psr\Http\Message\ResponseInterface; 34use Psr\Http\Message\ServerRequestInterface; 35use Psr\Http\Server\MiddlewareInterface; 36use Psr\Http\Server\RequestHandlerInterface; 37use Throwable; 38 39use function array_filter; 40use function array_map; 41use function assert; 42use function gethostbyaddr; 43use function gethostbyname; 44use function preg_match_all; 45use function random_int; 46use function response; 47use function str_contains; 48use function str_ends_with; 49 50/** 51 * Middleware to block bad robots before they waste our valuable CPU cycles. 52 */ 53class BadBotBlocker implements MiddlewareInterface 54{ 55 private const REGEX_OCTET = '(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'; 56 private const REGEX_IPV4 = '/\\b' . self::REGEX_OCTET . '(?:\\.' . self::REGEX_OCTET . '){3}\\b/'; 57 58 // Cache whois requests. Try to avoid all caches expiring at the same time. 59 private const WHOIS_TTL_MIN = 28 * 86400; 60 private const WHOIS_TTL_MAX = 35 * 86400; 61 private const WHOIS_TIMEOUT = 5; 62 63 // Bad robots - SEO optimisers, advertisers, etc. This list is shared with robots.txt. 64 public const BAD_ROBOTS = [ 65 'admantx', 66 'Adsbot', 67 'AhrefsBot', 68 'Amazonbot', // Until it understands crawl-delay and noindex / nofollow 69 'AspiegelBot', 70 'Awario', // Brand management 71 'Barkrowler', 72 'BLEXBot', 73 'DataForSEO', 74 'DataForSeoBot', // https://dataforseo.com/dataforseo-bot 75 'DotBot', 76 'Grapeshot', 77 'Honolulu-bot', // Aggressive crawer, no info available 78 'ia_archiver', 79 'linabot', // Aggressive crawer, no info available 80 'Linguee', 81 'MegaIndex.ru', 82 'MJ12bot', 83 'netEstate NE', 84 'panscient', 85 'PetalBot', 86 'proximic', 87 'SeekportBot', // Pretends to be a search engine - but isn't 88 'SemrushBot', 89 'serpstatbot', 90 'SEOkicks', 91 'SiteKiosk', 92 'Turnitin', 93 'wp_is_mobile', // Nothing to do with wordpress 94 'XoviBot', 95 'ZoominfoBot', 96 ]; 97 98 /** 99 * Some search engines use reverse/forward DNS to verify the IP address. 100 * 101 * @see https://developer.amazon.com/support/amazonbot 102 * @see https://support.google.com/webmasters/answer/80553?hl=en 103 * @see https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0 104 * @see https://www.bing.com/webmaster/help/how-to-verify-bingbot-3905dc26 105 * @see https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html 106 * @see https://www.mojeek.com/bot.html 107 * @see https://support.apple.com/en-gb/HT204683 108 */ 109 private const ROBOT_REV_FWD_DNS = [ 110 'Amazonbot' => ['.crawl.amazon.com'], 111 'Applebot' => ['.applebot.apple.com'], 112 'BingPreview' => ['.search.msn.com'], 113 'Google' => ['.google.com', '.googlebot.com'], 114 'Mail.RU_Bot' => ['.mail.ru'], 115 'MicrosoftPreview' => ['.search.msn.com'], 116 'MojeekBot' => ['.mojeek.com'], 117 'Qwantify' => ['.search.qwant.com'], 118 'Sogou' => ['.crawl.sogou.com'], 119 'Yahoo' => ['.crawl.yahoo.net'], 120 'Yandex' => ['.yandex.ru', '.yandex.net', '.yandex.com'], 121 'bingbot' => ['.search.msn.com'], 122 'msnbot' => ['.search.msn.com'], 123 ]; 124 125 /** 126 * Some search engines only use reverse DNS to verify the IP address. 127 * 128 * @see https://help.baidu.com/question?prod_id=99&class=0&id=3001 129 * @see https://napoveda.seznam.cz/en/full-text-search/seznambot-crawler 130 * @see https://www.ionos.de/terms-gtc/faq-crawler 131 */ 132 private const ROBOT_REV_ONLY_DNS = [ 133 'Baiduspider' => ['.baidu.com', '.baidu.jp'], 134 'FreshBot' => ['.seznam.cz'], 135 'IonCrawl' => ['.1und1.org'], 136 'Neevabot' => ['.neeva.com'], 137 'SeznamBot' => ['.seznam.cz'], 138 ]; 139 140 /** 141 * Some search engines operate from designated IP addresses. 142 * 143 * @see https://www.apple.com/go/applebot 144 * @see https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot 145 */ 146 private const ROBOT_IPS = [ 147 'AppleBot' => [ 148 '17.0.0.0/8', 149 ], 150 'Ask Jeeves' => [ 151 '65.214.45.143', 152 '65.214.45.148', 153 '66.235.124.192', 154 '66.235.124.7', 155 '66.235.124.101', 156 '66.235.124.193', 157 '66.235.124.73', 158 '66.235.124.196', 159 '66.235.124.74', 160 '63.123.238.8', 161 '202.143.148.61', 162 ], 163 'DuckDuckBot' => [ 164 '23.21.227.69', 165 '50.16.241.113', 166 '50.16.241.114', 167 '50.16.241.117', 168 '50.16.247.234', 169 '52.204.97.54', 170 '52.5.190.19', 171 '54.197.234.188', 172 '54.208.100.253', 173 '54.208.102.37', 174 '107.21.1.8', 175 ], 176 ]; 177 178 /** 179 * Some search engines operate from designated IP addresses. 180 * 181 * @see https://bot.seekport.com/ 182 */ 183 private const ROBOT_IP_FILES = [ 184 'SeekportBot' => 'https://bot.seekport.com/seekportbot_ips.txt', 185 ]; 186 187 /** 188 * Some search engines operate from within a designated autonomous system. 189 * 190 * @see https://developers.facebook.com/docs/sharing/webmasters/crawler 191 * @see https://www.facebook.com/peering/ 192 */ 193 private const ROBOT_ASNS = [ 194 'facebook' => ['AS32934', 'AS63293'], 195 'twitter' => ['AS13414'], 196 ]; 197 198 /** 199 * @param ServerRequestInterface $request 200 * @param RequestHandlerInterface $handler 201 * 202 * @return ResponseInterface 203 */ 204 public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface 205 { 206 $ua = Validator::serverParams($request)->string('HTTP_USER_AGENT', ''); 207 $ip = Validator::attributes($request)->string('client-ip'); 208 $address = IPFactory::parseAddressString($ip); 209 assert($address instanceof AddressInterface); 210 211 foreach (self::BAD_ROBOTS as $robot) { 212 if (str_contains($ua, $robot)) { 213 return $this->response(); 214 } 215 } 216 217 foreach (self::ROBOT_REV_FWD_DNS as $robot => $valid_domains) { 218 if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, false)) { 219 return $this->response(); 220 } 221 } 222 223 foreach (self::ROBOT_REV_ONLY_DNS as $robot => $valid_domains) { 224 if (str_contains($ua, $robot) && !$this->checkRobotDNS($ip, $valid_domains, true)) { 225 return $this->response(); 226 } 227 } 228 229 foreach (self::ROBOT_IPS as $robot => $valid_ip_ranges) { 230 if (str_contains($ua, $robot)) { 231 foreach ($valid_ip_ranges as $ip_range) { 232 $range = IPFactory::parseRangeString($ip_range); 233 234 if ($range instanceof RangeInterface && $range->contains($address)) { 235 continue 2; 236 } 237 } 238 239 return $this->response(); 240 } 241 } 242 243 foreach (self::ROBOT_IP_FILES as $robot => $url) { 244 if (str_contains($ua, $robot)) { 245 $valid_ip_ranges = $this->fetchIpRangesForUrl($robot, $url); 246 247 foreach ($valid_ip_ranges as $ip_range) { 248 $range = IPFactory::parseRangeString($ip_range); 249 250 if ($range instanceof RangeInterface && $range->contains($address)) { 251 continue 2; 252 } 253 } 254 255 return $this->response(); 256 } 257 } 258 259 foreach (self::ROBOT_ASNS as $robot => $asns) { 260 foreach ($asns as $asn) { 261 if (str_contains($ua, $robot)) { 262 foreach ($this->fetchIpRangesForAsn($asn) as $range) { 263 if ($range->contains($address)) { 264 continue 2; 265 } 266 } 267 268 return $this->response(); 269 } 270 } 271 } 272 273 // Allow sites to block access from entire networks. 274 $block_asn = Validator::attributes($request)->string('block_asn', ''); 275 preg_match_all('/(AS\d+)/', $block_asn, $matches); 276 277 foreach ($matches[1] as $asn) { 278 foreach ($this->fetchIpRangesForAsn($asn) as $range) { 279 if ($range->contains($address)) { 280 return $this->response(); 281 } 282 } 283 } 284 285 return $handler->handle($request); 286 } 287 288 /** 289 * Check that an IP address belongs to a robot operator using a forward/reverse DNS lookup. 290 * 291 * @param string $ip 292 * @param array<string> $valid_domains 293 * @param bool $reverse_only 294 * 295 * @return bool 296 */ 297 private function checkRobotDNS(string $ip, array $valid_domains, bool $reverse_only): bool 298 { 299 $host = gethostbyaddr($ip); 300 301 if ($host === false) { 302 return false; 303 } 304 305 foreach ($valid_domains as $domain) { 306 if (str_ends_with($host, $domain)) { 307 return $reverse_only || $ip === gethostbyname($host); 308 } 309 } 310 311 return false; 312 } 313 314 /** 315 * Perform a whois search for an ASN. 316 * 317 * @param string $asn - The autonomous system number to query 318 * 319 * @return array<RangeInterface> 320 */ 321 private function fetchIpRangesForAsn(string $asn): array 322 { 323 return Registry::cache()->file()->remember('whois-asn-' . $asn, static function () use ($asn): array { 324 $mapper = static fn (AsnRouteInfo $route_info): ?RangeInterface => IPFactory::parseRangeString($route_info->route ?: $route_info->route6); 325 326 try { 327 $loader = new CurlLoader(self::WHOIS_TIMEOUT); 328 $whois = new Whois($loader); 329 $info = $whois->loadAsnInfo($asn); 330 $routes = $info->routes; 331 $ranges = array_map($mapper, $routes); 332 333 return array_filter($ranges); 334 } catch (Throwable) { 335 return []; 336 } 337 }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX)); 338 } 339 340 /** 341 * Fetch a list of IP addresses from a remote file. 342 * 343 * @param string $ua 344 * @param string $url 345 * 346 * @return array<string> 347 */ 348 private function fetchIpRangesForUrl(string $ua, string $url): array 349 { 350 return Registry::cache()->file()->remember('url-ip-list-' . $ua, static function () use ($url): array { 351 try { 352 $client = new Client(); 353 $response = $client->get($url, ['timeout' => 5]); 354 $contents = $response->getBody()->getContents(); 355 356 preg_match_all(self::REGEX_IPV4, $contents, $matches); 357 358 return $matches[0]; 359 } catch (GuzzleException) { 360 return []; 361 } 362 }, random_int(self::WHOIS_TTL_MIN, self::WHOIS_TTL_MAX)); 363 } 364 365 /** 366 * @return ResponseInterface 367 */ 368 private function response(): ResponseInterface 369 { 370 return response('Not acceptable', StatusCodeInterface::STATUS_NOT_ACCEPTABLE); 371 } 372} 373