1<?php 2 3/** 4 * webtrees: online genealogy 5 * Copyright (C) 2023 webtrees development team 6 * This program is free software: you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation, either version 3 of the License, or 9 * (at your option) any later version. 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * You should have received a copy of the GNU General Public License 15 * along with this program. If not, see <https://www.gnu.org/licenses/>. 16 */ 17 18declare(strict_types=1); 19 20namespace Fisharebest\Webtrees\Services; 21 22use Fisharebest\Webtrees\Auth; 23use Fisharebest\Webtrees\DB; 24use Fisharebest\Webtrees\Encodings\UTF16BE; 25use Fisharebest\Webtrees\Encodings\UTF16LE; 26use Fisharebest\Webtrees\Encodings\UTF8; 27use Fisharebest\Webtrees\Encodings\Windows1252; 28use Fisharebest\Webtrees\Factories\AbstractGedcomRecordFactory; 29use Fisharebest\Webtrees\Gedcom; 30use Fisharebest\Webtrees\GedcomFilters\GedcomEncodingFilter; 31use Fisharebest\Webtrees\GedcomRecord; 32use Fisharebest\Webtrees\Header; 33use Fisharebest\Webtrees\Registry; 34use Fisharebest\Webtrees\Tree; 35use Fisharebest\Webtrees\Webtrees; 36use Illuminate\Database\Query\Builder; 37use Illuminate\Database\Query\Expression; 38use Illuminate\Support\Collection; 39use League\Flysystem\Filesystem; 40use League\Flysystem\FilesystemOperator; 41use League\Flysystem\ZipArchive\FilesystemZipArchiveProvider; 42use League\Flysystem\ZipArchive\ZipArchiveAdapter; 43use Psr\Http\Message\ResponseFactoryInterface; 44use Psr\Http\Message\ResponseInterface; 45use Psr\Http\Message\StreamFactoryInterface; 46use RuntimeException; 47 48use function addcslashes; 49use function date; 50use function explode; 51use function fclose; 52use function fopen; 53use function fwrite; 54use function is_string; 55use function pathinfo; 56use function preg_match_all; 57use function rewind; 58use function stream_filter_append; 59use function stream_get_meta_data; 60use function strlen; 61use function strpos; 62use function strtolower; 63use function strtoupper; 64use function tmpfile; 65 66use const PATHINFO_EXTENSION; 67use const PREG_SET_ORDER; 68use const STREAM_FILTER_WRITE; 69 70/** 71 * Export data in GEDCOM format 72 */ 73class GedcomExportService 74{ 75 private const ACCESS_LEVELS = [ 76 'gedadmin' => Auth::PRIV_NONE, 77 'user' => Auth::PRIV_USER, 78 'visitor' => Auth::PRIV_PRIVATE, 79 'none' => Auth::PRIV_HIDE, 80 ]; 81 82 private ResponseFactoryInterface $response_factory; 83 84 private StreamFactoryInterface $stream_factory; 85 86 /** 87 * @param ResponseFactoryInterface $response_factory 88 * @param StreamFactoryInterface $stream_factory 89 */ 90 public function __construct(ResponseFactoryInterface $response_factory, StreamFactoryInterface $stream_factory) 91 { 92 $this->response_factory = $response_factory; 93 $this->stream_factory = $stream_factory; 94 } 95 96 /** 97 * @param Tree $tree Export data from this tree 98 * @param bool $sort_by_xref Write GEDCOM records in XREF order 99 * @param string $encoding Convert from UTF-8 to other encoding 100 * @param string $privacy Filter records by role 101 * @param string $line_endings 102 * @param string $filename Name of download file, without an extension 103 * @param string $format One of: gedcom, zip, zipmedia, gedzip 104 * @param Collection<int,string|object|GedcomRecord>|null $records 105 * 106 * @return ResponseInterface 107 */ 108 public function downloadResponse( 109 Tree $tree, 110 bool $sort_by_xref, 111 string $encoding, 112 string $privacy, 113 string $line_endings, 114 string $filename, 115 string $format, 116 Collection $records = null 117 ): ResponseInterface { 118 $access_level = self::ACCESS_LEVELS[$privacy]; 119 120 if ($format === 'gedcom') { 121 $resource = $this->export($tree, $sort_by_xref, $encoding, $access_level, $line_endings, $records); 122 $stream = $this->stream_factory->createStreamFromResource($resource); 123 124 return $this->response_factory->createResponse() 125 ->withBody($stream) 126 ->withHeader('content-type', 'text/x-gedcom; charset=' . UTF8::NAME) 127 ->withHeader('content-disposition', 'attachment; filename="' . addcslashes($filename, '"') . '.ged"'); 128 } 129 130 // Create a new/empty .ZIP file 131 $temp_zip_file = stream_get_meta_data(tmpfile())['uri']; 132 $zip_provider = new FilesystemZipArchiveProvider($temp_zip_file, 0755); 133 $zip_adapter = new ZipArchiveAdapter($zip_provider); 134 $zip_filesystem = new Filesystem($zip_adapter); 135 136 if ($format === 'zipmedia') { 137 $media_path = $tree->getPreference('MEDIA_DIRECTORY'); 138 } elseif ($format === 'gedzip') { 139 $media_path = ''; 140 } else { 141 // Don't add media 142 $media_path = null; 143 } 144 145 $resource = $this->export($tree, $sort_by_xref, $encoding, $access_level, $line_endings, $records, $zip_filesystem, $media_path); 146 147 if ($format === 'gedzip') { 148 $zip_filesystem->writeStream('gedcom.ged', $resource); 149 $extension = '.gdz'; 150 } else { 151 $zip_filesystem->writeStream($filename . '.ged', $resource); 152 $extension = '.zip'; 153 } 154 155 fclose($resource); 156 157 $stream = $this->stream_factory->createStreamFromFile($temp_zip_file); 158 159 return $this->response_factory->createResponse() 160 ->withBody($stream) 161 ->withHeader('content-type', 'application/zip') 162 ->withHeader('content-disposition', 'attachment; filename="' . addcslashes($filename, '"') . $extension . '"'); 163 } 164 165 /** 166 * Write GEDCOM data to a stream. 167 * 168 * @param Tree $tree Export data from this tree 169 * @param bool $sort_by_xref Write GEDCOM records in XREF order 170 * @param string $encoding Convert from UTF-8 to other encoding 171 * @param int $access_level Apply privacy filtering 172 * @param string $line_endings CRLF or LF 173 * @param Collection<int,string|object|GedcomRecord>|null $records Just export these records 174 * @param FilesystemOperator|null $zip_filesystem Write media files to this filesystem 175 * @param string|null $media_path Location within the zip filesystem 176 * 177 * @return resource 178 */ 179 public function export( 180 Tree $tree, 181 bool $sort_by_xref = false, 182 string $encoding = UTF8::NAME, 183 int $access_level = Auth::PRIV_HIDE, 184 string $line_endings = 'CRLF', 185 Collection|null $records = null, 186 FilesystemOperator|null $zip_filesystem = null, 187 string $media_path = null 188 ) { 189 $stream = fopen('php://memory', 'wb+'); 190 191 if ($stream === false) { 192 throw new RuntimeException('Failed to create temporary stream'); 193 } 194 195 stream_filter_append($stream, GedcomEncodingFilter::class, STREAM_FILTER_WRITE, ['src_encoding' => UTF8::NAME, 'dst_encoding' => $encoding]); 196 197 if ($records instanceof Collection) { 198 // Export just these records - e.g. from clippings cart. 199 $data = [ 200 new Collection([$this->createHeader($tree, $encoding, false)]), 201 $records, 202 new Collection(['0 TRLR']), 203 ]; 204 } elseif ($access_level === Auth::PRIV_HIDE) { 205 // If we will be applying privacy filters, then we will need the GEDCOM record objects. 206 $data = [ 207 new Collection([$this->createHeader($tree, $encoding, true)]), 208 $this->individualQuery($tree, $sort_by_xref)->cursor(), 209 $this->familyQuery($tree, $sort_by_xref)->cursor(), 210 $this->sourceQuery($tree, $sort_by_xref)->cursor(), 211 $this->otherQuery($tree, $sort_by_xref)->cursor(), 212 $this->mediaQuery($tree, $sort_by_xref)->cursor(), 213 new Collection(['0 TRLR']), 214 ]; 215 } else { 216 // Disable the pending changes before creating GEDCOM records. 217 Registry::cache()->array()->remember(AbstractGedcomRecordFactory::class . $tree->id(), static fn(): Collection => new Collection()); 218 219 $data = [ 220 new Collection([$this->createHeader($tree, $encoding, true)]), 221 $this->individualQuery($tree, $sort_by_xref)->get()->map(Registry::individualFactory()->mapper($tree)), 222 $this->familyQuery($tree, $sort_by_xref)->get()->map(Registry::familyFactory()->mapper($tree)), 223 $this->sourceQuery($tree, $sort_by_xref)->get()->map(Registry::sourceFactory()->mapper($tree)), 224 $this->otherQuery($tree, $sort_by_xref)->get()->map(Registry::gedcomRecordFactory()->mapper($tree)), 225 $this->mediaQuery($tree, $sort_by_xref)->get()->map(Registry::mediaFactory()->mapper($tree)), 226 new Collection(['0 TRLR']), 227 ]; 228 } 229 230 $media_filesystem = $tree->mediaFilesystem(); 231 232 foreach ($data as $rows) { 233 foreach ($rows as $datum) { 234 if (is_string($datum)) { 235 $gedcom = $datum; 236 } elseif ($datum instanceof GedcomRecord) { 237 $gedcom = $datum->privatizeGedcom($access_level); 238 } else { 239 $gedcom = 240 $datum->i_gedcom ?? 241 $datum->f_gedcom ?? 242 $datum->s_gedcom ?? 243 $datum->m_gedcom ?? 244 $datum->o_gedcom; 245 } 246 247 if ($media_path !== null && $zip_filesystem !== null && preg_match('/0 @' . Gedcom::REGEX_XREF . '@ OBJE/', $gedcom) === 1) { 248 preg_match_all('/\n1 FILE (.+)/', $gedcom, $matches, PREG_SET_ORDER); 249 250 foreach ($matches as $match) { 251 $media_file = $match[1]; 252 253 if ($media_filesystem->fileExists($media_file)) { 254 $zip_filesystem->writeStream($media_path . $media_file, $media_filesystem->readStream($media_file)); 255 } 256 } 257 } 258 259 $gedcom = $this->wrapLongLines($gedcom, Gedcom::LINE_LENGTH) . "\n"; 260 261 if ($line_endings === 'CRLF') { 262 $gedcom = strtr($gedcom, ["\n" => "\r\n"]); 263 } 264 265 $bytes_written = fwrite($stream, $gedcom); 266 267 if ($bytes_written !== strlen($gedcom)) { 268 throw new RuntimeException('Unable to write to stream. Perhaps the disk is full?'); 269 } 270 } 271 } 272 273 if (rewind($stream) === false) { 274 throw new RuntimeException('Cannot rewind temporary stream'); 275 } 276 277 return $stream; 278 } 279 280 /** 281 * Create a header record for a gedcom file. 282 * 283 * @param Tree $tree 284 * @param string $encoding 285 * @param bool $include_sub 286 * 287 * @return string 288 */ 289 public function createHeader(Tree $tree, string $encoding, bool $include_sub): string 290 { 291 // Force a ".ged" suffix 292 $filename = $tree->name(); 293 294 if (strtolower(pathinfo($filename, PATHINFO_EXTENSION)) !== 'ged') { 295 $filename .= '.ged'; 296 } 297 298 $gedcom_encodings = [ 299 UTF16BE::NAME => 'UNICODE', 300 UTF16LE::NAME => 'UNICODE', 301 Windows1252::NAME => 'ANSI', 302 ]; 303 304 $encoding = $gedcom_encodings[$encoding] ?? $encoding; 305 306 // Build a new header record 307 $gedcom = '0 HEAD'; 308 $gedcom .= "\n1 SOUR " . Webtrees::NAME; 309 $gedcom .= "\n2 NAME " . Webtrees::NAME; 310 $gedcom .= "\n2 VERS " . Webtrees::VERSION; 311 $gedcom .= "\n1 DEST DISKETTE"; 312 $gedcom .= "\n1 DATE " . strtoupper(date('d M Y')); 313 $gedcom .= "\n2 TIME " . date('H:i:s'); 314 $gedcom .= "\n1 GEDC\n2 VERS 5.5.1\n2 FORM LINEAGE-LINKED"; 315 $gedcom .= "\n1 CHAR " . $encoding; 316 $gedcom .= "\n1 FILE " . $filename; 317 318 // Preserve some values from the original header 319 $header = Registry::headerFactory()->make('HEAD', $tree) ?? Registry::headerFactory()->new('HEAD', '0 HEAD', null, $tree); 320 321 // There should always be a header record. 322 if ($header instanceof Header) { 323 foreach ($header->facts(['COPR', 'LANG', 'PLAC', 'NOTE']) as $fact) { 324 $gedcom .= "\n" . $fact->gedcom(); 325 } 326 327 if ($include_sub) { 328 foreach ($header->facts(['SUBM', 'SUBN']) as $fact) { 329 $gedcom .= "\n" . $fact->gedcom(); 330 } 331 } 332 } 333 334 return $gedcom; 335 } 336 337 /** 338 * Wrap long lines using concatenation records. 339 * 340 * @param string $gedcom 341 * @param int $max_line_length 342 * 343 * @return string 344 */ 345 public function wrapLongLines(string $gedcom, int $max_line_length): string 346 { 347 $lines = []; 348 349 foreach (explode("\n", $gedcom) as $line) { 350 // Split long lines 351 // The total length of a GEDCOM line, including level number, cross-reference number, 352 // tag, value, delimiters, and terminator, must not exceed 255 (wide) characters. 353 if (mb_strlen($line) > $max_line_length) { 354 [$level, $tag] = explode(' ', $line, 3); 355 if ($tag !== 'CONT') { 356 $level++; 357 } 358 do { 359 // Split after $pos chars 360 $pos = $max_line_length; 361 // Split on a non-space (standard gedcom behavior) 362 while (mb_substr($line, $pos - 1, 1) === ' ') { 363 --$pos; 364 } 365 if ($pos === strpos($line, ' ', 3)) { 366 // No non-spaces in the data! Can’t split it :-( 367 break; 368 } 369 $lines[] = mb_substr($line, 0, $pos); 370 $line = $level . ' CONC ' . mb_substr($line, $pos); 371 } while (mb_strlen($line) > $max_line_length); 372 } 373 $lines[] = $line; 374 } 375 376 return implode("\n", $lines); 377 } 378 379 /** 380 * @param Tree $tree 381 * @param bool $sort_by_xref 382 * 383 * @return Builder 384 */ 385 private function familyQuery(Tree $tree, bool $sort_by_xref): Builder 386 { 387 $query = DB::table('families') 388 ->where('f_file', '=', $tree->id()) 389 ->select(['f_gedcom', 'f_id']); 390 391 392 if ($sort_by_xref) { 393 $query 394 ->orderBy(new Expression('LENGTH(f_id)')) 395 ->orderBy('f_id'); 396 } 397 398 return $query; 399 } 400 401 /** 402 * @param Tree $tree 403 * @param bool $sort_by_xref 404 * 405 * @return Builder 406 */ 407 private function individualQuery(Tree $tree, bool $sort_by_xref): Builder 408 { 409 $query = DB::table('individuals') 410 ->where('i_file', '=', $tree->id()) 411 ->select(['i_gedcom', 'i_id']); 412 413 if ($sort_by_xref) { 414 $query 415 ->orderBy(new Expression('LENGTH(i_id)')) 416 ->orderBy('i_id'); 417 } 418 419 return $query; 420 } 421 422 /** 423 * @param Tree $tree 424 * @param bool $sort_by_xref 425 * 426 * @return Builder 427 */ 428 private function sourceQuery(Tree $tree, bool $sort_by_xref): Builder 429 { 430 $query = DB::table('sources') 431 ->where('s_file', '=', $tree->id()) 432 ->select(['s_gedcom', 's_id']); 433 434 if ($sort_by_xref) { 435 $query 436 ->orderBy(new Expression('LENGTH(s_id)')) 437 ->orderBy('s_id'); 438 } 439 440 return $query; 441 } 442 443 /** 444 * @param Tree $tree 445 * @param bool $sort_by_xref 446 * 447 * @return Builder 448 */ 449 private function mediaQuery(Tree $tree, bool $sort_by_xref): Builder 450 { 451 $query = DB::table('media') 452 ->where('m_file', '=', $tree->id()) 453 ->select(['m_gedcom', 'm_id']); 454 455 if ($sort_by_xref) { 456 $query 457 ->orderBy(new Expression('LENGTH(m_id)')) 458 ->orderBy('m_id'); 459 } 460 461 return $query; 462 } 463 464 /** 465 * @param Tree $tree 466 * @param bool $sort_by_xref 467 * 468 * @return Builder 469 */ 470 private function otherQuery(Tree $tree, bool $sort_by_xref): Builder 471 { 472 $query = DB::table('other') 473 ->where('o_file', '=', $tree->id()) 474 ->whereNotIn('o_type', [Header::RECORD_TYPE, 'TRLR']) 475 ->select(['o_gedcom', 'o_id']); 476 477 if ($sort_by_xref) { 478 $query 479 ->orderBy('o_type') 480 ->orderBy(new Expression('LENGTH(o_id)')) 481 ->orderBy('o_id'); 482 } 483 484 return $query; 485 } 486} 487