xref: /haiku/src/kits/package/hpkg/PackageWriterImpl.cpp (revision abbeb15ac4f2b6524852c09a35db7b3b90b0cd3c) 
1 /*
2  * Copyright 2009-2011, Ingo Weinhold, ingo_weinhold@gmx.de.
3  * Copyright 2011, Oliver Tappe <zooey@hirschkaefer.de>
4  * Distributed under the terms of the MIT License.
5  */
6 
7 
8 #include <package/hpkg/PackageWriterImpl.h>
9 
10 #include <dirent.h>
11 #include <errno.h>
12 #include <fcntl.h>
13 #include <stdio.h>
14 #include <stdlib.h>
15 #include <string.h>
16 #include <sys/stat.h>
17 #include <unistd.h>
18 
19 #include <algorithm>
20 #include <new>
21 
22 #include <ByteOrder.h>
23 #include <Directory.h>
24 #include <Entry.h>
25 #include <FindDirectory.h>
26 #include <fs_attr.h>
27 #include <Path.h>
28 
29 #include <AutoDeleter.h>
30 
31 #include <package/hpkg/HPKGDefsPrivate.h>
32 
33 #include <package/hpkg/DataOutput.h>
34 #include <package/hpkg/DataReader.h>
35 #include <package/hpkg/Stacker.h>
36 
37 
38 using BPrivate::FileDescriptorCloser;
39 
40 
41 static const char* const kPublicDomainLicenseName = "Public Domain";
42 
43 
44 #include <typeinfo>
45 
46 namespace BPackageKit {
47 
48 namespace BHPKG {
49 
50 namespace BPrivate {
51 
52 
53 // minimum length of data we require before trying to zlib compress them
54 static const size_t kZlibCompressionSizeThreshold = 64;
55 
56 
57 // #pragma mark - Attributes
58 
59 
60 struct PackageWriterImpl::Attribute
61 	: public DoublyLinkedListLinkImpl<Attribute> {
62 	BHPKGAttributeID			id;
63 	AttributeValue				value;
64 	DoublyLinkedList<Attribute>	children;
65 
66 	Attribute(BHPKGAttributeID id_ = B_HPKG_ATTRIBUTE_ID_ENUM_COUNT)
67 		:
68 		id(id_)
69 	{
70 	}
71 
72 	~Attribute()
73 	{
74 		DeleteChildren();
75 	}
76 
77 	void AddChild(Attribute* child)
78 	{
79 		children.Add(child);
80 	}
81 
82 	void DeleteChildren()
83 	{
84 		while (Attribute* child = children.RemoveHead())
85 			delete child;
86 	}
87 };
88 
89 
90 // #pragma mark - Entry
91 
92 
93 struct PackageWriterImpl::Entry : DoublyLinkedListLinkImpl<Entry> {
94 	Entry(char* name, size_t nameLength, int fd, bool isImplicit)
95 		:
96 		fName(name),
97 		fNameLength(nameLength),
98 		fFD(fd),
99 		fIsImplicit(isImplicit)
100 	{
101 	}
102 
103 	~Entry()
104 	{
105 		DeleteChildren();
106 		free(fName);
107 	}
108 
109 	static Entry* Create(const char* name, size_t nameLength, int fd,
110 		bool isImplicit)
111 	{
112 		char* clonedName = (char*)malloc(nameLength + 1);
113 		if (clonedName == NULL)
114 			throw std::bad_alloc();
115 		memcpy(clonedName, name, nameLength);
116 		clonedName[nameLength] = '\0';
117 
118 		Entry* entry = new(std::nothrow) Entry(clonedName, nameLength, fd,
119 			isImplicit);
120 		if (entry == NULL) {
121 			free(clonedName);
122 			throw std::bad_alloc();
123 		}
124 
125 		return entry;
126 	}
127 
128 	const char* Name() const
129 	{
130 		return fName;
131 	}
132 
133 	int FD() const
134 	{
135 		return fFD;
136 	}
137 
138 	void SetFD(int fd)
139 	{
140 		fFD = fd;
141 	}
142 
143 	bool IsImplicit() const
144 	{
145 		return fIsImplicit;
146 	}
147 
148 	void SetImplicit(bool isImplicit)
149 	{
150 		fIsImplicit = isImplicit;
151 	}
152 
153 	bool HasName(const char* name, size_t nameLength)
154 	{
155 		return nameLength == fNameLength
156 			&& strncmp(name, fName, nameLength) == 0;
157 	}
158 
159 	void AddChild(Entry* child)
160 	{
161 		fChildren.Add(child);
162 	}
163 
164 	void DeleteChildren()
165 	{
166 		while (Entry* child = fChildren.RemoveHead())
167 			delete child;
168 	}
169 
170 	Entry* GetChild(const char* name, size_t nameLength) const
171 	{
172 		EntryList::ConstIterator it = fChildren.GetIterator();
173 		while (Entry* child = it.Next()) {
174 			if (child->HasName(name, nameLength))
175 				return child;
176 		}
177 
178 		return NULL;
179 	}
180 
181 	EntryList::ConstIterator ChildIterator() const
182 	{
183 		return fChildren.GetIterator();
184 	}
185 
186 private:
187 	char*		fName;
188 	size_t		fNameLength;
189 	int			fFD;
190 	bool		fIsImplicit;
191 	EntryList	fChildren;
192 };
193 
194 
195 // #pragma mark - SubPathAdder
196 
197 
198 struct PackageWriterImpl::SubPathAdder {
199 	SubPathAdder(BErrorOutput* errorOutput, char* pathBuffer,
200 		const char* subPath)
201 		:
202 		fOriginalPathEnd(pathBuffer + strlen(pathBuffer))
203 	{
204 		if (fOriginalPathEnd != pathBuffer)
205 			strlcat(pathBuffer, "/", B_PATH_NAME_LENGTH);
206 
207 		if (strlcat(pathBuffer, subPath, B_PATH_NAME_LENGTH)
208 				>= B_PATH_NAME_LENGTH) {
209 			*fOriginalPathEnd = '\0';
210 			errorOutput->PrintError("Path too long: \"%s/%s\"\n", pathBuffer,
211 				subPath);
212 			throw status_t(B_BUFFER_OVERFLOW);
213 		}
214 	}
215 
216 	~SubPathAdder()
217 	{
218 		*fOriginalPathEnd = '\0';
219 	}
220 
221 private:
222 	char* fOriginalPathEnd;
223 };
224 
225 
226 // #pragma mark - PackageWriterImpl (Inline Methods)
227 
228 
229 template<typename Type>
230 inline PackageWriterImpl::Attribute*
231 PackageWriterImpl::_AddAttribute(BHPKGAttributeID attributeID, Type value)
232 {
233 	AttributeValue attributeValue;
234 	attributeValue.SetTo(value);
235 	return _AddAttribute(attributeID, attributeValue);
236 }
237 
238 
239 // #pragma mark - PackageWriterImpl
240 
241 
242 PackageWriterImpl::PackageWriterImpl(BPackageWriterListener* listener)
243 	:
244 	inherited(listener),
245 	fListener(listener),
246 	fDataBuffer(NULL),
247 	fDataBufferSize(2 * B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB),
248 	fRootEntry(NULL),
249 	fRootAttribute(NULL),
250 	fTopAttribute(NULL)
251 {
252 }
253 
254 
255 PackageWriterImpl::~PackageWriterImpl()
256 {
257 	delete fRootAttribute;
258 
259 	delete fRootEntry;
260 
261 	free(fDataBuffer);
262 }
263 
264 
265 status_t
266 PackageWriterImpl::Init(const char* fileName)
267 {
268 	try {
269 		return _Init(fileName);
270 	} catch (status_t error) {
271 		return error;
272 	} catch (std::bad_alloc) {
273 		fListener->PrintError("Out of memory!\n");
274 		return B_NO_MEMORY;
275 	}
276 }
277 
278 
279 status_t
280 PackageWriterImpl::AddEntry(const char* fileName, int fd)
281 {
282 	try {
283 		// if it's ".PackageInfo", parse it
284 		if (strcmp(fileName, B_HPKG_PACKAGE_INFO_FILE_NAME) == 0) {
285 			struct ErrorListener : public BPackageInfo::ParseErrorListener {
286 				ErrorListener(BPackageWriterListener* _listener)
287 					: listener(_listener) {}
288 				virtual void OnError(const BString& msg, int line, int col) {
289 					listener->PrintError("Parse error in %s(%d:%d) -> %s\n",
290 						B_HPKG_PACKAGE_INFO_FILE_NAME, line, col, msg.String());
291 				}
292 				BPackageWriterListener* listener;
293 			} errorListener(fListener);
294 
295 			if (fd >= 0) {
296 				// a file descriptor is given -- read the config from there
297 				// stat the file to get the file size
298 				struct stat st;
299 				if (fstat(fd, &st) != 0)
300 					return errno;
301 
302 				BString packageInfoString;
303 				char* buffer = packageInfoString.LockBuffer(st.st_size);
304 				if (buffer == NULL)
305 					return B_NO_MEMORY;
306 
307 				ssize_t result = read_pos(fd, 0, buffer, st.st_size);
308 				if (result < 0) {
309 					packageInfoString.UnlockBuffer(0);
310 					return errno;
311 				}
312 
313 				buffer[st.st_size] = '\0';
314 				packageInfoString.UnlockBuffer(st.st_size);
315 
316 				result = fPackageInfo.ReadFromConfigString(packageInfoString,
317 					&errorListener);
318 				if (result != B_OK)
319 					return result;
320 			} else {
321 				// use the file name
322 				BEntry packageInfoEntry(fileName);
323 				status_t result = fPackageInfo.ReadFromConfigFile(
324 					packageInfoEntry, &errorListener);
325 				if (result != B_OK
326 					|| (result = fPackageInfo.InitCheck()) != B_OK) {
327 					return result;
328 				}
329 			}
330 		}
331 
332 		return _RegisterEntry(fileName, fd);
333 	} catch (status_t error) {
334 		return error;
335 	} catch (std::bad_alloc) {
336 		fListener->PrintError("Out of memory!\n");
337 		return B_NO_MEMORY;
338 	}
339 }
340 
341 
342 status_t
343 PackageWriterImpl::Finish()
344 {
345 	try {
346 		if (fPackageInfo.InitCheck() != B_OK) {
347 			fListener->PrintError("No package-info file found (%s)!\n",
348 				B_HPKG_PACKAGE_INFO_FILE_NAME);
349 			return B_BAD_DATA;
350 		}
351 
352 		RegisterPackageInfo(PackageAttributes(), fPackageInfo);
353 
354 		status_t result = _CheckLicenses();
355 		if (result != B_OK)
356 			return result;
357 
358 		return _Finish();
359 	} catch (status_t error) {
360 		return error;
361 	} catch (std::bad_alloc) {
362 		fListener->PrintError("Out of memory!\n");
363 		return B_NO_MEMORY;
364 	}
365 }
366 
367 
368 status_t
369 PackageWriterImpl::_Init(const char* fileName)
370 {
371 	status_t result = inherited::Init(fileName, "package");
372 	if (result != B_OK)
373 		return result;
374 
375 	// allocate data buffer
376 	fDataBuffer = malloc(fDataBufferSize);
377 	if (fDataBuffer == NULL)
378 		throw std::bad_alloc();
379 
380 	if (fStringCache.Init() != B_OK)
381 		throw std::bad_alloc();
382 
383 	// create entry list
384 	fRootEntry = new Entry(NULL, 0, -1, true);
385 
386 	fRootAttribute = new Attribute();
387 
388 	fHeapOffset = fHeapEnd = sizeof(hpkg_header);
389 	fTopAttribute = fRootAttribute;
390 
391 	return B_OK;
392 }
393 
394 
395 status_t
396 PackageWriterImpl::_CheckLicenses()
397 {
398 	BPath systemLicensePath;
399 	status_t result
400 #ifdef HAIKU_TARGET_PLATFORM_HAIKU
401 		= find_directory(B_SYSTEM_DATA_DIRECTORY, &systemLicensePath);
402 #else
403 		= systemLicensePath.SetTo(HAIKU_BUILD_SYSTEM_DATA_DIRECTORY);
404 #endif
405 	if (result != B_OK) {
406 		fListener->PrintError("unable to find system data path!\n");
407 		return result;
408 	}
409 	if ((result = systemLicensePath.Append("licenses")) != B_OK) {
410 		fListener->PrintError("unable to append to system data path!\n");
411 		return result;
412 	}
413 
414 	BDirectory systemLicenseDir(systemLicensePath.Path());
415 	BDirectory packageLicenseDir("./data/licenses");
416 
417 	const BObjectList<BString>& licenseList = fPackageInfo.LicenseList();
418 	for (int i = 0; i < licenseList.CountItems(); ++i) {
419 		const BString& licenseName = *licenseList.ItemAt(i);
420 		if (licenseName == kPublicDomainLicenseName)
421 			continue;
422 
423 		BEntry license;
424 		if (systemLicenseDir.FindEntry(licenseName.String(), &license) == B_OK)
425 			continue;
426 
427 		// license is not a system license, so it must be contained in package
428 		if (packageLicenseDir.FindEntry(licenseName.String(),
429 				&license) != B_OK) {
430 			fListener->PrintError("License '%s' isn't contained in package!\n",
431 				licenseName.String());
432 			return B_BAD_DATA;
433 		}
434 	}
435 
436 	return B_OK;
437 }
438 
439 
440 status_t
441 PackageWriterImpl::_Finish()
442 {
443 	// write entries
444 	for (EntryList::ConstIterator it = fRootEntry->ChildIterator();
445 			Entry* entry = it.Next();) {
446 		char pathBuffer[B_PATH_NAME_LENGTH];
447 		pathBuffer[0] = '\0';
448 		_AddEntry(AT_FDCWD, entry, entry->Name(), pathBuffer);
449 	}
450 
451 	off_t heapSize = fHeapEnd - sizeof(hpkg_header);
452 
453 	hpkg_header header;
454 
455 	// write the TOC and package attributes
456 	_WriteTOC(header);
457 	_WritePackageAttributes(header);
458 
459 	off_t totalSize = fHeapEnd;
460 
461 	fListener->OnPackageSizeInfo(sizeof(hpkg_header), heapSize,
462 		B_BENDIAN_TO_HOST_INT64(header.toc_length_compressed),
463 		B_BENDIAN_TO_HOST_INT32(header.attributes_length_compressed),
464 		totalSize);
465 
466 	// prepare the header
467 
468 	// general
469 	header.magic = B_HOST_TO_BENDIAN_INT32(B_HPKG_MAGIC);
470 	header.header_size = B_HOST_TO_BENDIAN_INT16(
471 		(uint16)sizeof(hpkg_header));
472 	header.version = B_HOST_TO_BENDIAN_INT16(B_HPKG_VERSION);
473 	header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize);
474 
475 	// write the header
476 	WriteBuffer(&header, sizeof(hpkg_header), 0);
477 
478 	SetFinished(true);
479 	return B_OK;
480 }
481 
482 
483 status_t
484 PackageWriterImpl::_RegisterEntry(const char* fileName, int fd)
485 {
486 	if (*fileName == '\0') {
487 		fListener->PrintError("Invalid empty file name\n");
488 		return B_BAD_VALUE;
489 	}
490 
491 	// add all components of the path
492 	Entry* entry = fRootEntry;
493 	while (*fileName != 0) {
494 		const char* nextSlash = strchr(fileName, '/');
495 		// no slash, just add the file name
496 		if (nextSlash == NULL) {
497 			entry = _RegisterEntry(entry, fileName, strlen(fileName), fd,
498 				false);
499 			break;
500 		}
501 
502 		// find the start of the next component, skipping slashes
503 		const char* nextComponent = nextSlash + 1;
504 		while (*nextComponent == '/')
505 			nextComponent++;
506 
507 		bool lastComponent = *nextComponent != '\0';
508 
509 		if (nextSlash == fileName) {
510 			// the FS root
511 			entry = _RegisterEntry(entry, fileName, 1, lastComponent ? fd : -1,
512 				lastComponent);
513 		} else {
514 			entry = _RegisterEntry(entry, fileName, nextSlash - fileName,
515 				lastComponent ? fd : -1, lastComponent);
516 		}
517 
518 		fileName = nextComponent;
519 	}
520 
521 	return B_OK;
522 }
523 
524 
525 PackageWriterImpl::Entry*
526 PackageWriterImpl::_RegisterEntry(Entry* parent, const char* name,
527 	size_t nameLength, int fd, bool isImplicit)
528 {
529 	// check the component name -- don't allow "." or ".."
530 	if (name[0] == '.'
531 		&& (nameLength == 1 || (nameLength == 2 && name[1] == '.'))) {
532 		fListener->PrintError("Invalid file name: \".\" and \"..\" "
533 			"are not allowed as path components\n");
534 		throw status_t(B_BAD_VALUE);
535 	}
536 
537 	// the entry might already exist
538 	Entry* entry = parent->GetChild(name, nameLength);
539 	if (entry != NULL) {
540 		// If the entry was implicit and is no longer, we mark it non-implicit
541 		// and delete all of it's children.
542 		if (entry->IsImplicit() && !isImplicit) {
543 			entry->DeleteChildren();
544 			entry->SetImplicit(false);
545 			entry->SetFD(fd);
546 		}
547 	} else {
548 		// nope -- create it
549 		entry = Entry::Create(name, nameLength, fd, isImplicit);
550 		parent->AddChild(entry);
551 	}
552 
553 	return entry;
554 }
555 
556 
557 void
558 PackageWriterImpl::_WriteTOC(hpkg_header& header)
559 {
560 	// prepare the writer (zlib writer on top of a file writer)
561 	off_t startOffset = fHeapEnd;
562 	FDDataWriter realWriter(FD(), startOffset, fListener);
563 	ZlibDataWriter zlibWriter(&realWriter);
564 	SetDataWriter(&zlibWriter);
565 	zlibWriter.Init();
566 
567 	// write the sections
568 	uint64 uncompressedStringsSize;
569 	uint64 uncompressedMainSize;
570 	int32 cachedStringsWritten
571 		= _WriteTOCSections(uncompressedStringsSize, uncompressedMainSize);
572 
573 	// finish the writer
574 	zlibWriter.Finish();
575 	fHeapEnd = realWriter.Offset();
576 	SetDataWriter(NULL);
577 
578 	off_t endOffset = fHeapEnd;
579 
580 	fListener->OnTOCSizeInfo(uncompressedStringsSize, uncompressedMainSize,
581 		zlibWriter.BytesWritten());
582 
583 	// update the header
584 
585 	// TOC
586 	header.toc_compression = B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB);
587 	header.toc_length_compressed = B_HOST_TO_BENDIAN_INT64(
588 		endOffset - startOffset);
589 	header.toc_length_uncompressed = B_HOST_TO_BENDIAN_INT64(
590 		zlibWriter.BytesWritten());
591 
592 	// TOC subsections
593 	header.toc_strings_length = B_HOST_TO_BENDIAN_INT64(
594 		uncompressedStringsSize);
595 	header.toc_strings_count = B_HOST_TO_BENDIAN_INT64(cachedStringsWritten);
596 }
597 
598 
599 int32
600 PackageWriterImpl::_WriteTOCSections(uint64& _stringsSize, uint64& _mainSize)
601 {
602 	// write the cached strings
603 	uint64 cachedStringsOffset = DataWriter()->BytesWritten();
604 	int32 cachedStringsWritten = WriteCachedStrings(fStringCache, 2);
605 
606 	// write the main TOC section
607 	uint64 mainOffset = DataWriter()->BytesWritten();
608 	_WriteAttributeChildren(fRootAttribute);
609 
610 	_stringsSize = mainOffset - cachedStringsOffset;
611 	_mainSize = DataWriter()->BytesWritten() - mainOffset;
612 
613 	return cachedStringsWritten;
614 }
615 
616 
617 void
618 PackageWriterImpl::_WriteAttributeChildren(Attribute* attribute)
619 {
620 	DoublyLinkedList<Attribute>::Iterator it
621 		= attribute->children.GetIterator();
622 	while (Attribute* child = it.Next()) {
623 		// write tag
624 		uint8 encoding = child->value.ApplicableEncoding();
625 		WriteUnsignedLEB128(HPKG_ATTRIBUTE_TAG_COMPOSE(child->id,
626 			child->value.type, encoding, !child->children.IsEmpty()));
627 
628 		// write value
629 		WriteAttributeValue(child->value, encoding);
630 
631 		if (!child->children.IsEmpty())
632 			_WriteAttributeChildren(child);
633 	}
634 
635 	WriteUnsignedLEB128(0);
636 }
637 
638 
639 void
640 PackageWriterImpl::_WritePackageAttributes(hpkg_header& header)
641 {
642 	// write the package attributes (zlib writer on top of a file writer)
643 	off_t startOffset = fHeapEnd;
644 	FDDataWriter realWriter(FD(), startOffset, fListener);
645 	ZlibDataWriter zlibWriter(&realWriter);
646 	SetDataWriter(&zlibWriter);
647 	zlibWriter.Init();
648 
649 	// write cached strings and package attributes tree
650 	uint32 stringsLengthUncompressed;
651 	uint32 stringsCount = WritePackageAttributes(PackageAttributes(),
652 		stringsLengthUncompressed);
653 
654 	zlibWriter.Finish();
655 	fHeapEnd = realWriter.Offset();
656 	SetDataWriter(NULL);
657 
658 	off_t endOffset = fHeapEnd;
659 
660 	fListener->OnPackageAttributesSizeInfo(stringsCount,
661 		zlibWriter.BytesWritten());
662 
663 	// update the header
664 	header.attributes_compression
665 		= B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB);
666 	header.attributes_length_compressed
667 		= B_HOST_TO_BENDIAN_INT32(endOffset - startOffset);
668 	header.attributes_length_uncompressed
669 		= B_HOST_TO_BENDIAN_INT32(zlibWriter.BytesWritten());
670 	header.attributes_strings_count = B_HOST_TO_BENDIAN_INT32(stringsCount);
671 	header.attributes_strings_length
672 		= B_HOST_TO_BENDIAN_INT32(stringsLengthUncompressed);
673 }
674 
675 
676 void
677 PackageWriterImpl::_AddEntry(int dirFD, Entry* entry, const char* fileName,
678 	char* pathBuffer)
679 {
680 	bool isImplicitEntry = entry != NULL && entry->IsImplicit();
681 
682 	SubPathAdder pathAdder(fListener, pathBuffer, fileName);
683 	if (!isImplicitEntry)
684 		fListener->OnEntryAdded(pathBuffer);
685 
686 	// open the node
687 	int fd;
688 	FileDescriptorCloser fdCloser;
689 
690 	if (entry != NULL && entry->FD() >= 0) {
691 		// a file descriptor is already given -- use that
692 		fd = entry->FD();
693 	} else {
694 		fd = openat(dirFD, fileName,
695 			O_RDONLY | (isImplicitEntry ? 0 : O_NOTRAVERSE));
696 		if (fd < 0) {
697 			fListener->PrintError("Failed to open entry \"%s\": %s\n",
698 				pathBuffer, strerror(errno));
699 			throw status_t(errno);
700 		}
701 		fdCloser.SetTo(fd);
702 	}
703 
704 	// stat the node
705 	struct stat st;
706 	if (fstat(fd, &st) < 0) {
707 		fListener->PrintError("Failed to fstat() file \"%s\": %s\n", pathBuffer,
708 			strerror(errno));
709 		throw status_t(errno);
710 	}
711 
712 	// implicit entries must be directories
713 	if (isImplicitEntry && !S_ISDIR(st.st_mode)) {
714 		fListener->PrintError("Non-leaf path component \"%s\" is not a "
715 			"directory\n", pathBuffer);
716 		throw status_t(B_BAD_VALUE);
717 	}
718 
719 	// check/translate the node type
720 	uint8 fileType;
721 	uint32 defaultPermissions;
722 	if (S_ISREG(st.st_mode)) {
723 		fileType = B_HPKG_FILE_TYPE_FILE;
724 		defaultPermissions = B_HPKG_DEFAULT_FILE_PERMISSIONS;
725 	} else if (S_ISLNK(st.st_mode)) {
726 		fileType = B_HPKG_FILE_TYPE_SYMLINK;
727 		defaultPermissions = B_HPKG_DEFAULT_SYMLINK_PERMISSIONS;
728 	} else if (S_ISDIR(st.st_mode)) {
729 		fileType = B_HPKG_FILE_TYPE_DIRECTORY;
730 		defaultPermissions = B_HPKG_DEFAULT_DIRECTORY_PERMISSIONS;
731 	} else {
732 		// unsupported node type
733 		fListener->PrintError("Unsupported node type, entry: \"%s\"\n",
734 			pathBuffer);
735 		throw status_t(B_UNSUPPORTED);
736 	}
737 
738 	// add attribute entry
739 	Attribute* entryAttribute = _AddStringAttribute(
740 		B_HPKG_ATTRIBUTE_ID_DIRECTORY_ENTRY, fileName);
741 	Stacker<Attribute> entryAttributeStacker(fTopAttribute, entryAttribute);
742 
743 	// add stat data
744 	if (fileType != B_HPKG_DEFAULT_FILE_TYPE)
745 		_AddAttribute(B_HPKG_ATTRIBUTE_ID_FILE_TYPE, fileType);
746 	if (defaultPermissions != uint32(st.st_mode & ALLPERMS)) {
747 		_AddAttribute(B_HPKG_ATTRIBUTE_ID_FILE_PERMISSIONS,
748 			uint32(st.st_mode & ALLPERMS));
749 	}
750 	_AddAttribute(B_HPKG_ATTRIBUTE_ID_FILE_ATIME, uint32(st.st_atime));
751 	_AddAttribute(B_HPKG_ATTRIBUTE_ID_FILE_MTIME, uint32(st.st_mtime));
752 #ifdef __HAIKU__
753 	_AddAttribute(B_HPKG_ATTRIBUTE_ID_FILE_CRTIME, uint32(st.st_crtime));
754 #else
755 	_AddAttribute(B_HPKG_ATTRIBUTE_ID_FILE_CRTIME, uint32(st.st_mtime));
756 #endif
757 	// TODO: File user/group!
758 
759 	// add file data/symlink path
760 	if (S_ISREG(st.st_mode)) {
761 		// regular file -- add data
762 		if (st.st_size > 0) {
763 			BFDDataReader dataReader(fd);
764 			status_t error = _AddData(dataReader, st.st_size);
765 			if (error != B_OK)
766 				throw status_t(error);
767 		}
768 	} else if (S_ISLNK(st.st_mode)) {
769 		// symlink -- add link address
770 		char path[B_PATH_NAME_LENGTH + 1];
771 		ssize_t bytesRead = readlinkat(dirFD, fileName, path,
772 			B_PATH_NAME_LENGTH);
773 		if (bytesRead < 0) {
774 			fListener->PrintError("Failed to read symlink \"%s\": %s\n",
775 				pathBuffer, strerror(errno));
776 			throw status_t(errno);
777 		}
778 
779 		path[bytesRead] = '\0';
780 		_AddStringAttribute(B_HPKG_ATTRIBUTE_ID_SYMLINK_PATH, path);
781 	}
782 
783 	// add attributes
784 	if (DIR* attrDir = fs_fopen_attr_dir(fd)) {
785 		CObjectDeleter<DIR, int> attrDirCloser(attrDir, fs_close_attr_dir);
786 
787 		while (dirent* entry = fs_read_attr_dir(attrDir)) {
788 			attr_info attrInfo;
789 			if (fs_stat_attr(fd, entry->d_name, &attrInfo) < 0) {
790 				fListener->PrintError(
791 					"Failed to stat attribute \"%s\" of file \"%s\": %s\n",
792 					entry->d_name, pathBuffer, strerror(errno));
793 				throw status_t(errno);
794 			}
795 
796 			// create attribute entry
797 			Attribute* attributeAttribute = _AddStringAttribute(
798 				B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE, entry->d_name);
799 			Stacker<Attribute> attributeAttributeStacker(fTopAttribute,
800 				attributeAttribute);
801 
802 			// add type
803 			_AddAttribute(B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE_TYPE,
804 				(uint32)attrInfo.type);
805 
806 			// add data
807 			BAttributeDataReader dataReader(fd, entry->d_name, attrInfo.type);
808 			status_t error = _AddData(dataReader, attrInfo.size);
809 			if (error != B_OK)
810 				throw status_t(error);
811 		}
812 	}
813 
814 	if (S_ISDIR(st.st_mode)) {
815 		// directory -- recursively add children
816 		if (isImplicitEntry) {
817 			// this is an implicit entry -- just add it's children
818 			for (EntryList::ConstIterator it = entry->ChildIterator();
819 					Entry* child = it.Next();) {
820 				_AddEntry(fd, child, child->Name(), pathBuffer);
821 			}
822 		} else {
823 			// we need to clone the directory FD for fdopendir()
824 			int clonedFD = dup(fd);
825 			if (clonedFD < 0) {
826 				fListener->PrintError(
827 					"Failed to dup() directory FD: %s\n", strerror(errno));
828 				throw status_t(errno);
829 			}
830 
831 			DIR* dir = fdopendir(clonedFD);
832 			if (dir == NULL) {
833 				fListener->PrintError(
834 					"Failed to open directory \"%s\": %s\n", pathBuffer,
835 					strerror(errno));
836 				close(clonedFD);
837 				throw status_t(errno);
838 			}
839 			CObjectDeleter<DIR, int> dirCloser(dir, closedir);
840 
841 			while (dirent* entry = readdir(dir)) {
842 				// skip "." and ".."
843 				if (strcmp(entry->d_name, ".") == 0
844 					|| strcmp(entry->d_name, "..") == 0) {
845 					continue;
846 				}
847 
848 				_AddEntry(fd, NULL, entry->d_name, pathBuffer);
849 			}
850 		}
851 	}
852 }
853 
854 
855 PackageWriterImpl::Attribute*
856 PackageWriterImpl::_AddAttribute(BHPKGAttributeID id,
857 	const AttributeValue& value)
858 {
859 	Attribute* attribute = new Attribute(id);
860 
861 	attribute->value = value;
862 	fTopAttribute->AddChild(attribute);
863 
864 	return attribute;
865 }
866 
867 
868 PackageWriterImpl::Attribute*
869 PackageWriterImpl::_AddStringAttribute(BHPKGAttributeID attributeID,
870 	const char* value)
871 {
872 	AttributeValue attributeValue;
873 	attributeValue.SetTo(fStringCache.Get(value));
874 	return _AddAttribute(attributeID, attributeValue);
875 }
876 
877 
878 PackageWriterImpl::Attribute*
879 PackageWriterImpl::_AddDataAttribute(BHPKGAttributeID attributeID,
880 	uint64 dataSize, uint64 dataOffset)
881 {
882 	AttributeValue attributeValue;
883 	attributeValue.SetToData(dataSize, dataOffset);
884 	return _AddAttribute(attributeID, attributeValue);
885 }
886 
887 
888 PackageWriterImpl::Attribute*
889 PackageWriterImpl::_AddDataAttribute(BHPKGAttributeID attributeID,
890 	uint64 dataSize, const uint8* data)
891 {
892 	AttributeValue attributeValue;
893 	attributeValue.SetToData(dataSize, data);
894 	return _AddAttribute(attributeID, attributeValue);
895 }
896 
897 
898 status_t
899 PackageWriterImpl::_AddData(BDataReader& dataReader, off_t size)
900 {
901 	// add short data inline
902 	if (size <= B_HPKG_MAX_INLINE_DATA_SIZE) {
903 		uint8 buffer[B_HPKG_MAX_INLINE_DATA_SIZE];
904 		status_t error = dataReader.ReadData(0, buffer, size);
905 		if (error != B_OK) {
906 			fListener->PrintError("Failed to read data: %s\n", strerror(error));
907 			return error;
908 		}
909 
910 		_AddDataAttribute(B_HPKG_ATTRIBUTE_ID_DATA, size, buffer);
911 		return B_OK;
912 	}
913 
914 	// longer data -- try to compress
915 	uint64 dataOffset = fHeapEnd;
916 
917 	uint64 compression = B_HPKG_COMPRESSION_NONE;
918 	uint64 compressedSize;
919 
920 	status_t error = _WriteZlibCompressedData(dataReader, size, dataOffset,
921 		compressedSize);
922 	if (error == B_OK) {
923 		compression = B_HPKG_COMPRESSION_ZLIB;
924 	} else {
925 		error = _WriteUncompressedData(dataReader, size, dataOffset);
926 		compressedSize = size;
927 	}
928 	if (error != B_OK)
929 		return error;
930 
931 	fHeapEnd = dataOffset + compressedSize;
932 
933 	// add data attribute
934 	Attribute* dataAttribute = _AddDataAttribute(B_HPKG_ATTRIBUTE_ID_DATA,
935 		compressedSize, dataOffset - fHeapOffset);
936 	Stacker<Attribute> attributeAttributeStacker(fTopAttribute, dataAttribute);
937 
938 	// if compressed, add compression attributes
939 	if (compression != B_HPKG_COMPRESSION_NONE) {
940 		_AddAttribute(B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION, compression);
941 		_AddAttribute(B_HPKG_ATTRIBUTE_ID_DATA_SIZE, (uint64)size);
942 			// uncompressed size
943 	}
944 
945 	return B_OK;
946 }
947 
948 
949 status_t
950 PackageWriterImpl::_WriteUncompressedData(BDataReader& dataReader, off_t size,
951 	uint64 writeOffset)
952 {
953 	// copy the data to the heap
954 	off_t readOffset = 0;
955 	off_t remainingSize = size;
956 	while (remainingSize > 0) {
957 		// read data
958 		size_t toCopy = std::min(remainingSize, (off_t)fDataBufferSize);
959 		status_t error = dataReader.ReadData(readOffset, fDataBuffer, toCopy);
960 		if (error != B_OK) {
961 			fListener->PrintError("Failed to read data: %s\n", strerror(error));
962 			return error;
963 		}
964 
965 		// write to heap
966 		ssize_t bytesWritten = pwrite(FD(), fDataBuffer, toCopy, writeOffset);
967 		if (bytesWritten < 0) {
968 			fListener->PrintError("Failed to write data: %s\n",
969 				strerror(errno));
970 			return errno;
971 		}
972 		if ((size_t)bytesWritten != toCopy) {
973 			fListener->PrintError("Failed to write all data\n");
974 			return B_ERROR;
975 		}
976 
977 		remainingSize -= toCopy;
978 		readOffset += toCopy;
979 		writeOffset += toCopy;
980 	}
981 
982 	return B_OK;
983 }
984 
985 
986 status_t
987 PackageWriterImpl::_WriteZlibCompressedData(BDataReader& dataReader, off_t size,
988 	uint64 writeOffset, uint64& _compressedSize)
989 {
990 	// Use zlib compression only for data large enough.
991 	if (size < (off_t)kZlibCompressionSizeThreshold)
992 		return B_BAD_VALUE;
993 
994 	// fDataBuffer is 2 * B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, so split it into
995 	// two halves we can use for reading and compressing
996 	const size_t chunkSize = B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB;
997 	uint8* inputBuffer = (uint8*)fDataBuffer;
998 	uint8* outputBuffer = (uint8*)fDataBuffer + chunkSize;
999 
1000 	// account for the offset table
1001 	uint64 chunkCount = (size + (chunkSize - 1)) / chunkSize;
1002 	off_t offsetTableOffset = writeOffset;
1003 	uint64* offsetTable = NULL;
1004 	if (chunkCount > 1) {
1005 		offsetTable = new uint64[chunkCount - 1];
1006 		writeOffset = offsetTableOffset + (chunkCount - 1) * sizeof(uint64);
1007 	}
1008 	ArrayDeleter<uint64> offsetTableDeleter(offsetTable);
1009 
1010 	const uint64 dataOffset = writeOffset;
1011 	const uint64 dataEndLimit = offsetTableOffset + size;
1012 
1013 	// read the data, compress them and write them to the heap
1014 	off_t readOffset = 0;
1015 	off_t remainingSize = size;
1016 	uint64 chunkIndex = 0;
1017 	while (remainingSize > 0) {
1018 		// read data
1019 		size_t toCopy = std::min(remainingSize, (off_t)chunkSize);
1020 		status_t error = dataReader.ReadData(readOffset, inputBuffer, toCopy);
1021 		if (error != B_OK) {
1022 			fListener->PrintError("Failed to read data: %s\n", strerror(error));
1023 			return error;
1024 		}
1025 
1026 		// compress
1027 		size_t compressedSize;
1028 		error = ZlibCompressor::CompressSingleBuffer(inputBuffer, toCopy,
1029 			outputBuffer, toCopy, compressedSize);
1030 
1031 		const void* writeBuffer;
1032 		size_t bytesToWrite;
1033 		if (error == B_OK) {
1034 			writeBuffer = outputBuffer;
1035 			bytesToWrite = compressedSize;
1036 		} else {
1037 			if (error != B_BUFFER_OVERFLOW)
1038 				return error;
1039 			writeBuffer = inputBuffer;
1040 			bytesToWrite = toCopy;
1041 		}
1042 
1043 		// check the total compressed data size
1044 		if (writeOffset + bytesToWrite >= dataEndLimit)
1045 			return B_BUFFER_OVERFLOW;
1046 
1047 		if (chunkIndex > 0)
1048 			offsetTable[chunkIndex - 1] = writeOffset - dataOffset;
1049 
1050 		// write to heap
1051 		ssize_t bytesWritten = pwrite(FD(), writeBuffer, bytesToWrite,
1052 			writeOffset);
1053 		if (bytesWritten < 0) {
1054 			fListener->PrintError("Failed to write data: %s\n",
1055 				strerror(errno));
1056 			return errno;
1057 		}
1058 		if ((size_t)bytesWritten != bytesToWrite) {
1059 			fListener->PrintError("Failed to write all data\n");
1060 			return B_ERROR;
1061 		}
1062 
1063 		remainingSize -= toCopy;
1064 		readOffset += toCopy;
1065 		writeOffset += bytesToWrite;
1066 		chunkIndex++;
1067 	}
1068 
1069 	// write the offset table
1070 	if (chunkCount > 1) {
1071 		size_t bytesToWrite = (chunkCount - 1) * sizeof(uint64);
1072 		ssize_t bytesWritten = pwrite(FD(), offsetTable, bytesToWrite,
1073 			offsetTableOffset);
1074 		if (bytesWritten < 0) {
1075 			fListener->PrintError("Failed to write data: %s\n",
1076 				strerror(errno));
1077 			return errno;
1078 		}
1079 		if ((size_t)bytesWritten != bytesToWrite) {
1080 			fListener->PrintError("Failed to write all data\n");
1081 			return B_ERROR;
1082 		}
1083 	}
1084 
1085 	_compressedSize = writeOffset - offsetTableOffset;
1086 	return B_OK;
1087 }
1088 
1089 
1090 }	// namespace BPrivate
1091 
1092 }	// namespace BHPKG
1093 
1094 }	// namespace BPackageKit
1095