1 /* 2 * Copyright 2017-2024, Andrew Lindesay <apl@lindesay.co.nz>. 3 * All rights reserved. Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "ServerPkgDataUpdateProcess.h" 8 9 #include <stdio.h> 10 #include <sys/stat.h> 11 #include <time.h> 12 13 #include <AutoDeleter.h> 14 #include <AutoLocker.h> 15 #include <Catalog.h> 16 #include <FileIO.h> 17 #include <StopWatch.h> 18 #include <Url.h> 19 20 #include "DumpExportPkg.h" 21 #include "DumpExportPkgCategory.h" 22 #include "DumpExportPkgJsonListener.h" 23 #include "DumpExportPkgScreenshot.h" 24 #include "DumpExportPkgVersion.h" 25 #include "HaikuDepotConstants.h" 26 #include "Logger.h" 27 #include "PackageUtils.h" 28 #include "ServerSettings.h" 29 #include "StorageUtils.h" 30 31 32 #undef B_TRANSLATION_CONTEXT 33 #define B_TRANSLATION_CONTEXT "ServerPkgDataUpdateProcess" 34 35 36 /*! This package listener (not at the JSON level) is feeding in the 37 packages as they are parsed and processing them. 38 */ 39 40 class PackageFillingPkgListener : public DumpExportPkgListener { 41 public: 42 PackageFillingPkgListener(Model *model, 43 BString& depotName, Stoppable* stoppable); 44 virtual ~PackageFillingPkgListener(); 45 46 virtual bool ConsumePackage(const PackageInfoRef& package, 47 DumpExportPkg* pkg); 48 virtual bool Handle(DumpExportPkg* item); 49 virtual void Complete(); 50 51 uint32 Count(); 52 53 private: 54 static ScreenshotInfoRef _CreateScreenshot(DumpExportPkgScreenshot* screenshot); 55 56 private: 57 BString fDepotName; 58 Model* fModel; 59 std::vector<CategoryRef> 60 fCategories; 61 Stoppable* fStoppable; 62 uint32 fCount; 63 bool fDebugEnabled; 64 }; 65 66 67 PackageFillingPkgListener::PackageFillingPkgListener(Model* model, 68 BString& depotName, Stoppable* stoppable) 69 : 70 fDepotName(depotName), 71 fModel(model), 72 fStoppable(stoppable), 73 fCount(0), 74 fDebugEnabled(Logger::IsDebugEnabled()) 75 { 76 } 77 78 79 PackageFillingPkgListener::~PackageFillingPkgListener() 80 { 81 } 82 83 84 bool 85 PackageFillingPkgListener::ConsumePackage(const PackageInfoRef& package, 86 DumpExportPkg* pkg) 87 { 88 int32 i; 89 90 // Collects all of the changes here into one set of notifications to 91 // the package's listeners. This way the quantity of BMessages 92 // communicated back to listeners is considerably reduced. See stop 93 // invocation later in this method. 94 95 package->StartCollatingChanges(); 96 97 PackageClassificationInfoRef packageClassificationInfo(new PackageClassificationInfo(), true); 98 99 PackageLocalizedTextRef localizedText = PackageUtils::NewLocalizedText(package); 100 PackageLocalInfoRef localInfo = PackageUtils::NewLocalInfo(package); 101 102 localizedText->SetHasChangelog(pkg->HasChangelog()); 103 104 if (0 != pkg->CountPkgVersions()) { 105 106 // this makes the assumption that the only version will be the 107 // latest one. 108 109 DumpExportPkgVersion* pkgVersion = pkg->PkgVersionsItemAt(0); 110 111 if (!pkgVersion->TitleIsNull()) 112 localizedText->SetTitle(*(pkgVersion->Title())); 113 114 if (!pkgVersion->SummaryIsNull()) 115 localizedText->SetSummary(*(pkgVersion->Summary())); 116 117 if (!pkgVersion->DescriptionIsNull()) 118 localizedText->SetDescription(*(pkgVersion->Description())); 119 120 if (!pkgVersion->PayloadLengthIsNull()) 121 localInfo->SetSize(static_cast<off_t>(pkgVersion->PayloadLength())); 122 123 if (!pkgVersion->CreateTimestampIsNull()) 124 package->SetVersionCreateTimestamp(pkgVersion->CreateTimestamp()); 125 } 126 127 int32 countPkgCategories = pkg->CountPkgCategories(); 128 129 for (i = 0; i < countPkgCategories; i++) { 130 BString* categoryCode = pkg->PkgCategoriesItemAt(i)->Code(); 131 CategoryRef category = fModel->CategoryByCode(*categoryCode); 132 133 if (!category.IsSet()) { 134 HDERROR("unable to find the category for [%s]", 135 categoryCode->String()); 136 } else 137 packageClassificationInfo->AddCategory(category); 138 } 139 140 if (!pkg->DerivedRatingIsNull()) { 141 UserRatingInfoRef userRatingInfo(new UserRatingInfo(), true); 142 UserRatingSummaryRef userRatingSummary(new UserRatingSummary(), true); 143 // TODO; unify the naming here! 144 userRatingSummary->SetAverageRating(pkg->DerivedRating()); 145 userRatingSummary->SetRatingCount(pkg->DerivedRatingSampleSize()); 146 userRatingInfo->SetSummary(userRatingSummary); 147 package->SetUserRatingInfo(userRatingInfo); 148 } 149 150 if (!pkg->ProminenceOrderingIsNull()) 151 packageClassificationInfo->SetProminence(static_cast<uint32>(pkg->ProminenceOrdering())); 152 153 if (!pkg->IsNativeDesktopIsNull()) 154 packageClassificationInfo->SetIsNativeDesktop(pkg->IsNativeDesktop()); 155 156 int32 countPkgScreenshots = pkg->CountPkgScreenshots(); 157 PackageScreenshotInfoRef screenshotInfo(new PackageScreenshotInfo(), true); 158 159 for (i = 0; i < countPkgScreenshots; i++) { 160 DumpExportPkgScreenshot* screenshot = pkg->PkgScreenshotsItemAt(i); 161 screenshotInfo->AddScreenshot(_CreateScreenshot(screenshot)); 162 } 163 164 package->SetScreenshotInfo(screenshotInfo); 165 package->SetLocalizedText(localizedText); 166 package->SetLocalInfo(localInfo); 167 168 HDTRACE("did populate data for [%s] (%s)", pkg->Name()->String(), 169 fDepotName.String()); 170 171 fCount++; 172 173 package->SetPackageClassificationInfo(packageClassificationInfo); 174 175 package->EndCollatingChanges(); 176 177 return !fStoppable->WasStopped(); 178 } 179 180 181 /*static*/ ScreenshotInfoRef 182 PackageFillingPkgListener::_CreateScreenshot(DumpExportPkgScreenshot* screenshot) 183 { 184 return ScreenshotInfoRef( 185 new ScreenshotInfo(*(screenshot->Code()), static_cast<int32>(screenshot->Width()), 186 static_cast<int32>(screenshot->Height()), static_cast<int32>(screenshot->Length())), 187 true); 188 } 189 190 191 uint32 192 PackageFillingPkgListener::Count() 193 { 194 return fCount; 195 } 196 197 198 bool 199 PackageFillingPkgListener::Handle(DumpExportPkg* pkg) 200 { 201 AutoLocker<BLocker> locker(fModel->Lock()); 202 DepotInfoRef depot = fModel->DepotForName(fDepotName); 203 204 if (depot.Get() != NULL) { 205 const BString packageName = *(pkg->Name()); 206 PackageInfoRef package = depot->PackageByName(packageName); 207 if (package.Get() != NULL) 208 ConsumePackage(package, pkg); 209 else { 210 HDINFO("[PackageFillingPkgListener] unable to find the pkg [%s]", 211 packageName.String()); 212 } 213 } else { 214 HDINFO("[PackageFillingPkgListener] unable to find the depot [%s]", 215 fDepotName.String()); 216 } 217 218 return !fStoppable->WasStopped(); 219 } 220 221 222 void 223 PackageFillingPkgListener::Complete() 224 { 225 } 226 227 228 ServerPkgDataUpdateProcess::ServerPkgDataUpdateProcess( 229 BString depotName, 230 Model *model, 231 uint32 serverProcessOptions) 232 : 233 AbstractSingleFileServerProcess(serverProcessOptions), 234 fModel(model), 235 fDepotName(depotName) 236 { 237 fName.SetToFormat("ServerPkgDataUpdateProcess<%s>", depotName.String()); 238 fDescription.SetTo( 239 B_TRANSLATE("Synchronizing package data for repository " 240 "'%REPO_NAME%'")); 241 fDescription.ReplaceAll("%REPO_NAME%", depotName.String()); 242 } 243 244 245 ServerPkgDataUpdateProcess::~ServerPkgDataUpdateProcess() 246 { 247 } 248 249 250 const char* 251 ServerPkgDataUpdateProcess::Name() const 252 { 253 return fName.String(); 254 } 255 256 257 const char* 258 ServerPkgDataUpdateProcess::Description() const 259 { 260 return fDescription.String(); 261 } 262 263 264 BString 265 ServerPkgDataUpdateProcess::UrlPathComponent() 266 { 267 BString urlPath; 268 urlPath.SetToFormat("/__pkg/all-%s-%s.json.gz", _DeriveWebAppRepositorySourceCode().String(), 269 fModel->PreferredLanguage()->ID()); 270 return urlPath; 271 } 272 273 274 status_t 275 ServerPkgDataUpdateProcess::GetLocalPath(BPath& path) const 276 { 277 BString webAppRepositorySourceCode = _DeriveWebAppRepositorySourceCode(); 278 279 if (!webAppRepositorySourceCode.IsEmpty()) { 280 AutoLocker<BLocker> locker(fModel->Lock()); 281 return StorageUtils::DumpExportPkgDataPath(path, webAppRepositorySourceCode, 282 fModel->PreferredLanguage()); 283 } 284 285 return B_ERROR; 286 } 287 288 289 status_t 290 ServerPkgDataUpdateProcess::ProcessLocalData() 291 { 292 BStopWatch watch("ServerPkgDataUpdateProcess::ProcessLocalData", true); 293 294 PackageFillingPkgListener* itemListener = 295 new PackageFillingPkgListener(fModel, fDepotName, this); 296 ObjectDeleter<PackageFillingPkgListener> 297 itemListenerDeleter(itemListener); 298 299 BulkContainerDumpExportPkgJsonListener* listener = 300 new BulkContainerDumpExportPkgJsonListener(itemListener); 301 ObjectDeleter<BulkContainerDumpExportPkgJsonListener> 302 listenerDeleter(listener); 303 304 BPath localPath; 305 status_t result = GetLocalPath(localPath); 306 307 if (result != B_OK) 308 return result; 309 310 result = ParseJsonFromFileWithListener(listener, localPath); 311 312 if (B_OK != result) 313 return result; 314 315 if (Logger::IsInfoEnabled()) { 316 double secs = watch.ElapsedTime() / 1000000.0; 317 HDINFO("[%s] did process %" B_PRIi32 " packages' data " 318 "in (%6.3g secs)", Name(), itemListener->Count(), secs); 319 } 320 321 return listener->ErrorStatus(); 322 } 323 324 325 status_t 326 ServerPkgDataUpdateProcess::GetStandardMetaDataPath(BPath& path) const 327 { 328 return GetLocalPath(path); 329 } 330 331 332 void 333 ServerPkgDataUpdateProcess::GetStandardMetaDataJsonPath( 334 BString& jsonPath) const 335 { 336 jsonPath.SetTo("$.info"); 337 } 338 339 340 BString 341 ServerPkgDataUpdateProcess::_DeriveWebAppRepositorySourceCode() const 342 { 343 const DepotInfo* depot = fModel->DepotForName(fDepotName); 344 345 if (depot == NULL) { 346 return BString(); 347 } 348 349 return depot->WebAppRepositorySourceCode(); 350 } 351 352 353 status_t 354 ServerPkgDataUpdateProcess::RunInternal() 355 { 356 if (_DeriveWebAppRepositorySourceCode().IsEmpty()) { 357 HDINFO("[%s] am not updating data for depot [%s] as there is no" 358 " web app repository source code available", 359 Name(), fDepotName.String()); 360 return B_OK; 361 } 362 363 return AbstractSingleFileServerProcess::RunInternal(); 364 } 365