1 /* 2 * Copyright 2017-2020, Andrew Lindesay <apl@lindesay.co.nz>. 3 * All rights reserved. Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "ServerPkgDataUpdateProcess.h" 8 9 #include <stdio.h> 10 #include <sys/stat.h> 11 #include <time.h> 12 13 #include <AutoDeleter.h> 14 #include <AutoLocker.h> 15 #include <Catalog.h> 16 #include <FileIO.h> 17 #include <support/StopWatch.h> 18 #include <Url.h> 19 20 #include "Logger.h" 21 #include "ServerSettings.h" 22 #include "StorageUtils.h" 23 #include "DumpExportPkg.h" 24 #include "DumpExportPkgCategory.h" 25 #include "DumpExportPkgJsonListener.h" 26 #include "DumpExportPkgScreenshot.h" 27 #include "DumpExportPkgVersion.h" 28 #include "HaikuDepotConstants.h" 29 30 31 #undef B_TRANSLATION_CONTEXT 32 #define B_TRANSLATION_CONTEXT "ServerPkgDataUpdateProcess" 33 34 35 /*! This package listener (not at the JSON level) is feeding in the 36 packages as they are parsed and processing them. 37 */ 38 39 class PackageFillingPkgListener : public DumpExportPkgListener { 40 public: 41 PackageFillingPkgListener(Model *model, 42 BString& depotName, Stoppable* stoppable); 43 virtual ~PackageFillingPkgListener(); 44 45 virtual bool ConsumePackage(const PackageInfoRef& package, 46 DumpExportPkg* pkg); 47 virtual bool Handle(DumpExportPkg* item); 48 virtual void Complete(); 49 50 uint32 Count(); 51 52 private: 53 int32 IndexOfPackageByName(const BString& name) const; 54 int32 IndexOfCategoryByName( 55 const BString& name) const; 56 int32 IndexOfCategoryByCode( 57 const BString& code) const; 58 59 BString fDepotName; 60 Model* fModel; 61 CategoryList fCategories; 62 Stoppable* fStoppable; 63 uint32 fCount; 64 bool fDebugEnabled; 65 }; 66 67 68 PackageFillingPkgListener::PackageFillingPkgListener(Model* model, 69 BString& depotName, Stoppable* stoppable) 70 : 71 fDepotName(depotName), 72 fModel(model), 73 fStoppable(stoppable), 74 fCount(0), 75 fDebugEnabled(Logger::IsDebugEnabled()) 76 { 77 fCategories = model->Categories(); 78 } 79 80 81 PackageFillingPkgListener::~PackageFillingPkgListener() 82 { 83 } 84 85 86 // TODO; performance could be improved by not needing the linear search 87 88 inline int32 89 PackageFillingPkgListener::IndexOfCategoryByCode( 90 const BString& code) const 91 { 92 int32 i; 93 int32 categoryCount = fCategories.CountItems(); 94 95 for (i = 0; i < categoryCount; i++) { 96 const CategoryRef categoryRef = fCategories.ItemAtFast(i); 97 98 if (categoryRef->Code() == code) 99 return i; 100 } 101 102 return -1; 103 } 104 105 106 bool 107 PackageFillingPkgListener::ConsumePackage(const PackageInfoRef& package, 108 DumpExportPkg* pkg) 109 { 110 int32 i; 111 112 // Collects all of the changes here into one set of notifications to 113 // the package's listeners. This way the quantity of BMessages 114 // communicated back to listeners is considerably reduced. See stop 115 // invocation later in this method. 116 117 package->StartCollatingChanges(); 118 119 if (0 != pkg->CountPkgVersions()) { 120 121 // this makes the assumption that the only version will be the 122 // latest one. 123 124 DumpExportPkgVersion* pkgVersion = pkg->PkgVersionsItemAt(0); 125 126 if (!pkgVersion->TitleIsNull()) 127 package->SetTitle(*(pkgVersion->Title())); 128 129 if (!pkgVersion->SummaryIsNull()) 130 package->SetShortDescription(*(pkgVersion->Summary())); 131 132 if (!pkgVersion->DescriptionIsNull()) 133 package->SetFullDescription(*(pkgVersion->Description())); 134 135 if (!pkgVersion->PayloadLengthIsNull()) 136 package->SetSize(pkgVersion->PayloadLength()); 137 } 138 139 int32 countPkgCategories = pkg->CountPkgCategories(); 140 141 for (i = 0; i < countPkgCategories; i++) { 142 BString* categoryCode = pkg->PkgCategoriesItemAt(i)->Code(); 143 int categoryIndex = IndexOfCategoryByCode(*(categoryCode)); 144 145 if (categoryIndex == -1) { 146 printf("unable to find the category for [%s]\n", 147 categoryCode->String()); 148 } else { 149 package->AddCategory( 150 fCategories.ItemAtFast(categoryIndex)); 151 } 152 } 153 154 RatingSummary summary; 155 summary.averageRating = RATING_MISSING; 156 157 if (!pkg->DerivedRatingIsNull()) 158 summary.averageRating = pkg->DerivedRating(); 159 160 package->SetRatingSummary(summary); 161 162 package->SetHasChangelog(pkg->HasChangelog()); 163 164 if (!pkg->ProminenceOrderingIsNull()) 165 package->SetProminence(pkg->ProminenceOrdering()); 166 167 int32 countPkgScreenshots = pkg->CountPkgScreenshots(); 168 169 for (i = 0; i < countPkgScreenshots; i++) { 170 DumpExportPkgScreenshot* screenshot = pkg->PkgScreenshotsItemAt(i); 171 package->AddScreenshotInfo(ScreenshotInfo( 172 *(screenshot->Code()), 173 static_cast<int32>(screenshot->Width()), 174 static_cast<int32>(screenshot->Height()), 175 static_cast<int32>(screenshot->Length()) 176 )); 177 } 178 179 if (fDebugEnabled) { 180 printf("did populate data for [%s] (%s)\n", pkg->Name()->String(), 181 fDepotName.String()); 182 } 183 184 fCount++; 185 186 package->EndCollatingChanges(); 187 188 return !fStoppable->WasStopped(); 189 } 190 191 192 uint32 193 PackageFillingPkgListener::Count() 194 { 195 return fCount; 196 } 197 198 199 bool 200 PackageFillingPkgListener::Handle(DumpExportPkg* pkg) 201 { 202 const DepotInfo* depotInfo = fModel->DepotForName(fDepotName); 203 204 if (depotInfo != NULL) { 205 const BString packageName = *(pkg->Name()); 206 int32 packageIndex = depotInfo->PackageIndexByName(packageName); 207 208 if (-1 != packageIndex) { 209 const PackageList& packages = depotInfo->Packages(); 210 const PackageInfoRef& packageInfoRef = 211 packages.ItemAtFast(packageIndex); 212 213 AutoLocker<BLocker> locker(fModel->Lock()); 214 ConsumePackage(packageInfoRef, pkg); 215 } else { 216 printf("[PackageFillingPkgListener] unable to find the pkg [%s]\n", 217 packageName.String()); 218 } 219 } else { 220 printf("[PackageFillingPkgListener] unable to find the depot [%s]\n", 221 fDepotName.String()); 222 } 223 224 return !fStoppable->WasStopped(); 225 } 226 227 228 void 229 PackageFillingPkgListener::Complete() 230 { 231 } 232 233 234 ServerPkgDataUpdateProcess::ServerPkgDataUpdateProcess( 235 BString naturalLanguageCode, 236 BString depotName, 237 Model *model, 238 uint32 serverProcessOptions) 239 : 240 AbstractSingleFileServerProcess(serverProcessOptions), 241 fNaturalLanguageCode(naturalLanguageCode), 242 fModel(model), 243 fDepotName(depotName) 244 { 245 fName.SetToFormat("ServerPkgDataUpdateProcess<%s>", depotName.String()); 246 fDescription.SetTo( 247 B_TRANSLATE("Synchronizing package data for repository " 248 "'%REPO_NAME%'")); 249 fDescription.ReplaceAll("%REPO_NAME%", depotName.String()); 250 } 251 252 253 ServerPkgDataUpdateProcess::~ServerPkgDataUpdateProcess() 254 { 255 } 256 257 258 const char* 259 ServerPkgDataUpdateProcess::Name() const 260 { 261 return fName.String(); 262 } 263 264 265 const char* 266 ServerPkgDataUpdateProcess::Description() const 267 { 268 return fDescription.String(); 269 } 270 271 272 BString 273 ServerPkgDataUpdateProcess::UrlPathComponent() 274 { 275 BString urlPath; 276 urlPath.SetToFormat("/__pkg/all-%s-%s.json.gz", 277 _DeriveWebAppRepositorySourceCode().String(), 278 fNaturalLanguageCode.String()); 279 return urlPath; 280 } 281 282 283 status_t 284 ServerPkgDataUpdateProcess::GetLocalPath(BPath& path) const 285 { 286 BString webAppRepositorySourceCode = _DeriveWebAppRepositorySourceCode(); 287 288 if (!webAppRepositorySourceCode.IsEmpty()) { 289 AutoLocker<BLocker> locker(fModel->Lock()); 290 return fModel->DumpExportPkgDataPath(path, webAppRepositorySourceCode); 291 } 292 293 return B_ERROR; 294 } 295 296 297 status_t 298 ServerPkgDataUpdateProcess::ProcessLocalData() 299 { 300 BStopWatch watch("ServerPkgDataUpdateProcess::ProcessLocalData", true); 301 302 PackageFillingPkgListener* itemListener = 303 new PackageFillingPkgListener(fModel, fDepotName, this); 304 ObjectDeleter<PackageFillingPkgListener> 305 itemListenerDeleter(itemListener); 306 307 BulkContainerDumpExportPkgJsonListener* listener = 308 new BulkContainerDumpExportPkgJsonListener(itemListener); 309 ObjectDeleter<BulkContainerDumpExportPkgJsonListener> 310 listenerDeleter(listener); 311 312 BPath localPath; 313 status_t result = GetLocalPath(localPath); 314 315 if (result != B_OK) 316 return result; 317 318 result = ParseJsonFromFileWithListener(listener, localPath); 319 320 if (B_OK != result) 321 return result; 322 323 if (Logger::IsInfoEnabled()) { 324 double secs = watch.ElapsedTime() / 1000000.0; 325 printf("[%s] did process %" B_PRIi32 " packages' data " 326 "in (%6.3g secs)\n", Name(), itemListener->Count(), secs); 327 } 328 329 return listener->ErrorStatus(); 330 } 331 332 333 status_t 334 ServerPkgDataUpdateProcess::GetStandardMetaDataPath(BPath& path) const 335 { 336 return GetLocalPath(path); 337 } 338 339 340 void 341 ServerPkgDataUpdateProcess::GetStandardMetaDataJsonPath( 342 BString& jsonPath) const 343 { 344 jsonPath.SetTo("$.info"); 345 } 346 347 348 BString 349 ServerPkgDataUpdateProcess::_DeriveWebAppRepositorySourceCode() const 350 { 351 const DepotInfo* depot = fModel->DepotForName(fDepotName); 352 353 if (depot == NULL) { 354 return BString(); 355 } 356 357 return depot->WebAppRepositorySourceCode(); 358 } 359 360 361 status_t 362 ServerPkgDataUpdateProcess::RunInternal() 363 { 364 if (_DeriveWebAppRepositorySourceCode().IsEmpty()) { 365 if (Logger::IsInfoEnabled()) { 366 printf("[%s] am not updating data for depot [%s] as there is no" 367 " web app repository source code available\n", 368 Name(), fDepotName.String()); 369 } 370 return B_OK; 371 } 372 373 return AbstractSingleFileServerProcess::RunInternal(); 374 } 375