1 /* 2 * Copyright 2017-2019, Andrew Lindesay <apl@lindesay.co.nz>. 3 * All rights reserved. Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "ServerPkgDataUpdateProcess.h" 8 9 #include <stdio.h> 10 #include <sys/stat.h> 11 #include <time.h> 12 13 #include <AutoDeleter.h> 14 #include <AutoLocker.h> 15 #include <Catalog.h> 16 #include <FileIO.h> 17 #include <support/StopWatch.h> 18 #include <Url.h> 19 20 #include "Logger.h" 21 #include "ServerSettings.h" 22 #include "StorageUtils.h" 23 #include "DumpExportPkg.h" 24 #include "DumpExportPkgCategory.h" 25 #include "DumpExportPkgJsonListener.h" 26 #include "DumpExportPkgScreenshot.h" 27 #include "DumpExportPkgVersion.h" 28 #include "HaikuDepotConstants.h" 29 30 31 #undef B_TRANSLATION_CONTEXT 32 #define B_TRANSLATION_CONTEXT "ServerPkgDataUpdateProcess" 33 34 35 /*! This package listener (not at the JSON level) is feeding in the 36 packages as they are parsed and processing them. 37 */ 38 39 class PackageFillingPkgListener : public DumpExportPkgListener { 40 public: 41 PackageFillingPkgListener(Model *model, 42 BString& depotName, Stoppable* stoppable); 43 virtual ~PackageFillingPkgListener(); 44 45 virtual bool ConsumePackage(const PackageInfoRef& package, 46 DumpExportPkg* pkg); 47 virtual bool Handle(DumpExportPkg* item); 48 virtual void Complete(); 49 50 uint32 Count(); 51 52 private: 53 int32 IndexOfPackageByName(const BString& name) const; 54 int32 IndexOfCategoryByName( 55 const BString& name) const; 56 int32 IndexOfCategoryByCode( 57 const BString& code) const; 58 59 BString fDepotName; 60 Model* fModel; 61 CategoryList fCategories; 62 Stoppable* fStoppable; 63 uint32 fCount; 64 bool fDebugEnabled; 65 }; 66 67 68 PackageFillingPkgListener::PackageFillingPkgListener(Model* model, 69 BString& depotName, Stoppable* stoppable) 70 : 71 fDepotName(depotName), 72 fModel(model), 73 fStoppable(stoppable), 74 fCount(0), 75 fDebugEnabled(Logger::IsDebugEnabled()) 76 { 77 fCategories = model->Categories(); 78 } 79 80 81 PackageFillingPkgListener::~PackageFillingPkgListener() 82 { 83 } 84 85 86 // TODO; performance could be improved by not needing the linear search 87 88 inline int32 89 PackageFillingPkgListener::IndexOfCategoryByCode( 90 const BString& code) const 91 { 92 int32 i; 93 int32 categoryCount = fCategories.CountItems(); 94 95 for (i = 0; i < categoryCount; i++) { 96 const CategoryRef categoryRef = fCategories.ItemAtFast(i); 97 98 if (categoryRef->Code() == code) 99 return i; 100 } 101 102 return -1; 103 } 104 105 106 bool 107 PackageFillingPkgListener::ConsumePackage(const PackageInfoRef& package, 108 DumpExportPkg* pkg) 109 { 110 int32 i; 111 112 // Collects all of the changes here into one set of notifications to 113 // the package's listeners. This way the quantity of BMessages 114 // communicated back to listeners is considerably reduced. See stop 115 // invocation later in this method. 116 117 package->StartCollatingChanges(); 118 119 if (0 != pkg->CountPkgVersions()) { 120 121 // this makes the assumption that the only version will be the 122 // latest one. 123 124 DumpExportPkgVersion* pkgVersion = pkg->PkgVersionsItemAt(0); 125 126 if (!pkgVersion->TitleIsNull()) 127 package->SetTitle(*(pkgVersion->Title())); 128 129 if (!pkgVersion->SummaryIsNull()) 130 package->SetShortDescription(*(pkgVersion->Summary())); 131 132 if (!pkgVersion->DescriptionIsNull()) 133 package->SetFullDescription(*(pkgVersion->Description())); 134 135 if (!pkgVersion->PayloadLengthIsNull()) 136 package->SetSize(pkgVersion->PayloadLength()); 137 } 138 139 int32 countPkgCategories = pkg->CountPkgCategories(); 140 141 for (i = 0; i < countPkgCategories; i++) { 142 BString* categoryCode = pkg->PkgCategoriesItemAt(i)->Code(); 143 int categoryIndex = IndexOfCategoryByCode(*(categoryCode)); 144 145 if (categoryIndex == -1) { 146 printf("unable to find the category for [%s]\n", 147 categoryCode->String()); 148 } else { 149 package->AddCategory( 150 fCategories.ItemAtFast(categoryIndex)); 151 } 152 } 153 154 RatingSummary summary; 155 summary.averageRating = RATING_MISSING; 156 157 if (!pkg->DerivedRatingIsNull()) 158 summary.averageRating = pkg->DerivedRating(); 159 160 package->SetRatingSummary(summary); 161 162 if (!pkg->ProminenceOrderingIsNull()) 163 package->SetProminence(pkg->ProminenceOrdering()); 164 165 int32 countPkgScreenshots = pkg->CountPkgScreenshots(); 166 167 for (i = 0; i < countPkgScreenshots; i++) { 168 DumpExportPkgScreenshot* screenshot = pkg->PkgScreenshotsItemAt(i); 169 package->AddScreenshotInfo(ScreenshotInfo( 170 *(screenshot->Code()), 171 static_cast<int32>(screenshot->Width()), 172 static_cast<int32>(screenshot->Height()), 173 static_cast<int32>(screenshot->Length()) 174 )); 175 } 176 177 if (fDebugEnabled) { 178 printf("did populate data for [%s] (%s)\n", pkg->Name()->String(), 179 fDepotName.String()); 180 } 181 182 fCount++; 183 184 package->EndCollatingChanges(); 185 186 return !fStoppable->WasStopped(); 187 } 188 189 190 uint32 191 PackageFillingPkgListener::Count() 192 { 193 return fCount; 194 } 195 196 197 bool 198 PackageFillingPkgListener::Handle(DumpExportPkg* pkg) 199 { 200 const DepotInfo* depotInfo = fModel->DepotForName(fDepotName); 201 202 if (depotInfo != NULL) { 203 BString packageName = *(pkg->Name()); 204 int32 packageIndex = depotInfo->PackageIndexByName(packageName); 205 206 if (-1 != packageIndex) { 207 PackageList packages = depotInfo->Packages(); 208 const PackageInfoRef& packageInfoRef = 209 packages.ItemAtFast(packageIndex); 210 211 AutoLocker<BLocker> locker(fModel->Lock()); 212 ConsumePackage(packageInfoRef, pkg); 213 } else { 214 printf("[PackageFillingPkgListener] unable to find the pkg [%s]\n", 215 packageName.String()); 216 } 217 } else { 218 printf("[PackageFillingPkgListener] unable to find the depot [%s]\n", 219 fDepotName.String()); 220 } 221 222 return !fStoppable->WasStopped(); 223 } 224 225 226 void 227 PackageFillingPkgListener::Complete() 228 { 229 } 230 231 232 ServerPkgDataUpdateProcess::ServerPkgDataUpdateProcess( 233 BString naturalLanguageCode, 234 BString depotName, 235 Model *model, 236 uint32 serverProcessOptions) 237 : 238 AbstractSingleFileServerProcess(serverProcessOptions), 239 fNaturalLanguageCode(naturalLanguageCode), 240 fModel(model), 241 fDepotName(depotName) 242 { 243 fName.SetToFormat("ServerPkgDataUpdateProcess<%s>", depotName.String()); 244 fDescription.SetTo( 245 B_TRANSLATE("Synchronizing package data for repository " 246 "'%REPO_NAME%'")); 247 fDescription.ReplaceAll("%REPO_NAME%", depotName.String()); 248 } 249 250 251 ServerPkgDataUpdateProcess::~ServerPkgDataUpdateProcess() 252 { 253 } 254 255 256 const char* 257 ServerPkgDataUpdateProcess::Name() const 258 { 259 return fName.String(); 260 } 261 262 263 const char* 264 ServerPkgDataUpdateProcess::Description() const 265 { 266 return fDescription.String(); 267 } 268 269 270 BString 271 ServerPkgDataUpdateProcess::UrlPathComponent() 272 { 273 BString urlPath; 274 urlPath.SetToFormat("/__pkg/all-%s-%s.json.gz", 275 _DeriveWebAppRepositorySourceCode().String(), 276 fNaturalLanguageCode.String()); 277 return urlPath; 278 } 279 280 281 status_t 282 ServerPkgDataUpdateProcess::GetLocalPath(BPath& path) const 283 { 284 BString webAppRepositorySourceCode = _DeriveWebAppRepositorySourceCode(); 285 286 if (!webAppRepositorySourceCode.IsEmpty()) { 287 AutoLocker<BLocker> locker(fModel->Lock()); 288 return fModel->DumpExportPkgDataPath(path, webAppRepositorySourceCode); 289 } 290 291 return B_ERROR; 292 } 293 294 295 status_t 296 ServerPkgDataUpdateProcess::ProcessLocalData() 297 { 298 BStopWatch watch("ServerPkgDataUpdateProcess::ProcessLocalData", true); 299 300 PackageFillingPkgListener* itemListener = 301 new PackageFillingPkgListener(fModel, fDepotName, this); 302 ObjectDeleter<PackageFillingPkgListener> 303 itemListenerDeleter(itemListener); 304 305 BulkContainerDumpExportPkgJsonListener* listener = 306 new BulkContainerDumpExportPkgJsonListener(itemListener); 307 ObjectDeleter<BulkContainerDumpExportPkgJsonListener> 308 listenerDeleter(listener); 309 310 BPath localPath; 311 status_t result = GetLocalPath(localPath); 312 313 if (result != B_OK) 314 return result; 315 316 result = ParseJsonFromFileWithListener(listener, localPath); 317 318 if (B_OK != result) 319 return result; 320 321 if (Logger::IsInfoEnabled()) { 322 double secs = watch.ElapsedTime() / 1000000.0; 323 printf("[%s] did process %" B_PRIi32 " packages' data " 324 "in (%6.3g secs)\n", Name(), itemListener->Count(), secs); 325 } 326 327 return listener->ErrorStatus(); 328 } 329 330 331 status_t 332 ServerPkgDataUpdateProcess::GetStandardMetaDataPath(BPath& path) const 333 { 334 return GetLocalPath(path); 335 } 336 337 338 void 339 ServerPkgDataUpdateProcess::GetStandardMetaDataJsonPath( 340 BString& jsonPath) const 341 { 342 jsonPath.SetTo("$.info"); 343 } 344 345 346 BString 347 ServerPkgDataUpdateProcess::_DeriveWebAppRepositorySourceCode() const 348 { 349 const DepotInfo* depot = fModel->DepotForName(fDepotName); 350 351 if (depot == NULL) { 352 return BString(); 353 } 354 355 return depot->WebAppRepositorySourceCode(); 356 } 357 358 359 status_t 360 ServerPkgDataUpdateProcess::RunInternal() 361 { 362 if (_DeriveWebAppRepositorySourceCode().IsEmpty()) { 363 if (Logger::IsInfoEnabled()) { 364 printf("[%s] am not updating data for depot [%s] as there is no" 365 " web app repository source code available\n", 366 Name(), fDepotName.String()); 367 } 368 return B_OK; 369 } 370 371 return AbstractSingleFileServerProcess::RunInternal(); 372 } 373