1 /* 2 * Copyright 2017-2020, Andrew Lindesay <apl@lindesay.co.nz>. 3 * All rights reserved. Distributed under the terms of the MIT License. 4 */ 5 6 7 #include "ServerPkgDataUpdateProcess.h" 8 9 #include <stdio.h> 10 #include <sys/stat.h> 11 #include <time.h> 12 13 #include <AutoDeleter.h> 14 #include <AutoLocker.h> 15 #include <Catalog.h> 16 #include <FileIO.h> 17 #include <support/StopWatch.h> 18 #include <Url.h> 19 20 #include "Logger.h" 21 #include "ServerSettings.h" 22 #include "StorageUtils.h" 23 #include "DumpExportPkg.h" 24 #include "DumpExportPkgCategory.h" 25 #include "DumpExportPkgJsonListener.h" 26 #include "DumpExportPkgScreenshot.h" 27 #include "DumpExportPkgVersion.h" 28 #include "HaikuDepotConstants.h" 29 30 31 #undef B_TRANSLATION_CONTEXT 32 #define B_TRANSLATION_CONTEXT "ServerPkgDataUpdateProcess" 33 34 35 /*! This package listener (not at the JSON level) is feeding in the 36 packages as they are parsed and processing them. 37 */ 38 39 class PackageFillingPkgListener : public DumpExportPkgListener { 40 public: 41 PackageFillingPkgListener(Model *model, 42 BString& depotName, Stoppable* stoppable); 43 virtual ~PackageFillingPkgListener(); 44 45 virtual bool ConsumePackage(const PackageInfoRef& package, 46 DumpExportPkg* pkg); 47 virtual bool Handle(DumpExportPkg* item); 48 virtual void Complete(); 49 50 uint32 Count(); 51 52 private: 53 int32 IndexOfPackageByName(const BString& name) const; 54 55 private: 56 BString fDepotName; 57 Model* fModel; 58 std::vector<CategoryRef> 59 fCategories; 60 Stoppable* fStoppable; 61 uint32 fCount; 62 bool fDebugEnabled; 63 }; 64 65 66 PackageFillingPkgListener::PackageFillingPkgListener(Model* model, 67 BString& depotName, Stoppable* stoppable) 68 : 69 fDepotName(depotName), 70 fModel(model), 71 fStoppable(stoppable), 72 fCount(0), 73 fDebugEnabled(Logger::IsDebugEnabled()) 74 { 75 } 76 77 78 PackageFillingPkgListener::~PackageFillingPkgListener() 79 { 80 } 81 82 83 bool 84 PackageFillingPkgListener::ConsumePackage(const PackageInfoRef& package, 85 DumpExportPkg* pkg) 86 { 87 int32 i; 88 89 // Collects all of the changes here into one set of notifications to 90 // the package's listeners. This way the quantity of BMessages 91 // communicated back to listeners is considerably reduced. See stop 92 // invocation later in this method. 93 94 package->StartCollatingChanges(); 95 96 if (0 != pkg->CountPkgVersions()) { 97 98 // this makes the assumption that the only version will be the 99 // latest one. 100 101 DumpExportPkgVersion* pkgVersion = pkg->PkgVersionsItemAt(0); 102 103 if (!pkgVersion->TitleIsNull()) 104 package->SetTitle(*(pkgVersion->Title())); 105 106 if (!pkgVersion->SummaryIsNull()) 107 package->SetShortDescription(*(pkgVersion->Summary())); 108 109 if (!pkgVersion->DescriptionIsNull()) 110 package->SetFullDescription(*(pkgVersion->Description())); 111 112 if (!pkgVersion->PayloadLengthIsNull()) 113 package->SetSize(pkgVersion->PayloadLength()); 114 } 115 116 int32 countPkgCategories = pkg->CountPkgCategories(); 117 118 for (i = 0; i < countPkgCategories; i++) { 119 BString* categoryCode = pkg->PkgCategoriesItemAt(i)->Code(); 120 CategoryRef category = fModel->CategoryByCode(*categoryCode); 121 122 if (category.Get() == NULL) { 123 HDERROR("unable to find the category for [%s]", 124 categoryCode->String()); 125 } else 126 package->AddCategory(category); 127 } 128 129 RatingSummary summary; 130 summary.averageRating = RATING_MISSING; 131 132 if (!pkg->DerivedRatingIsNull()) 133 summary.averageRating = pkg->DerivedRating(); 134 135 package->SetRatingSummary(summary); 136 137 package->SetHasChangelog(pkg->HasChangelog()); 138 139 if (!pkg->ProminenceOrderingIsNull()) 140 package->SetProminence(pkg->ProminenceOrdering()); 141 142 int32 countPkgScreenshots = pkg->CountPkgScreenshots(); 143 144 for (i = 0; i < countPkgScreenshots; i++) { 145 DumpExportPkgScreenshot* screenshot = pkg->PkgScreenshotsItemAt(i); 146 package->AddScreenshotInfo(ScreenshotInfoRef(new ScreenshotInfo( 147 *(screenshot->Code()), 148 static_cast<int32>(screenshot->Width()), 149 static_cast<int32>(screenshot->Height()), 150 static_cast<int32>(screenshot->Length()) 151 ), true)); 152 } 153 154 HDDEBUG("did populate data for [%s] (%s)", pkg->Name()->String(), 155 fDepotName.String()); 156 157 fCount++; 158 159 package->EndCollatingChanges(); 160 161 return !fStoppable->WasStopped(); 162 } 163 164 165 uint32 166 PackageFillingPkgListener::Count() 167 { 168 return fCount; 169 } 170 171 172 bool 173 PackageFillingPkgListener::Handle(DumpExportPkg* pkg) 174 { 175 const DepotInfo* depotInfo = fModel->DepotForName(fDepotName); 176 177 if (depotInfo != NULL) { 178 const BString packageName = *(pkg->Name()); 179 int32 packageIndex = depotInfo->PackageIndexByName(packageName); 180 181 if (-1 != packageIndex) { 182 const PackageList& packages = depotInfo->Packages(); 183 const PackageInfoRef& packageInfoRef = 184 packages.ItemAtFast(packageIndex); 185 186 AutoLocker<BLocker> locker(fModel->Lock()); 187 ConsumePackage(packageInfoRef, pkg); 188 } else { 189 HDINFO("[PackageFillingPkgListener] unable to find the pkg [%s]", 190 packageName.String()); 191 } 192 } else { 193 HDINFO("[PackageFillingPkgListener] unable to find the depot [%s]", 194 fDepotName.String()); 195 } 196 197 return !fStoppable->WasStopped(); 198 } 199 200 201 void 202 PackageFillingPkgListener::Complete() 203 { 204 } 205 206 207 ServerPkgDataUpdateProcess::ServerPkgDataUpdateProcess( 208 BString naturalLanguageCode, 209 BString depotName, 210 Model *model, 211 uint32 serverProcessOptions) 212 : 213 AbstractSingleFileServerProcess(serverProcessOptions), 214 fNaturalLanguageCode(naturalLanguageCode), 215 fModel(model), 216 fDepotName(depotName) 217 { 218 fName.SetToFormat("ServerPkgDataUpdateProcess<%s>", depotName.String()); 219 fDescription.SetTo( 220 B_TRANSLATE("Synchronizing package data for repository " 221 "'%REPO_NAME%'")); 222 fDescription.ReplaceAll("%REPO_NAME%", depotName.String()); 223 } 224 225 226 ServerPkgDataUpdateProcess::~ServerPkgDataUpdateProcess() 227 { 228 } 229 230 231 const char* 232 ServerPkgDataUpdateProcess::Name() const 233 { 234 return fName.String(); 235 } 236 237 238 const char* 239 ServerPkgDataUpdateProcess::Description() const 240 { 241 return fDescription.String(); 242 } 243 244 245 BString 246 ServerPkgDataUpdateProcess::UrlPathComponent() 247 { 248 BString urlPath; 249 urlPath.SetToFormat("/__pkg/all-%s-%s.json.gz", 250 _DeriveWebAppRepositorySourceCode().String(), 251 fNaturalLanguageCode.String()); 252 return urlPath; 253 } 254 255 256 status_t 257 ServerPkgDataUpdateProcess::GetLocalPath(BPath& path) const 258 { 259 BString webAppRepositorySourceCode = _DeriveWebAppRepositorySourceCode(); 260 261 if (!webAppRepositorySourceCode.IsEmpty()) { 262 AutoLocker<BLocker> locker(fModel->Lock()); 263 return fModel->DumpExportPkgDataPath(path, webAppRepositorySourceCode); 264 } 265 266 return B_ERROR; 267 } 268 269 270 status_t 271 ServerPkgDataUpdateProcess::ProcessLocalData() 272 { 273 BStopWatch watch("ServerPkgDataUpdateProcess::ProcessLocalData", true); 274 275 PackageFillingPkgListener* itemListener = 276 new PackageFillingPkgListener(fModel, fDepotName, this); 277 ObjectDeleter<PackageFillingPkgListener> 278 itemListenerDeleter(itemListener); 279 280 BulkContainerDumpExportPkgJsonListener* listener = 281 new BulkContainerDumpExportPkgJsonListener(itemListener); 282 ObjectDeleter<BulkContainerDumpExportPkgJsonListener> 283 listenerDeleter(listener); 284 285 BPath localPath; 286 status_t result = GetLocalPath(localPath); 287 288 if (result != B_OK) 289 return result; 290 291 result = ParseJsonFromFileWithListener(listener, localPath); 292 293 if (B_OK != result) 294 return result; 295 296 if (Logger::IsInfoEnabled()) { 297 double secs = watch.ElapsedTime() / 1000000.0; 298 HDINFO("[%s] did process %" B_PRIi32 " packages' data " 299 "in (%6.3g secs)", Name(), itemListener->Count(), secs); 300 } 301 302 return listener->ErrorStatus(); 303 } 304 305 306 status_t 307 ServerPkgDataUpdateProcess::GetStandardMetaDataPath(BPath& path) const 308 { 309 return GetLocalPath(path); 310 } 311 312 313 void 314 ServerPkgDataUpdateProcess::GetStandardMetaDataJsonPath( 315 BString& jsonPath) const 316 { 317 jsonPath.SetTo("$.info"); 318 } 319 320 321 BString 322 ServerPkgDataUpdateProcess::_DeriveWebAppRepositorySourceCode() const 323 { 324 const DepotInfo* depot = fModel->DepotForName(fDepotName); 325 326 if (depot == NULL) { 327 return BString(); 328 } 329 330 return depot->WebAppRepositorySourceCode(); 331 } 332 333 334 status_t 335 ServerPkgDataUpdateProcess::RunInternal() 336 { 337 if (_DeriveWebAppRepositorySourceCode().IsEmpty()) { 338 HDINFO("[%s] am not updating data for depot [%s] as there is no" 339 " web app repository source code available", 340 Name(), fDepotName.String()); 341 return B_OK; 342 } 343 344 return AbstractSingleFileServerProcess::RunInternal(); 345 } 346