Skip to content

Commit

Permalink
NEW: Static cache batching improvements.
Browse files Browse the repository at this point in the history
  • Loading branch information
mfendeksilverstripe committed Jun 9, 2021
1 parent 730af25 commit 62d9390
Show file tree
Hide file tree
Showing 6 changed files with 388 additions and 25 deletions.
2 changes: 2 additions & 0 deletions _config/staticpublishqueue.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ SilverStripe\Core\Injector\Injector:
properties:
States:
staticPublisherState: '%$SilverStripe\StaticPublishQueue\Dev\StaticPublisherState'
SilverStripe\StaticPublishQueue\Service\UrlBundleInterface:
class: SilverStripe\StaticPublishQueue\Service\UrlBundleService
SilverStripe\CMS\Model\SiteTree:
extensions:
- SilverStripe\StaticPublishQueue\Extension\Engine\SiteTreePublishingEngine
Expand Down
62 changes: 37 additions & 25 deletions src/Extension/Engine/SiteTreePublishingEngine.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,17 @@

namespace SilverStripe\StaticPublishQueue\Extension\Engine;

use SilverStripe\CMS\Model\SiteTree;
use SilverStripe\CMS\Model\SiteTreeExtension;
use SilverStripe\Core\Environment;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Core\Resettable;
use SilverStripe\ORM\ValidationException;
use SilverStripe\StaticPublishQueue\Contract\StaticPublishingTrigger;
use SilverStripe\StaticPublishQueue\Extension\Publishable\PublishableSiteTree;
use SilverStripe\StaticPublishQueue\Job\DeleteStaticCacheJob;
use SilverStripe\StaticPublishQueue\Job\GenerateStaticCacheJob;
use SilverStripe\StaticPublishQueue\Service\UrlBundleInterface;
use Symbiote\QueuedJobs\Services\QueuedJobService;

/**
Expand Down Expand Up @@ -99,15 +102,16 @@ public function setToDelete($toDelete)
}

/**
* @param \SilverStripe\CMS\Model\SiteTree|null $original
* @param SiteTree|SiteTreePublishingEngine|null $original
* @throws ValidationException
*/
public function onAfterPublishRecursive(&$original)
{
// if the site tree has been "reorganised" (ie: the parentID has changed)
// then this is eht equivalent of an unpublish and publish as far as the
// If the site tree has been "reorganised" (ie: the parentID has changed)
// then this is the equivalent of an un-publish and publish as far as the
// static publisher is concerned
if ($original && (
$original->ParentID !== $this->getOwner()->ParentID
(int) $original->ParentID !== (int) $this->getOwner()->ParentID
|| $original->URLSegment !== $this->getOwner()->URLSegment
)
) {
Expand All @@ -132,6 +136,9 @@ public function onBeforeUnpublish()
$this->collectChanges($context);
}

/**
* @throws ValidationException
*/
public function onAfterUnpublish()
{
$this->flushChanges();
Expand Down Expand Up @@ -160,44 +167,49 @@ public function collectChanges($context)

/**
* Execute URL deletions, enqueue URL updates.
* @throws ValidationException
*/
public function flushChanges()
{
$queue = static::$queueService ?? QueuedJobService::singleton();
$queueService = static::$queueService ?? QueuedJobService::singleton();

if (!empty($this->toUpdate)) {
foreach ($this->toUpdate as $queueItem) {
$job = Injector::inst()->create(GenerateStaticCacheJob::class);
if (count($this->toUpdate) > 0) {
/** @var UrlBundleInterface $urlService */
$urlService = Injector::inst()->create(UrlBundleInterface::class);

$jobData = new \stdClass();
$urls = $queueItem->urlsToCache();
foreach ($this->toUpdate as $item) {
$urls = $item->urlsToCache();
ksort($urls);
$jobData->URLsToProcess = $urls;
$urls = array_keys($urls);
$urlService->addUrls($urls);
}

$job->setJobData(0, 0, false, $jobData, [
'Building URLs: ' . var_export(array_keys($jobData->URLsToProcess), true),
]);
$jobs = $urlService->getJobsForUrls(GenerateStaticCacheJob::class, 'Building URLs', $this->owner);

$queue->queueJob($job);
foreach ($jobs as $job) {
$queueService->queueJob($job);
}

$this->toUpdate = [];
}

if (!empty($this->toDelete)) {
foreach ($this->toDelete as $queueItem) {
$job = Injector::inst()->create(DeleteStaticCacheJob::class);
if (count($this->toDelete) > 0) {
/** @var UrlBundleInterface $urlService */
$urlService = Injector::inst()->create(UrlBundleInterface::class);

$jobData = new \stdClass();
$urls = $queueItem->urlsToCache();
foreach ($this->toDelete as $item) {
$urls = $item->urlsToCache();
ksort($urls);
$jobData->URLsToProcess = $urls;
$urls = array_keys($urls);
$urlService->addUrls($urls);
}

$job->setJobData(0, 0, false, $jobData, [
'Purging URLs: ' . var_export(array_keys($jobData->URLsToProcess), true),
]);
$jobs = $urlService->getJobsForUrls(DeleteStaticCacheJob::class, 'Purging URLs', $this->owner);

$queue->queueJob($job);
foreach ($jobs as $job) {
$queueService->queueJob($job);
}

$this->toDelete = [];
}
}
Expand Down
49 changes: 49 additions & 0 deletions src/Job.php
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,45 @@ abstract class Job extends AbstractQueuedJob
*/
private static $chunk_size = 200;

/**
* Number of URLs per job allows you to split work into multiple smaller jobs instead of having one large job
* this is useful if you're running a queue setup will parallel processing or if you have too many URLs in general
* if this number is too high you're limiting the parallel processing opportunity
* if this number is too low you're using your resources inefficiently
* as every job processing has a fixed overhead which adds up if there are too many jobs
*
* in case you project is complex and you are struggling to find the correct number
* it's possible to move this value to a CMS setting and adjust as needed without the need of changing the code
* use @see Job::getUrlsPerJob() to override the value lookup
* you can subclass your jobs and implement your own getUrlsPerJob() method which will look into CMS setting
*
* batching capability can be disabled if urls per job is set to 0
* in such case, all URLs will be put into one job
*
* @var int
* @config
*/
private static $urls_per_job = 0;

/**
* Use this method to populate newly created job with data
*
* @param array $urls
* @param string|null $message
*/
public function hydrate(array $urls, ?string $message): void
{
$this->URLsToProcess = $urls;

if (!$message) {
return;
}

$this->messages = [
sprintf('%s: %s', $message, var_export(array_keys($urls), true)),
];
}

/**
* Static cache manipulation jobs need to run without a user
* this is because we don't want any session related data to become part of URLs
Expand Down Expand Up @@ -86,6 +125,16 @@ public function process(): void
$this->updateCompletedState();
}

/**
* @return int
*/
public function getUrlsPerJob(): int
{
$urlsPerJob = (int) $this->config()->get('urls_per_job');

return ($urlsPerJob > 0) ? $urlsPerJob : 0;
}

/**
* Implement this method to process URL
*
Expand Down
25 changes: 25 additions & 0 deletions src/Service/UrlBundleInterface.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
<?php

namespace SilverStripe\StaticPublishQueue\Service;

use SilverStripe\ORM\DataObject;

interface UrlBundleInterface
{
/**
* Add URLs to this bundle
*
* @param array $urls
*/
public function addUrls(array $urls): void;

/**
* Package URLs into jobs
*
* @param string $jobClass
* @param string|null $message
* @param DataObject|null $contextModel
* @return array
*/
public function getJobsForUrls(string $jobClass, ?string $message = null, ?DataObject $contextModel = null): array;
}
129 changes: 129 additions & 0 deletions src/Service/UrlBundleService.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
<?php

namespace SilverStripe\StaticPublishQueue\Service;

use SilverStripe\Core\Extensible;
use SilverStripe\Core\Injector\Injectable;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\ORM\DataObject;
use SilverStripe\StaticPublishQueue\Job;

/**
* Class UrlBundleService
*
* This service is responsible for bundling URLs which to static cache jobs
* Several extension points are available to allow further customisation
*/
class UrlBundleService implements UrlBundleInterface
{
use Extensible;
use Injectable;

/**
* @var array
*/
protected $urls = [];

/**
* @inheritDoc
*/
public function addUrls(array $urls): void
{
foreach ($urls as $url) {
$this->urls[$url] = $url;
}
}

/**
* @inheritDoc
*/
public function getJobsForUrls(string $jobClass, ?string $message = null, ?DataObject $contextModel = null): array
{
$singleton = singleton($jobClass);

if (!$singleton instanceof Job) {
return [];
}

$urls = $this->getUrls();
$urlsPerJob = $singleton->getUrlsPerJob();
$batches = $urlsPerJob > 0 ? array_chunk($urls, $urlsPerJob) : [$urls];
$jobs = [];

foreach ($batches as $urlBatch) {
$priorityUrls = $this->assignPriorityToUrls($urlBatch);

/** @var Job $job */
$job = Injector::inst()->create($jobClass);
$job->hydrate($priorityUrls, $message);

// Use this extension point to inject some additional data into the job
$this->extend('updateHydratedJob', $job, $contextModel);

$jobs[] = $job;
}

return $jobs;
}

/**
* Get URLs for further processing
*
* @return array
*/
protected function getUrls(): array
{
$urls = [];

foreach ($this->urls as $url) {
$url = $this->formatUrl($url);

if (!$url) {
continue;
}

$urls[] = $url;
}

$urls = array_unique($urls);

// Use this extension point to change the order of the URLs if needed
$this->extend('updateGetUrls', $urls);

return $urls;
}

/**
* Extensibility function which allows to handle custom formatting / encoding needs for URLs
* Returning "falsy" value will make the URL to be skipped
*
* @param string $url
* @return string|null
*/
protected function formatUrl(string $url): ?string
{
// Use this extension point to reformat URLs, for example encode special characters
$this->extend('updateFormatUrl', $url);

return $url;
}

/**
* Add priority data to URLs
*
* @param array $urls
* @return array
*/
protected function assignPriorityToUrls(array $urls): array
{
$priority = 0;
$priorityUrls = [];

foreach ($urls as $url) {
$priorityUrls[$url] = $priority;
$priority += 1;
}

return $priorityUrls;
}
}
Loading

0 comments on commit 62d9390

Please sign in to comment.