Skip to content

Commit

Permalink
refactor(cache): extract and encapsulate cache expiration logic (RSS-…
Browse files Browse the repository at this point in the history 
…Bridge#3547)

* refactor(cache): extract and encapsulate cache expiration logic

* fix: logic bug in getSimpleHTMLDOMCached

* fix: silly me, index should of course be on the key column

* silly me again, PRIMARY keys get index by default lol

* comment out the delete portion in loadData

* remove a few log statements

* tweak twitter cache timeout
  • Loading branch information
@dvikan
dvikan authored Jul 19, 2023
1 parent 087e790 commit 6254b85
Show file tree
Hide file tree
Showing 12 changed files with 123 additions and 136 deletions.
29 changes: 14 additions & 15 deletions actions/DisplayAction.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,36 +90,34 @@ public function execute(array $request)
$cache = RssBridge::getCache();
$cache->setScope('');
$cache->setKey($cache_params);
// This cache purge will basically delete all cache items older than 24h, regardless of scope and key
$cache->purgeCache(86400);

$items = [];
$infos = [];
$mtime = $cache->getTime();

$feed = $cache->loadData($cacheTimeout);

if (
$mtime
&& (time() - $cacheTimeout < $mtime)
$feed
&& !Debug::isEnabled()
) {
// At this point we found the feed in the cache and debug mode is disabled

if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) {
$modificationTime = $cache->getTime();
// The client wants to know if the feed has changed since its last check
$stime = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']);
if ($mtime <= $stime) {
$lastModified2 = gmdate('D, d M Y H:i:s ', $mtime) . 'GMT';
$modifiedSince = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']);
if ($modificationTime <= $modifiedSince) {
$lastModified2 = gmdate('D, d M Y H:i:s ', $modificationTime) . 'GMT';
return new Response('', 304, ['Last-Modified' => $lastModified2]);
}
}

// Load the feed from cache and prepare it
$cached = $cache->loadData();
if (isset($cached['items']) && isset($cached['extraInfos'])) {
foreach ($cached['items'] as $item) {
if (
isset($feed['items'])
&& isset($feed['extraInfos'])
) {
foreach ($feed['items'] as $item) {
$items[] = new FeedItem($item);
}
$infos = $cached['extraInfos'];
$infos = $feed['extraInfos'];
}
} else {
// At this point we did NOT find the feed in the cache or debug mode is enabled.
Expand Down Expand Up @@ -173,6 +171,7 @@ public function execute(array $request)
}, $items),
'extraInfos' => $infos
]);
$cache->purgeCache();
}

$format->setItems($items);
Expand Down
1 change: 1 addition & 0 deletions bridges/DemoBridge.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ class DemoBridge extends BridgeAbstract
const NAME = 'DemoBridge';
const URI = 'http://github.com/rss-bridge/rss-bridge';
const DESCRIPTION = 'Bridge used for demos';
const CACHE_TIMEOUT = 15;

const PARAMETERS = [
'testCheckbox' => [
Expand Down
17 changes: 6 additions & 11 deletions bridges/SpotifyBridge.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,14 +282,10 @@ private function fetchAccessToken()
$cacheKey = sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret'));
$cache->setScope('SpotifyBridge');
$cache->setKey([$cacheKey]);

$time = null;
if ($cache->getTime()) {
$time = (new DateTime())->getTimestamp() - $cache->getTime();
}

if (!$cache->getTime() || $time >= 3600) {
// fetch token
$token = $cache->loadData(3600);
if ($token) {
$this->token = $token;
} else {
$basicAuth = base64_encode(sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret')));
$json = getContents('https://accounts.spotify.com/api/token', [
"Authorization: Basic $basicAuth"
Expand All @@ -298,10 +294,9 @@ private function fetchAccessToken()
]);
$data = Json::decode($json);
$this->token = $data['access_token'];

$cache->setScope('SpotifyBridge');
$cache->setKey([$cacheKey]);
$cache->saveData($this->token);
} else {
$this->token = $cache->loadData();
}
}

Expand Down
6 changes: 1 addition & 5 deletions bridges/TwitterBridge.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ class TwitterBridge extends BridgeAbstract
const API_URI = 'https://api.twitter.com';
const GUEST_TOKEN_USES = 100;
const GUEST_TOKEN_EXPIRY = 10800; // 3hrs
const CACHE_TIMEOUT = 300; // 5min
const CACHE_TIMEOUT = 60 * 15; // 15min
const DESCRIPTION = 'returns tweets';
const MAINTAINER = 'arnd-s';
const PARAMETERS = [
Expand Down Expand Up @@ -224,10 +224,6 @@ public function collectData()
switch ($this->queriedContext) {
case 'By username':
$cache = RssBridge::getCache();
$cache->setScope('twitter');
$cache->setKey(['cache']);
// todo: inspect mtime instead of purging with 3h
$cache->purgeCache(60 * 60 * 3);
$api = new TwitterClient($cache);

$screenName = $this->getInput('u');
Expand Down
30 changes: 19 additions & 11 deletions caches/FileCache.php
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
<?php

/**
* @link https://www.php.net/manual/en/function.clearstatcache.php
*/
class FileCache implements CacheInterface
{
private array $config;
Expand All @@ -25,18 +28,25 @@ public function getConfig()
return $this->config;
}

public function loadData()
public function loadData(int $timeout = 86400)
{
clearstatcache();
if (!file_exists($this->getCacheFile())) {
return null;
}
$data = unserialize(file_get_contents($this->getCacheFile()));
if ($data === false) {
// Intentionally not throwing an exception
Logger::warning(sprintf('Failed to unserialize: %s', $this->getCacheFile()));
return null;
$modificationTime = filemtime($this->getCacheFile());
if (time() - $timeout < $modificationTime) {
$data = unserialize(file_get_contents($this->getCacheFile()));
if ($data === false) {
Logger::warning(sprintf('Failed to unserialize: %s', $this->getCacheFile()));
// Intentionally not throwing an exception
return null;
}
return $data;
}
return $data;
// It's a good idea to delete the expired item here, but commented out atm
// unlink($this->getCacheFile());
return null;
}

public function saveData($data): void
Expand All @@ -49,9 +59,7 @@ public function saveData($data): void

public function getTime(): ?int
{
// https://www.php.net/manual/en/function.clearstatcache.php
clearstatcache();

$cacheFile = $this->getCacheFile();
if (file_exists($cacheFile)) {
$time = filemtime($cacheFile);
Expand All @@ -64,7 +72,7 @@ public function getTime(): ?int
return null;
}

public function purgeCache(int $seconds): void
public function purgeCache(int $timeout = 86400): void
{
if (! $this->config['enable_purge']) {
return;
Expand All @@ -90,7 +98,7 @@ public function purgeCache(int $seconds): void
continue;
} elseif ($cacheFile->isFile()) {
$filepath = $cacheFile->getPathname();
if (filemtime($filepath) < time() - $seconds) {
if (filemtime($filepath) < time() - $timeout) {
// todo: sometimes this file doesn't exists
unlink($filepath);
}
Expand Down
51 changes: 26 additions & 25 deletions caches/MemcachedCache.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ class MemcachedCache implements CacheInterface
private string $key;
private $conn;
private $expiration = 0;
private $time = null;
private $data = null;

public function __construct()
{
Expand Down Expand Up @@ -43,50 +41,53 @@ public function __construct()
$this->conn = $conn;
}

public function loadData()
public function loadData(int $timeout = 86400)
{
if ($this->data) {
return $this->data;
}
$result = $this->conn->get($this->getCacheKey());
if ($result === false) {
$value = $this->conn->get($this->getCacheKey());
if ($value === false) {
return null;
}

$this->time = $result['time'];
$this->data = $result['data'];
return $result['data'];
if (time() - $timeout < $value['time']) {
return $value['data'];
}
return null;
}

public function saveData($data): void
{
$time = time();
$object_to_save = [
$value = [
'data' => $data,
'time' => $time,
'time' => time(),
];
$result = $this->conn->set($this->getCacheKey(), $object_to_save, $this->expiration);

$result = $this->conn->set($this->getCacheKey(), $value, $this->expiration);
if ($result === false) {
throw new \Exception('Cannot write the cache to memcached server');
Logger::warning('Failed to store an item in memcached', [
'scope' => $this->scope,
'key' => $this->key,
'expiration' => $this->expiration,
'code' => $this->conn->getLastErrorCode(),
'message' => $this->conn->getLastErrorMessage(),
'number' => $this->conn->getLastErrorErrno(),
]);
// Intentionally not throwing an exception
}

$this->time = $time;
}

public function getTime(): ?int
{
if ($this->time === null) {
$this->loadData();
$value = $this->conn->get($this->getCacheKey());
if ($value === false) {
return null;
}
return $this->time;
return $value['time'];
}

public function purgeCache(int $seconds): void
public function purgeCache(int $timeout = 86400): void
{
$this->conn->flush();
// Note: does not purges cache right now
// Just sets cache expiration and leave cache purging for memcached itself
$this->expiration = $seconds;
$this->expiration = $timeout;
}

public function setScope(string $scope): void
Expand Down
4 changes: 2 additions & 2 deletions caches/NullCache.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public function setKey(array $key): void
{
}

public function loadData()
public function loadData(int $timeout = 86400)
{
}

Expand All @@ -25,7 +25,7 @@ public function getTime(): ?int
return null;
}

public function purgeCache(int $seconds): void
public function purgeCache(int $timeout = 86400): void
{
}
}
38 changes: 24 additions & 14 deletions caches/SQLiteCache.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,27 +29,37 @@ public function __construct(array $config)
$this->db = new \SQLite3($config['file']);
$this->db->enableExceptions(true);
$this->db->exec("CREATE TABLE storage ('key' BLOB PRIMARY KEY, 'value' BLOB, 'updated' INTEGER)");
$this->db->exec('CREATE INDEX idx_storage_updated ON storage (updated)');
}
$this->db->busyTimeout($config['timeout']);
}

public function loadData()
public function loadData(int $timeout = 86400)
{
$stmt = $this->db->prepare('SELECT value FROM storage WHERE key = :key');
$stmt = $this->db->prepare('SELECT value, updated FROM storage WHERE key = :key');
$stmt->bindValue(':key', $this->getCacheKey());
$result = $stmt->execute();
if ($result) {
$row = $result->fetchArray(\SQLITE3_ASSOC);
if ($row !== false) {
$blob = $row['value'];
$data = unserialize($blob);
if ($data !== false) {
return $data;
}
Logger::error(sprintf("Failed to unserialize: '%s'", mb_substr($blob, 0, 100)));
if (!$result) {
return null;
}
$row = $result->fetchArray(\SQLITE3_ASSOC);
if ($row === false) {
return null;
}
$value = $row['value'];
$modificationTime = $row['updated'];
if (time() - $timeout < $modificationTime) {
$data = unserialize($value);
if ($data === false) {
Logger::error(sprintf("Failed to unserialize: '%s'", mb_substr($value, 0, 100)));
return null;
}
return $data;
}
// It's a good idea to delete expired cache items.
// However I'm seeing lots of SQLITE_BUSY errors so commented out for now
// $stmt = $this->db->prepare('DELETE FROM storage WHERE key = :key');
// $stmt->bindValue(':key', $this->getCacheKey());
// $stmt->execute();
return null;
}

Expand Down Expand Up @@ -78,13 +88,13 @@ public function getTime(): ?int
return null;
}

public function purgeCache(int $seconds): void
public function purgeCache(int $timeout = 86400): void
{
if (!$this->config['enable_purge']) {
return;
}
$stmt = $this->db->prepare('DELETE FROM storage WHERE updated < :expired');
$stmt->bindValue(':expired', time() - $seconds);
$stmt->bindValue(':expired', time() - $timeout);
$stmt->execute();
}

Expand Down
22 changes: 5 additions & 17 deletions lib/BridgeAbstract.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,26 +409,15 @@ public function detectParameters($url)
/**
* Loads a cached value for the specified key
*
* @param int $duration Cache duration (optional)
* @param int $timeout Cache duration (optional)
* @return mixed Cached value or null if the key doesn't exist or has expired
*/
protected function loadCacheValue(string $key, $duration = null)
protected function loadCacheValue(string $key, int $timeout = 86400)
{
$cache = RssBridge::getCache();
// Create class name without the namespace part
$scope = $this->getShortName();
$cache->setScope($scope);
$cache->setScope($this->getShortName());
$cache->setKey([$key]);
$timestamp = $cache->getTime();

if (
$duration
&& $timestamp
&& $timestamp < time() - $duration
) {
return null;
}
return $cache->loadData();
return $cache->loadData($timeout);
}

/**
Expand All @@ -439,8 +428,7 @@ protected function loadCacheValue(string $key, $duration = null)
protected function saveCacheValue(string $key, $value)
{
$cache = RssBridge::getCache();
$scope = $this->getShortName();
$cache->setScope($scope);
$cache->setScope($this->getShortName());
$cache->setKey([$key]);
$cache->saveData($value);
}
Expand Down
Loading

0 comments on commit 6254b85

Please sign in to comment.