Skip to content

Commit

Permalink
adjust pvc storage if crawler is running and is used more than 2.2x a…
Browse files Browse the repository at this point in the history
…llocated storage

shouldn't happen generally, but could occasionally if crawler is downloading lots of large files in one page
  • Loading branch information
ikreymer committed Aug 14, 2024
1 parent 916813a commit 38c0165
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 1 deletion.
11 changes: 11 additions & 0 deletions backend/btrixcloud/operator/crawls.py
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,7 @@ def _load_crawler(self, params, i, status: CrawlStatus, children):
params["memory_limit"] = float(params["memory"]) * MEM_LIMIT_PADDING
else:
params["memory_limit"] = self.k8s.max_crawler_memory_size
params["storage"] = pod_info.newStorage or params.get("crawler_storage")
params["workers"] = params.get(worker_field) or 1
params["do_restart"] = False
if has_pod:
Expand Down Expand Up @@ -1325,6 +1326,16 @@ async def update_crawl_state(
pod_info = status.podStatus[key]
pod_info.used.storage = value

if (
status.state == "running"
and pod_info.allocated.storage
and pod_info.used.storage * 2.2 > pod_info.allocated.storage
):
pod_info.newStorage = int(pod_info.used.storage * 2.2)
print(
f"Attempting to adjust storage to {pod_info.newStorage} for {key}"
)

if not status.stopReason:
status.stopReason = await self.is_crawl_stopping(crawl, status, data)
status.stopping = status.stopReason is not None
Expand Down
1 change: 1 addition & 0 deletions backend/btrixcloud/operator/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ class PodInfo(BaseModel):

newCpu: Optional[int] = None
newMemory: Optional[int] = None
newStorage: Optional[int] = None
signalAtMem: Optional[int] = None

evicted: Optional[bool] = False
Expand Down
2 changes: 1 addition & 1 deletion chart/app-templates/crawler.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ spec:

resources:
requests:
storage: {{ crawler_storage }}
storage: {{ storage }}

{% if volume_storage_class %}
storageClassName: {{ volume_storage_class }}
Expand Down

0 comments on commit 38c0165

Please sign in to comment.