Skip to content

Commit

Permalink
fix: react correctly when pods fail to spawn
Browse files Browse the repository at this point in the history
  • Loading branch information
JanPokorny committed Oct 14, 2024
1 parent a98371f commit 495becb
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ async def _validate_request(
try:
protovalidate.validate(request)
except protovalidate.ValidationError as e:
logger.warning("Invalid request: error %s when validating %s", e, request)
logger.warning(
"Invalid request: error when validating %s", request, exc_info=True
)
await context.abort(grpc.StatusCode.INVALID_ARGUMENT, str(e.errors()))

async def Execute(
Expand All @@ -67,7 +69,7 @@ async def Execute(
files=request.files,
)
except Exception as e:
logger.error("Error executing code: %s", e)
logger.exception("Error executing code")
raise e

logger.info("Code execution completed with result %s", result)
Expand Down
2 changes: 1 addition & 1 deletion src/code_interpreter/services/http_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ async def execute(
files=request.files,
)
except Exception as e:
logger.error("Error executing code: %s", e)
logger.exception("Error executing code")
raise HTTPException(status_code=500, detail=str(e))
logger.info("Code execution completed with result %s", result)
return result
Expand Down
12 changes: 7 additions & 5 deletions src/code_interpreter/services/kubernetes_code_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,8 @@ async def fill_executor_pod_queue(self):
try:
self.executor_pod_queue.append(await pod_spawn_task)
spawned_pods += 1
except Exception as e:
logger.error("Failed to spawn executor pod:", e)
except Exception:
logger.exception("Failed to spawn executor pod")
finally:
self.executor_pod_queue_spawning_count -= 1
logger.info(
Expand Down Expand Up @@ -211,7 +211,7 @@ async def spawn_executor_pod(self):
random.choice(string.ascii_lowercase + string.digits) for _ in range(6)
)

pod = await self.kubectl.create(
await self.kubectl.create(
filename="-",
input={
"apiVersion": "v1",
Expand Down Expand Up @@ -242,12 +242,14 @@ async def spawn_executor_pod(self):
},
},
)
return await self.kubectl.wait("pod", name, _for="condition=Ready")
return await self.kubectl.wait(
"pod", name, _for="condition=Ready", timeout="60s"
)
except Exception:
try:
await self.kubectl.delete("pod", name)
finally:
pass
raise RuntimeError("Failed to spawn the pod")

@asynccontextmanager
async def executor_pod(self) -> AsyncGenerator[dict, None]:
Expand Down

0 comments on commit 495becb

Please sign in to comment.