Merge pull request #13 from eduardocerqueira/job

code improvement and adding more CPU respurces when running in k8s
eduardocerqueira · Jan 17, 2023 · 9cf6836 · 9cf6836
2 parents 9306260 + a69cd5a
commit 9cf6836
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 15 deletions.
diff --git a/linknotfound/phase.py b/linknotfound/phase.py
@@ -19,6 +19,7 @@
 from linknotfound.storage import upload_file
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
+from retry import retry
 
 logging.basicConfig(
     format="%(asctime)s,%(msecs)03d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s",
@@ -83,6 +84,14 @@ def filter_repos(self, repos) -> [Repo]:
         self.rp.total_repos_filtered = l_filtered.__len__()
         return l_filtered
 
+    @retry(tries=3, delay=10, logger=logging)
+    def clone_repo(self, full_name, local_path):
+        logging.info(f"cloning {full_name} at {local_path}")
+        Repo.clone_from(
+            url=f"https://{self.cfg.LNF_GITHUB_TOKEN}@github.com/{full_name}.git",
+            to_path=f"{local_path}",
+        )
+
     def scan(self, repos):
         """
         scan files
@@ -99,12 +108,7 @@ def scan(self, repos):
             rp_repo.url = repo.html_url
             rp_repo.path = f"{self.cfg.LNF_SCAN_PATH}/{repo.name}"
             logging.info(f"scanning {counter} out of {len(repos)} repos")
-            logging.info(f"cloning {repo.full_name} at {rp_repo.path}")
-            Repo.clone_from(
-                url=f"https://{self.cfg.LNF_GITHUB_TOKEN}@github.com/{repo.full_name}.git",
-                to_path=f"{rp_repo.path}",
-            )
-
+            self.clone_repo(full_name=repo.full_name, local_path=rp_repo.path)
             # repo files
             l_files = []
             for curr_path, currentDirectory, files in walk(f"{rp_repo.path}"):
@@ -153,7 +157,10 @@ def scan(self, repos):
                                 rp_doc.status = "ERROR"
 
                             lk.append(rp_doc)
-                            logging.info(f"{rp_doc.status}\n\t{rp_doc.file_name}")
+                            logging.info(f"FILE={rp_doc.file_name}")
+                            logging.info(
+                                f"HTTP_STATUS_CODE={rp_doc.status} {rp_doc.url}"
+                            )
                 except UnicodeError:
                     pass
             rp_repo.link = lk

diff --git a/ops/ocp/job.yaml b/ops/ocp/job.yaml
@@ -21,11 +21,11 @@ spec:
           imagePullPolicy: Always
           resources:
             limits:
-              cpu: 500m
-              memory: 4Gi
-            requests:
-              cpu: 200m
+              cpu: 1500m
               memory: 2Gi
+            requests:
+              cpu: 1000m
+              memory: "1.5Gi"
           terminationMessagePath: /dev/termination-log
           terminationMessagePolicy: File
       dnsPolicy: ClusterFirst

diff --git a/ops/ocp/job_cron.yaml b/ops/ocp/job_cron.yaml
@@ -32,11 +32,11 @@ spec:
               imagePullPolicy: Always
               resources:
                 limits:
-                  cpu: 500m
-                  memory: 4Gi
-                requests:
-                  cpu: 200m
+                  cpu: 1000m
                   memory: 2Gi
+                requests:
+                  cpu: 500m
+                  memory: "1.5Gi"
               terminationMessagePath: /dev/termination-log
               terminationMessagePolicy: File
           dnsPolicy: ClusterFirst

diff --git a/requirements.txt b/requirements.txt
@@ -5,3 +5,4 @@ gevent
 gitpython
 PyGithub
 requests
+retry
diff --git a/setup.py b/setup.py
@@ -28,6 +28,7 @@
         "GitPython==3.1.27",
         "PyGithub==1.55",
         "requests==2.28.1",
+        "retry==0.9.2",
     ],
     py_modules=["linknotfound"],
     entry_points={