Add a failure_counts.py and improve download parallelism

README.md

@@ -27,7 +27,15 @@ python3 scrape_tests.py dapr dapr
 
 You can scrape the output of one or more test suites for a particular regex.
 ```
-python3 grep_test_output.py dapr cli ".*expected.*Running.*"
+python3 grep_test_output.py dapr cli ".*expected.*Running.*" output.txt
+```
+
+## Failure counts
+
+To get the test failure counts by test for the last N days
+
+```
+python3 failure_counts.py dapr dapr 4 
 ```
 
 ## TODO: Document log scraper

failure_counts.py

@@ -0,0 +1,126 @@
+from ghapi.all import GhApi
+import requests
+from zipfile import ZipFile
+import json
+from termcolor import colored
+from prettytable import PrettyTable
+from dialog import Dialog
+from utils import FileCache, TestArtifacts
+from dateutil import parser
+from datetime import datetime, timezone
+import os
+import sys
+import re
+import urllib
+import threading
+import time
+
+api = GhApi(owner=sys.argv[1], repo=sys.argv[2])
+
+'''
+class AsyncRequestor(threading.Thread):
+    def run(self):
+        response = None
+        backoff = 1.0
+        while response is None:
+            try:
+                response = api.actions.list_artifacts_for_repo(per_page=100, page=self._args[0])
+                print("Reading page ", self._args[0])
+                self.response = response
+            except urllib.error.HTTPError as e:
+                sys.stderr.write(f"Http error thread_id {threading.get_ident()} (retrying): {str(e)}\n")
+                time.sleep(backoff)
+                if backoff < 30:
+                    backoff += backoff
+
+    def join(self):
+        threading.Thread.join(self)
+        return self.response
+'''
+
+class MultiSelectArtifactGenerator(TestArtifacts):
+    def top_menu(self):
+        choices = []
+        index = 1
+        for artifact in sorted(self.available_artifacts.keys()):
+            choices.append((artifact, '', False))
+            index += 1
+        d = Dialog(dialog="dialog")
+        code, chosen_test_suite = d.checklist('Select from available test outputs', choices=choices)
+
+        if code != d.OK:
+            sys.exit(1)
+
+        run_choice = ''
+
+        return chosen_test_suite
+
+    def get_for_test_suite(self, suite_names):
+        for suite_name in suite_names:
+            for artifact in self.available_artifacts[suite_name]:
+                yield artifact
+
+        for artifact in self.get_next_artifact(suite_names):
+            yield artifact
+
+
+
+global_cache = FileCache()
+all_artifacts = MultiSelectArtifactGenerator()
+
+output_filename = None
+if len(sys.argv)>=5:
+    output_filename = sys.argv[4]
+
+class TestSuite:
+    def __init__(self, test_suite_names):
+        self.run_results = {}
+        self.names = test_suite_names
+
+    def print_results(self):
+        output = PrettyTable()
+
+        output.field_names = ['Test', 'Failure Count']
+
+
+        for test in sorted(self.run_results.keys(), key=lambda x:self.run_results[x]):
+            output.add_row([test, self.run_results[test]])
+        output.align = "l"
+
+        print(output.get_string())
+
+    def count_failures(self, duration_days):
+        prefetch_artifacts = all_artifacts.get_for_test_suite(self.names)
+
+        for filepath, artifact in global_cache.prefetch(prefetch_artifacts):
+            with ZipFile(filepath) as zfile:
+                for index in range(len(zfile.namelist())):
+                    try:
+                        for line in zfile.read(zfile.namelist()[index]).decode('utf-8').split('\n'):
+                            if len(line) and ".json" in zfile.namelist()[index]:
+                                event = json.loads(line)
+                                event_time = parser.parse(event['Time'])
+                                if (datetime.now(timezone.utc) - event_time).days > duration_days:
+                                    self.print_results()  
+                                    return
+                                if event['Action'] == 'fail':
+                                    if 'Test' in event:
+                                        test_name = event['Test']
+
+                                        if test_name not in self.run_results:
+                                            self.run_results[test_name] = 0
+
+                                        self.run_results[test_name] += 1
+
+                    except Exception as e:
+                        print("Error reading artifact:", artifact, str(e), zfile.namelist()[index], line)
+
+
+
+
+all_artifacts.get_available_test_results()
+chosen_test_suites = all_artifacts.top_menu()
+
+ts = TestSuite(chosen_test_suites)
+
+ts.count_failures(int(sys.argv[3]))

grep_test_output.py

@@ -82,10 +82,9 @@ def grep_results(self, regex_string):
 
     def print_outputs(self, regex_string, outputs):
         prefetch_artifacts = all_artifacts.get_for_test_suite(self.names)
-        artifacts = all_artifacts.get_for_test_suite(self.names)
-        for artifact in artifacts:
-            global_cache.prefetch(prefetch_artifacts)
-            with ZipFile(global_cache.get(f"{artifact.name}.{artifact.id}", artifact.archive_download_url)) as zfile:
+
+        for filepath, artifact in global_cache.prefetch(prefetch_artifacts):
+            with ZipFile(filepath) as zfile:
                 for index in range(len(zfile.namelist())):
                     try:
                         for line in zfile.read(zfile.namelist()[index]).decode('utf-8').split('\n'):

scrape_tests.py

@@ -38,63 +38,71 @@ def top_menu(self):
         return chosen_test_suite
 
     def get_for_test_suite(self, suite_name):
-        for artifact in self.available_artifacts[suite_name]:
-            yield artifact
-
-        for artifact in self.get_next_artifact(suite_name):
-            yield artifact
+        position = 0
+        while True:
+            if position >= len(self.available_artifacts[suite_name]):
+                self.get_artifacts(batch_size=200)
+                continue
+
+            yield self.available_artifacts[suite_name][position]
+            position += 1
 
 global_cache = FileCache()
 all_artifacts = SingleTestSuiteView()
 
+object_process_counts = {}
+
 class TestSuite:
     def __init__(self, test_suite_name):
         self.run_results = {}
         self.name = test_suite_name
+        self.prefetch_artifacts = all_artifacts.get_for_test_suite(self.name)
+        self.run_ids = set()
 
-    def generate_results_table(self, artifacts, prefetch_artifacts, batch_size=10):
-        artifact_count = 0
-        for artifact in artifacts:
-            global_cache.prefetch(prefetch_artifacts)
-            if artifact_count > batch_size:
+    def generate_results_table(self, num_columns=1):
+        if len(self.run_ids) > num_columns:
+            return
+        for filepath, artifact in global_cache.prefetch(self.prefetch_artifacts):
+            try:
+                with ZipFile(filepath) as zfile:
+                    for index in range(len(zfile.namelist())):
+                        try:
+                            for line in zfile.read(zfile.namelist()[index]).decode('utf-8').split('\n'):
+                                if len(line) and ".json" in zfile.namelist()[index]:
+                                    event = json.loads(line)
+
+                                    key = event["Package"]
+
+                                    self.run_ids.add(artifact.id)
+
+                                    if 'Test' in event:
+                                        key += ":" + event['Test']
+
+                                    if key not in self.run_results:
+                                        self.run_results[key] = {artifact.id: {'output': ''}}
+                                    if artifact.id not in self.run_results[key]:
+                                        self.run_results[key][artifact.id] = {'output': ''}
+
+                                    if event['Action'] in ('pass', 'fail'):
+                                        self.run_results[key][artifact.id]['result'] = event['Action']
+
+                                    if event['Action'] == 'output':
+                                        self.run_results[key][artifact.id]['output'] += event['Output']
+                        except Exception as e:
+                            sys.stdout.write(f"Error reading artifact: {artifact} {str(e)} {zfile.namelist()[index]}\n")
+
+            except Exception as e:
+                sys.stdout.write(f"Error reading artifact: {artifact} {str(e)}\n")
+
+            if len(self.run_ids) > num_columns:
                 return
-            artifact_count+=1
-            with ZipFile(global_cache.get(f"{artifact.name}.{artifact.id}", artifact.archive_download_url)) as zfile:
-
-                for index in range(len(zfile.namelist())):
-                    try:
-                        for line in zfile.read(zfile.namelist()[index]).decode('utf-8').split('\n'):
-                            if len(line) and ".json" in zfile.namelist()[index]:
-                                event = json.loads(line)
-
-                                key = event["Package"]
-
-                                self.run_ids.add(artifact.id)
-
-                                if 'Test' in event:
-                                    key += ":" + event['Test']
-
-                                if key not in self.run_results:
-                                    self.run_results[key] = {artifact.id: {'output': ''}}
-                                if artifact.id not in self.run_results[key]:
-                                    self.run_results[key][artifact.id] = {'output': ''}
-
-                                if event['Action'] in ('pass', 'fail'):
-                                    self.run_results[key][artifact.id]['result'] = event['Action']
-
-                                if event['Action'] == 'output':
-                                    self.run_results[key][artifact.id]['output'] += event['Output']
-                    except Exception as e:
-                        sys.stdout.write(f"Error reading artifact: {artifact} {str(e)} {zfile.namelist()[index]} {line}\n")
-
 
     def drilldown(self, run):
         choices = []
         run_choice = sorted(self.run_ids, reverse=True)[run-1]
         for run in all_artifacts.get_for_test_suite(self.name):
             if run.id == run_choice:
                 test_results = { key: self.run_results[key][run_choice]['result'] for key in self.run_results.keys() if run_choice in self.run_results[key] and 'result' in self.run_results[key][run_choice] }
-                print(test_results)
                 choices = []
                 for test in sorted(self.run_results.keys()):
                     result = 'no_data'
@@ -168,23 +176,21 @@ def generate_table_string(self, start = 0, length = 1):
 
 
     def print_test_history(self):
-        self.run_ids = set()
         cw, ch = terminal_size()
-
-        prefetch_artifacts = all_artifacts.get_for_test_suite(self.name)
-        artifacts = all_artifacts.get_for_test_suite(self.name)
-        self.generate_results_table(prefetch_artifacts, artifacts)
+        if len(self.run_results) == 0:
+            self.generate_results_table()
         # First generate the table to see how wide it is
         table = self.generate_table_string(0, 1)
         table_width = len(table.split("\n")[0])
 
 
         # If we've got more space, keep fetching results until we can fill the screen
         if cw > table_width:
-            sys.stderr.write(f"Console width: {cw}, table size: {table_width}. Should have room for  {(cw - table_width)/5} more runs\n")
-            target_artifacts = int((cw - table_width)/5) + 1
+            remaining_runs = int((cw - table_width)/5)
+            sys.stderr.write(f"Console width: {cw}, table size: {table_width}. Should have room for  {remaining_runs} more test runs\n")
+            target_artifacts = remaining_runs + 1
             while len(self.run_ids) < target_artifacts:
-                self.generate_results_table(prefetch_artifacts, artifacts)
+                self.generate_results_table(target_artifacts)
             table = self.generate_table_string(0,target_artifacts)
             table_width = len(table.split("\n")[0])
 
@@ -197,10 +203,9 @@ def print_test_history(self):
 
 all_artifacts.get_available_test_results()
 chosen_test_suite = all_artifacts.top_menu()
+all_test_suites = {chosen_test_suite: TestSuite(chosen_test_suite)}
 
-ts = TestSuite(chosen_test_suite)
-
-ts.print_test_history()
+all_test_suites[chosen_test_suite].print_test_history()
 print('Press "d" to drilldown. "q" to quit. "m" for top menu')
 while True:
     choice = getkey()
@@ -209,7 +214,7 @@ def print_test_history(self):
         sys.exit(0)
 
     if choice == "o":
-        ts.print_test_history()
+        all_test_suites[chosen_test_suite].print_test_history()
         print('Press "d" to drilldown. "q" to quit. "m" for top menu')
 
     if choice == "d":
@@ -218,18 +223,19 @@ def print_test_history(self):
         except ValueError:
             print('Invalid run id\nPress "d" to drilldown. "q" to quit. "o" for overview')
             continue
-        if not ts.drilldown(run):
-            ts.print_test_history()
+        if not all_test_suites[chosen_test_suite].drilldown(run):
+            all_test_suites[chosen_test_suite].print_test_history()
             print('Press "d" to drilldown. "q" to quit. "m" for top menu')
         else:
             print('Press "q" to quit. "o" for overview. "m" for top menu')
 
     if choice == "m":
         chosen_test_suite = all_artifacts.top_menu()
 
-        ts = TestSuite(chosen_test_suite)
+        if chosen_test_suite not in all_test_suites:
+            all_test_suites[chosen_test_suite] = TestSuite(chosen_test_suite)
 
         #ts.generate_results_table()
-        ts.print_test_history()
+        all_test_suites[chosen_test_suite].print_test_history()
         print('Press "d" to drilldown. "q" to quit. "m" for top menu')