Add scripts for generating developer metrics about EdgeX

Signed-off-by: Michael Hall <[email protected]>

Author: mhall119

project-metrics/README.md

@@ -0,0 +1,32 @@
+
+## Fetch repos
+
+Scripts that generate Git statistics use a local checkout of EdgeX git repos. Before you run those scripts, you will need to fetch the latest contents from Github.  To do that, run:
+
+```
+./retch_repos.sh
+```
+
+This will create a local ./repos folder with a checkout of all the EdgeX repositories from Github.
+
+## Release Statistics
+
+The Release Notes document should include Git and Github statistics. To create those run:
+
+```
+./release_metrics.py <YYYY-MM> <YYYY-MM> ./repos/*
+```
+
+Where the first YYYY-MM is the beginning month of the release cycle, and the second YYYY-MM is the ending month.
+
+(Tip: To generate statistics for just one repo, specify it as the last parameter instead of ./repos/*)
+
+(Tip: If you run this more than once, you'll need to get an access token from Github and set it to the environment variable GITHUB_ACCESS_TOKEN)
+
+## Dockerhub stats
+
+This script will report download numbers for every Docker image in the edgexfoundry organization:
+
+```
+./dockerhub-downloads.py
+```

project-metrics/dockerhub-downloads.py

@@ -0,0 +1,37 @@
+#!/usr/bin/python3
+
+import requests
+
+LIST_REPOS = 'https://hub.docker.com/v2/repositories/edgexfoundry/'
+GET_STATS = 'https://hub.docker.com/v2/repositories/%(namespace)s/%(name)s/'
+
+counts = dict()
+col_width = 0;
+
+next_page = LIST_REPOS
+while next_page is not None:
+    resp = requests.get(next_page)
+    next_page = None
+    if resp.status_code == 200:
+        data = resp.json()
+        # Read project data
+        for repo in data['results']:
+            counts[repo['name']] = repo['pull_count']
+            if len(repo['name']) > col_width:
+                col_width = len(repo['name'])
+
+        if data['next'] is not None:
+            next_page = data['next']
+
+total = 0
+header = "{0:<%d} : {1}" % col_width
+row = "{0:<%d} : {1:>9,}" % col_width
+print()
+print(header.format('Image', 'Downloads'))
+print('-'*(col_width+1) + ':' + '-'*10)
+for name, count in counts.items():
+    print(row.format(name, count))
+    total += count
+print('-'*(col_width+1) + ':' + '-'*10)
+print(row.format('Total', total))
+

project-metrics/fetch_repos.sh

@@ -0,0 +1,92 @@
+repos=(ci-management
+core-command
+core-command-client
+core-config-seed
+core-config-watcher
+core-data
+core-data-client
+core-domain
+core-exception
+core-metadata
+core-metadata-client
+core-test
+developer-scripts
+device-sdk
+device-sdk-tools
+device-virtual
+docker-core-command
+docker-core-config-seed
+docker-core-consul
+docker-core-data
+docker-core-metadata
+docker-device-virtual
+docker-edgex-mongo
+docker-edgex-mongo-seed
+docker-edgex-volume
+docker-export-client
+docker-export-distro
+docker-support-logging
+docker-support-notifications
+docker-support-rulesengine
+docker-support-scheduler
+export-client
+export-distro
+export-domain
+export-test
+edgex-ui-go
+support-domain
+support-logging
+support-logging-client
+support-notifications
+support-notifications-client
+support-rulesengine
+support-scheduler
+device-bacnet
+device-bluetooth
+device-fischertechnik
+device-modbus
+device-mqtt
+device-snmp
+device-domain
+device-controller
+device-sdk-go
+device-sdk-c
+blackbox-testing
+export-go
+consul-client-go
+core-clients-go
+core-command-go
+core-data-go
+core-domain-go
+core-metadata-go
+support-logging-client-go
+support-notifications-client-go
+support-domain-go
+device-scheduling
+edgex-go
+core-config-seed-go
+security-api-gateway
+security-secret-store)
+
+
+for repo in "${repos[@]}"
+do
+  #echo $repo
+  if [ ! -d ./repos/$repo ]; then
+      echo "Cloning $repo from https://github.com/edgexfoundry/$repo"
+      git clone https://github.com/edgexfoundry/$repo ./repos/$repo
+  else
+      cd ./repos/$repo
+      echo "Updating $repo"
+      git checkout master && git fetch --all && git pull
+      cd ../..
+  fi;
+
+  ### Uncomment/comment out as needed.
+  #git checkout master && git fetch --all && git rebase upstream/master
+  #git log --format='%aN'
+  #git rev-list --count --no-merges HEAD
+  #git log  | grep Author
+  #git log --format='%aN' | sort -u | wc -l
+  #echo "Total number of commits: $(git rev-list --count --no-merges HEAD)"
+done

project-metrics/release_metrics.py

@@ -0,0 +1,97 @@
+#!/usr/bin/python3
+
+import os, sys
+import subprocess
+import requests
+
+# Retrieve the git log of the current directory with line change stats (except renames and type changes)
+GIT_LOG_CMD = ['git', 'log', '--pretty=format:%aI%x20%aN%x20%ae', '--diff-filter=rt', '--numstat']
+#print(' '.join(GIT_LOG_CMD))
+
+GITHUB_ACCESS_TOKEN = os.environ.get('GITHUB_ACCESS_TOKEN', '')
+GITHUB_ISSUES_URL = 'https://api.github.com/repos/edgexfoundry/%(repo)s/issues?state=all&access_token=%(access_token)s&since=%(since)s&page=%(page)s'
+
+stats = 0
+commits = 0
+authors = dict()
+old_authors = set()
+repos = set()
+issues_opened = 0
+issues_closed = 0
+
+basedir = os.getcwd()
+
+from_date = sys.argv[1]
+to_date = sys.argv[2]
+#print('From: %s\tTo: %s' % (from_date, to_date))
+
+# Iterate through every local repo passed as a command-line argument
+for repo in sys.argv[3:]:
+
+    repo_name = repo.split('/')[-1]
+    issues_page = 1
+    while (issues_page):
+        repo_issues_url = GITHUB_ISSUES_URL % {'repo': repo_name, 'since': from_date, 'page': issues_page, 'access_token': GITHUB_ACCESS_TOKEN}
+        #print('Calling: %s' % repo_issues_url)
+        resp = requests.get(repo_issues_url)
+        if resp.status_code == 200:
+            issues = resp.json()
+            for issue in issues:
+                created = issue['created_at'][:10]
+                if created >= from_date and created <= to_date:
+                    issues_opened += 1
+                if issue['state'] == 'closed':
+                    closed  = issue['closed_at'][:10]
+                    if closed >= from_date and closed <= to_date:
+                        issues_closed += 1
+        if 'link' in resp.headers and 'rel="next"' in resp.headers['link']:
+            issues_page+= 1
+        else:
+            break
+
+    os.chdir(os.path.join(basedir, repo))
+    log = os.popen(' '.join(GIT_LOG_CMD))
+
+    date = None
+    for line in log:
+      args = line.split()
+      if len(args) > 2 and len(args[0]) == 25:
+          date = args[0][:10]
+          if date >= from_date and date <= to_date:
+              commits += 1
+              repos.add(repo)
+
+              email = args[-1]
+              author = ' '.join(args[1:-1])
+              if email not in authors or len(author) > len(authors[email]):
+                  authors[email] = author
+          elif date <= from_date:
+              email = args[-1]
+              old_authors.add(email)
+
+      elif len(args) == 3:
+          if date is not None and date >= from_date and date <= to_date:
+              if args[2].startswith('vendor'):
+                  continue
+              try:
+                  added = abs(int(args[0]))
+                  removed = abs(int(args[1]))
+                  stats += max(added, removed)# the larger value
+              except ValueError:
+                  pass
+
+print('%d Active Repos' % len(repos))
+print('%d Issues Opened' % issues_opened)
+print('%d Issues Closed' % issues_closed)
+print('%d Commits' % commits)
+print('%d Lines of code' % stats)
+print('%d Authors' % len(authors))
+print('%d New Authors' % len(authors.keys() - old_authors))
+for email in sorted(authors, key=lambda e: authors.get(e).lower()):
+    if email in old_authors:
+        print('%s <%s>' % (authors[email], email))
+    else:
+        print('**%s** <%s>' % (authors[email], email))
+
+
+