-
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathfeeds.py
181 lines (159 loc) · 6.88 KB
/
feeds.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
import os
import sys
import argparse
from urllib.parse import urlparse
import xml.etree.ElementTree as ET
parser = argparse.ArgumentParser(description='Extracts release feeds from roles')
parser.add_argument('root_dir', help='Root dir which to traverse recursively for defaults/main.yml roles files')
parser.add_argument('action', help='Pass "check" to list roles with missing feeds or "dump" to dump an OPML file')
args = parser.parse_args()
if args.action not in ['check', 'dump']:
sys.exit('Error: possible arguments are "check" or "dump"')
excluded_paths = [
# appservice-kakaotalk defines a Project URL, but that Gitea repository does not have an Atom/RSS feed.
# It doesn't have any tags anyway.
'./upstream/roles/custom/matrix-bridge-appservice-kakaotalk/defaults',
]
project_source_url_str = '# Project source code URL:'
def get_roles_files_from_dir(root_dir):
file_paths = []
for dir_name, sub_dur_list, file_list in os.walk(root_dir):
for file_name in file_list:
if not dir_name.endswith('defaults') or file_name != 'main.yml':
continue
if dir_name in excluded_paths:
continue
file_paths.append(os.path.join(dir_name, file_name))
return file_paths
def get_git_repos_from_files(file_paths, break_on_missing_repos=False):
git_repos = {}
missing_repos = []
for file in file_paths:
file_lines = open(file, 'r').readlines()
found_project_repo = False
for line in file_lines:
project_repo_val = ''
if project_source_url_str in line:
# extract the value from a line like this:
# Project source code URL: https://github.com/mautrix/signal
project_repo_val = line.split(project_source_url_str)[1].strip()
if not validate_url(project_repo_val):
print('Invalid url for line ', line)
break
if project_repo_val != '':
if file not in git_repos:
git_repos[file] = []
git_repos[file].append(project_repo_val)
found_project_repo = True
if not found_project_repo:
missing_repos.append(file)
if break_on_missing_repos and len(missing_repos) > 0:
print('Missing `{0}` comment for:\n{1}'.format(project_source_url_str, '\n'.join(missing_repos)))
return git_repos
def validate_url(text):
if text == '':
return False
try:
result = urlparse(text)
return all([result.scheme, result.netloc])
except:
return False
def format_feeds_from_git_repos(git_repos):
feeds = {
'ansible': {
'text': 'ansible',
'title': 'ansible',
'type': 'rss',
'htmlUrl': 'https://pypi.org/project/ansible/#history',
'xmlUrl': 'https://pypi.org/rss/project/ansible/releases.xml'
},
'ansible-core': {
'text': 'ansible-core',
'title': 'ansible-core',
'type': 'rss',
'htmlUrl': 'https://pypi.org/project/ansible-core/#history',
'xmlUrl': 'https://pypi.org/rss/project/ansible-core/releases.xml'
},
'alpinelinux': {
'text': 'alpinelinux',
'title': 'alpinelinux',
'type': 'rss',
'htmlUrl': 'https://github.com/alpinelinux/aports/releases',
'xmlUrl': 'https://github.com/alpinelinux/aports/releases.atom'
},
'borg': {
'text': 'borg',
'title': 'borg',
'type': 'rss',
'htmlUrl': 'https://github.com/borgbackup/borg/releases',
'xmlUrl': 'https://github.com/borgbackup/borg/releases.atom'
},
'borgmatic': {
'text': 'borgmatic',
'title': 'borgmatic',
'type': 'rss',
'htmlUrl': 'https://github.com/borgmatic-collective/borgmatic/releases',
'xmlUrl': 'https://github.com/borgmatic-collective/borgmatic/releases.atom'
},
'mautrix-go': {
'text': 'mautrix-go',
'title': 'mautrix-go',
'type': 'rss',
'htmlUrl': 'https://github.com/mautrix/go/releases',
'xmlUrl': 'https://github.com/mautrix/go/releases.atom'
},
}
for role, git_repos in git_repos.items():
for idx, git_repo in enumerate(git_repos):
if 'github' in git_repo:
atomFilePath = git_repo.replace('.git', '') + '/releases.atom'
elif ('gitlab' in git_repo or 'mau.dev' in git_repo):
atomFilePath = git_repo.replace('.git', '') + '/-/tags?format=atom'
elif 'git.zx2c4.com' in git_repo: # cgit
atomFilePath = git_repo + '/atom/'
elif 'framagit.org' in git_repo: # gitlab
atomFilePath = git_repo.replace('.git', '') + '/-/tags?format=atom'
elif 'git.osgeo.org' in git_repo: # gitea
atomFilePath = git_repo.replace('.git', '') + '.atom'
elif 'dev.funkwhale.audio' in git_repo: # gitlab
atomFilePath = git_repo.replace('.git', '') + '/-/tags?format=atom'
else:
print('Unrecognized git repository: %s' % git_repo)
continue
role_name = role.split('/')[4]
if role_name == 'defaults':
role_name = role.split('/')[3]
role_name = role_name.removeprefix('matrix-bot-').removeprefix('matrix-bridge-').removeprefix('matrix-client-').removeprefix('matrix-')
if idx > 0:
# there is more than 1 project source code for this role
role_name += '-' + str(idx+1)
feeds[role_name] = {
'text': role_name,
'title': role_name,
'type': 'rss',
'htmlUrl': git_repo,
'xmlUrl': atomFilePath
}
feeds = {key: val for key, val in sorted(feeds.items(), key = lambda item: item[0])}
return feeds
def dump_opml_file_from_feeds(feeds):
tree = ET.ElementTree('tree')
opml = ET.Element('opml', {'version': '1.0'})
head = ET.SubElement(opml, 'head')
title = ET.SubElement(head, 'title')
title.text = 'Release feeds for roles'
body = ET.SubElement(opml, 'body')
for role, feed_dict in feeds.items():
outline = ET.SubElement(body, 'outline', feed_dict)
ET.indent(opml)
tree._setroot(opml)
file_name = 'releases.opml'
tree.write(file_name, encoding = 'UTF-8', xml_declaration = True)
print('Generated %s' % file_name)
if __name__ == '__main__':
file_paths = get_roles_files_from_dir(root_dir=args.root_dir)
break_on_missing = args.action == 'check'
git_repos = get_git_repos_from_files(file_paths=file_paths, break_on_missing_repos=break_on_missing)
feeds = format_feeds_from_git_repos(git_repos)
if args.action == 'dump':
dump_opml_file_from_feeds(feeds)