-
Notifications
You must be signed in to change notification settings - Fork 6
/
Copy pathgenerate_html.py
executable file
·280 lines (262 loc) · 10.3 KB
/
generate_html.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
#!/usr/bin/env python3
import argparse
import collections
import pathlib
from jinja2 import Environment, FileSystemLoader
from ros_metrics import analytics, answers, packages, repos, wiki
from ros_metrics import charts, tables, scholar
from ros_metrics.constants import distros, os_list
from ros_metrics.metric_db import MetricDB
from tqdm import tqdm
parser = argparse.ArgumentParser()
parser.add_argument('-f', '--filter')
args = parser.parse_args()
OUTPUT_FOLDER = pathlib.Path('docs')
STRUCTURE = [
{'name': 'Users',
'filename': 'index.html',
'chart': charts.get_users_plot,
'caption': 'A collection of different metrics for measuring the number of users in the ROS community.'
},
{'name': 'Packages',
'subpages': [
{'name': 'Top',
'template': 'top.html',
'tops': packages.top_report,
'caption': 'Most downloaded packages via packages.ros.org, '
'and the most downloaded packages introduced in each year.'},
{'name': 'ROS Distro',
'chart': charts.get_package_ratio_chart,
'args': ['rosdistro', 'ROS Distro Usage by packages.ros.org traffic', distros],
'caption': 'Relative usage of each distro based on downloads from packages.ros.org. '
'Note that data after late 2018 is not complete.'
},
{'name': 'Architecture',
'chart': charts.get_package_ratio_chart,
'args': ['arch', 'Architecture Usage by packages.ros.org traffic'],
'caption': 'Chip architecture usage by packages.ros.org downloads.'
},
{'name': 'AptRepo',
'chart': charts.get_package_ratio_chart,
'args': ['apt_repo', 'Apt-repo Usage by packages.ros.org traffic'],
'caption': 'Apt-repo usage based on packages.ros.org downloads.'
},
{'name': 'Linux',
'chart': charts.get_package_ratio_chart,
'args': ['distro', 'Linux Distro Usage by packages.ros.org traffic', os_list, 0.5],
'caption': 'Linux distro usage based on packages.ros.org downloads.'
},
# {'name': 'Library',
# 'chart': get_package_ratio_chart(dbs['packages'], 'library')
# },
{'name': 'Country',
'template': 'zing_chart.html',
'chart': charts.get_package_country_chart,
'caption': 'Top ROS-using countries based on packages.ros.org downloads.'
},
{'name': 'Tarballs',
'chart': charts.get_tarball_chart,
'caption': 'Number of tarball/zip file downloads from GitHub Release pages per ROS2 release, '
'broken down by OS.'
}
# {'name': 'OS',
# 'chart': charts.get_package_os_chart(dbs['packages'])
# }
]
},
{'name': 'ROS Distro',
'subpages': [
# {'name': 'Classification',
# 'chart': charts.get_rosdistro_plot},
{'name': 'ROS Distro',
'chart': charts.get_rosdistro_distros,
'caption': 'Relative maintenance of ROS distros by commits to ros/rosdistro.'},
{'name': 'Verbs',
'chart': charts.get_rosdistro_verbs,
'caption': 'Commits to ros/rosdistro by action-type'},
{'name': 'Version',
'chart': charts.get_rosdistro_versions,
'caption': 'Types of version bumps, by ros/rosdistro commits.'},
{'name': 'Deps',
'chart': charts.get_rosdistro_deps,
'caption': 'Types of dependencies added, measured by commits to ros/rosdistro.'},
{'name': 'Number of Repos',
'chart': charts.get_rosdistro_repos,
'caption': 'Number of repos in ros/rosdistro'
},
]
},
{'name': 'Repos',
'subpages': [
{'name': 'Table',
'template': 'table.html',
'table': tables.github_repos_report,
'caption': 'Github repos listed in ros/rosdistro, ranked by number of forks/stars/subscriptions'
},
{'name': 'Total Backlog',
'chart': charts.get_repo_issues},
{'name': 'Issues and PRs',
'chart': charts.get_ticket_totals},
{'name': 'NumCommits',
'chart': charts.get_commits_chart
},
]
},
{'name': 'Answers',
'subpages': [
{'name': 'Questions',
'chart': charts.get_questions_plot,
'caption': 'Total number of questions, answers, questions with an accepted answer on Q and A websites, '
'and the percent of questions with an accepted answer.'},
{'name': 'Karma',
'chart': charts.get_karma_chart,
'caption': 'Distribution of karma among answers.ros.org users.',
},
{'name': 'ROS Distro',
'chart': charts.get_answers_distro_chart,
'caption': 'Relative usage of ROS distros by question tags on answers on Q and A websites.'
},
{'name': 'Top Answerers',
'template': 'top.html',
'tops': answers.get_top_users,
'caption': 'Top answerers of questions on answers.ros.org, overall and by year.'
},
{'name': 'Interesting Qs',
'template': 'top.html',
'tops': answers.get_top_questions,
'caption': 'Various "interesting" statistics'
}
]
},
{'name': 'Analytics',
'subpages': [
{'name': 'Pageviews',
'chart': charts.get_analytics_totals_chart,
'caption': 'Pageviews to key ROS websites as reported by Google Analytics '
'(and Apache logs for packages.ros.org)'
},
{'name': 'Users',
'chart': charts.get_analytics_totals_chart,
'args': ['users', False],
'caption': 'Users to key ROS websites as reported by Google Analytics'
},
{'name': 'Sessions',
'chart': charts.get_analytics_totals_chart,
'args': ['sessions', False],
'caption': 'Number of sessions to key ROS websites as reported by Google Analytics'
},
# {'name': 'By Country',
# 'chart': charts.get_analytics_country_chart(dbs['analytics'])
# }
]
},
{'name': 'Wiki',
'subpages': [
{'name': 'Activity',
'chart': charts.get_wiki_chart,
'caption': 'Total number of pages and edits for wiki.ros.org'
},
{'name': 'TopPages',
'template': 'top.html',
'tops': analytics.top_wiki_report,
'caption': 'Top wiki pages, as measured by Google Analytics, both overall and by year created.'
},
{'name': 'Interesting',
'template': 'top.html',
'tops': wiki.interesting_wiki_report,
'caption': 'Interesting factoids for the wiki.'
},
]},
{'name': 'Misc',
'subpages': [
{'name': 'Emails',
'chart': charts.get_emails_plot,
'caption': 'Number of posts/threads on the two email platforms. '
'Note that answers.ros.org was introduced in early 2011.'
},
{'name': 'Citations',
'chart': charts.get_scholar_plot,
'template': 'scholar.html',
'caption': scholar.generate_caption(),
'replacements': {
'"y_display"': 'y_display'
}
}
]
},
]
j2_env = Environment(loader=FileSystemLoader('viz/templates'))
OUTPUT_FOLDER.mkdir(exist_ok=True)
menu = collections.OrderedDict()
for blob in STRUCTURE:
name = blob['name']
if 'subpages' not in blob:
if 'filename' not in blob:
filename = name.lower() + '.html'
blob['filename'] = filename
else:
filename = blob['filename']
menu[filename] = name
else:
submenu = collections.OrderedDict()
for subpage in blob['subpages']:
if 'filename' not in subpage:
filename = name.lower().replace(' ', '') + '_' + subpage['name'].lower().replace(' ', '') + '.html'
subpage['filename'] = filename
else:
filename = subpage['filename']
submenu[filename] = subpage['name']
blob['submenu'] = submenu
chosen = blob['subpages'][0]['filename']
menu[chosen] = blob['name']
blob['filename'] = chosen
for blob in STRUCTURE:
level1 = blob['filename']
if 'subpages' not in blob:
template = j2_env.get_template(blob.get('template', 'basic_chart.html'))
blob['level1'] = level1
blob['menu'] = menu
if args.filter and args.filter not in level1:
continue
print(level1)
for key, value in blob.items():
if hasattr(value, '__call__'):
blob[key] = value(*blob.get('args', []))
with open(OUTPUT_FOLDER / level1, 'w') as f:
f.write(template.render(**blob))
else:
for subpage in blob['subpages']:
template = j2_env.get_template(subpage.get('template', 'basic_chart.html'))
level2 = subpage['filename']
if args.filter and args.filter not in level2:
continue
print(level2)
for key, value in subpage.items():
if hasattr(value, '__call__'):
subpage[key] = value(*subpage.get('args', []))
subpage['level1'] = level1
subpage['level2'] = level2
subpage['menu'] = menu
subpage['submenu'] = blob['submenu']
html_contents = template.render(**subpage)
for k, v in subpage.get('replacements', {}).items():
html_contents = html_contents.replace(k, v)
with open(OUTPUT_FOLDER / level2, 'w') as f:
f.write(html_contents)
if args.filter and args.filter != 'repos':
exit(0)
rosdistro_db = MetricDB('rosdistro')
repos_db = MetricDB('repos')
github_repos = repos.get_github_repos(rosdistro_db)
REPOS_FOLDER = OUTPUT_FOLDER / 'repos'
REPOS_FOLDER.mkdir(exist_ok=True)
for repo_id, repo_dict in tqdm(github_repos.items()):
name = '{org}/{repo}'.format(**repo_dict)
template = j2_env.get_template('basic_chart.html')
filename = '{org}_{repo}.html'.format(**repo_dict)
repo_page = {'name': name, 'chart': charts.get_repo_issues(repos_db, name, repo_id)}
repo_page['level1'] = 'repos.html'
repo_page['menu'] = menu
repo_page['prefix'] = '../'
with open(REPOS_FOLDER / filename, 'w') as f:
f.write(template.render(**repo_page))