Skip to content

Add pagination and PEP-8 improvements. #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LondonAppDev wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Add pagination and PEP-8 improvements. #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c5304cf
PEP-8 fixes.
Dec 8, 2015
b9bb838
PEP-8 improvements.
Dec 8, 2015
27074ad
Changes.
Dec 8, 2015
81862d8
Pagination working.
Dec 9, 2015
75f284f
Removed commented out code.
Dec 9, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ ckanext-report.notes.dataset = ' '.join(('Unpublished' if asbool(pkg.extras.get(

A report has three key elements:

1. Report Code - a python function that produces the report.
1. Report Code - a python function that produces the report.
2. Template - HTML for displaying the report data.
3. Registration - containing the configuration of the report.

Expand All @@ -113,7 +113,7 @@ The returned data should be a dict like this:
'average_tags_per_package': 3.5,
}
```

There should be a `table` with the main body of the data, and any other totals or incidental pieces of data.

Note: the table is required because of the CSV download facility, and CSV demands a table. (The CSV download only includes the table, ignoring any other values in the data.) Although the data has to essentially be stored as a table, you do have the option to display it differently in the web page by using a clever template.
Expand Down Expand Up @@ -242,6 +242,7 @@ Info dict spec:
* option_defaults - dict of ALL option names and their default values. Use ckan.common.OrderedDict. If there are no options, you can return None.
* option_combinations - function returning a list of all the options combinations (reports for these combinations are generated by default). If there are no options, return None.
* authorize (optional) - a function that says if the user is allowed to view the report. Takes params: (user_object, options_dict) and should return a boolean - if the user is authorized or not. The default is that anyone can see reports.
* paginate_by (optional) - number of items to paginate the report view by. If excluded, pagination will be disabled.

Finally we need to define the function that returns the option_combinations:
```python
Expand Down
91 changes: 69 additions & 22 deletions ckanext/report/controllers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from ckan.lib.render import TemplateNotFound
from ckan.common import OrderedDict


log = __import__('logging').getLogger(__name__)

c = t.c
Expand All @@ -33,11 +32,11 @@ def view(self, report_name, organization=None, refresh=False):

# ensure correct url is being used
if 'organization' in t.request.environ['pylons.routes_dict'] and \
'organization' not in report['option_defaults']:
'organization' not in report['option_defaults']:
t.redirect_to(helpers.relative_url_for(organization=None))
elif 'organization' not in t.request.environ['pylons.routes_dict'] and\
'organization' in report['option_defaults'] and \
report['option_defaults']['organization']:
report['option_defaults']['organization']:
org = report['option_defaults']['organization']
t.redirect_to(helpers.relative_url_for(organization=org))
if 'organization' in t.request.params:
Expand All @@ -46,7 +45,11 @@ def view(self, report_name, organization=None, refresh=False):
t.redirect_to(helpers.relative_url_for())

# options
options = Report.add_defaults_to_options(t.request.params, report['option_defaults'])
options = Report.add_defaults_to_options(
t.request.params,
report['option_defaults']
)

option_display_params = {}
if 'format' in options:
format = options.pop('format')
Expand All @@ -59,19 +62,22 @@ def view(self, report_name, organization=None, refresh=False):
for option in options:
if option not in report['option_defaults']:
# e.g. 'refresh' param
log.warn('Not displaying report option HTML for param %s as option not recognized')
log.warn('Not displaying report option HTML for param %s '
'as option not recognized')
continue
option_display_params = {'value': options[option],
'default': report['option_defaults'][option]}
option_display_params = {
'value': options[option],
'default': report['option_defaults'][option]
}
try:
options_html[option] = \
t.render_snippet('report/option_%s.html' % option,
data=option_display_params)
except TemplateNotFound:
log.warn('Not displaying report option HTML for param %s as no template found')
log.warn('Not displaying report option HTML for param %s as '
'no template found')
continue


# Alternative way to refresh the cache - not in the UI, but is
# handy for testing
try:
Expand All @@ -87,35 +93,45 @@ def view(self, report_name, organization=None, refresh=False):

if refresh:
try:
t.get_action('report_refresh')({}, {'id': report_name, 'options': options})
t.get_action('report_refresh')(
{}, {'id': report_name, 'options': options})
except t.NotAuthorized:
t.abort(401)
t.abort(401)
# Don't want the refresh=1 in the url once it is done
t.redirect_to(helpers.relative_url_for(refresh=None))

# Check for any options not allowed by the report
for key in options:
if key not in report['option_defaults']:
if key not in report['option_defaults'] and \
key not in ['page', 'limit']:
t.abort(400, 'Option not allowed by report: %s' % key)

try:
data, report_date = t.get_action('report_data_get')({}, {'id': report_name, 'options': options})
data, report_date = t.get_action('report_data_get')(
{},
{'id': report_name, 'options': options}
)
except t.ObjectNotFound:
t.abort(404)
except t.NotAuthorized:
t.abort(401)

if format and format != 'html':
ensure_data_is_dicts(data)
anonymise_user_names(data, organization=options.get('organization'))
anonymise_user_names(
data, organization=options.get('organization'))
if format == 'csv':
try:
key = t.get_action('report_key_get')({}, {'id': report_name, 'options': options})
key = t.get_action('report_key_get')(
{},
{'id': report_name, 'options': options}
)
except t.NotAuthorized:
t.abort(401)
filename = 'report_%s.csv' % key
t.response.headers['Content-Type'] = 'application/csv'
t.response.headers['Content-Disposition'] = str('attachment; filename=%s' % (filename))
t.response.headers['Content-Disposition'] = \
str('attachment; filename=%s' % (filename))
return make_csv_from_dicts(data['table'])
elif format == 'json':
t.response.headers['Content-Type'] = 'application/json'
Expand All @@ -126,15 +142,46 @@ def view(self, report_name, organization=None, refresh=False):

are_some_results = bool(data['table'] if 'table' in data
else data)

# Pagination
paginate_by = report.get('paginate_by')
total_results = len(data['table'])
if paginate_by:
page = int(options.get('page', 1))
limit = int(options.get('limit', paginate_by))
min_val = (page * limit) - limit
max_val = (page * limit)
data['table'] = data['table'][min_val:max_val]
options['page'] = page
options['limit'] = limit
else:
page = None
limit = None

pagination = {
'total': total_results,
'page': page,
'limit': limit
}

# A couple of context variables for legacy genshi reports
c.data = data
c.options = options
return t.render('report/view.html', extra_vars={
'report': report, 'report_name': report_name, 'data': data,
'report_date': report_date, 'options': options,
'options_html': options_html,
'report_template': report['template'],
'are_some_results': are_some_results})

return t.render(
'report/view.html',
extra_vars={
'report': report,
'report_name': report_name,
'data': data,
'report_date': report_date,
'options': options,
'options_html': options_html,
'report_template': report['template'],
'are_some_results': are_some_results,
'pagination': pagination
}
)


def make_csv_from_dicts(rows):
Expand Down
5 changes: 3 additions & 2 deletions ckanext/report/logic/action/get.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,13 @@ def report_data_get(context=None, data_dict=None):
:returns: A list containing the data and the date on which it was created
:rtype: list
"""

logic.check_access('report_data_get', context, data_dict)

id = logic.get_or_bust(data_dict, 'id')
report_id = logic.get_or_bust(data_dict, 'id')
options = data_dict.get('options', {})

report = ReportRegistry.instance().get_report(id)
report = ReportRegistry.instance().get_report(report_id)

data, date = report.get_fresh_report(**options)

Expand Down
16 changes: 9 additions & 7 deletions ckanext/report/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from ckan import model
from ckan.lib.helpers import OrderedDict


log = logging.getLogger(__name__)

__all__ = ['DataCache', 'data_cache_table', 'init_tables']
Expand Down Expand Up @@ -39,7 +40,7 @@ class DataCache(object):
This model makes no assumptions on what is stored, and so it is up to the
producer and consumer to agree on a format for the value stored. It is
suggested that for data that is not a basic type (int, string etc) that
json is used as decoding of JSON will still be a lot faster than performing
JSON is used as decoding of JSON will still be a lot faster than performing
the initial queries.

Example usage:
Expand Down Expand Up @@ -71,35 +72,35 @@ def get(cls, object_id, key, convert_json=False, max_age=None):
Retrieves the value and date that it was written if the record with
object_id/key exists. If not it will return None/None.
"""

item = model.Session.query(cls)\
.filter(cls.key == key)\
.filter(cls.object_id == object_id)\
.first()
if not item:
#log.debug('Does not exist in cache: %s/%s', object_id, key)
return (None, None)

if max_age:
age = datetime.datetime.now() - item.created
if age > max_age:
log.debug('Cache not returned - it is older than requested %s/%s %r > %r',
log.debug('Cache not returned - it is older than requested '
'%s/%s %r > %r',
object_id, key, age, max_age)
return (None, None)

value = item.value
if convert_json:
# Use OrderedDict instead of dict, so that the order of the columns
# in the data is preserved from the data when it was written (assuming
# it was written as an OrderedDict in the report's code).
# in the data is preserved from the data when it was written
# (assuming it was written as an OrderedDict in the report's code).
try:
# Python 2.7's json library has object_pairs_hook
import json
value = json.loads(value, object_pairs_hook=OrderedDict)
except TypeError: # Untested
except TypeError: # Untested
# Python 2.4-2.6
import simplejson as json
value = json.loads(value, object_pairs_hook=OrderedDict)
#log.debug('Cache load: %s/%s "%s"...', object_id, key, repr(value)[:40])
return value, item.created

@classmethod
Expand Down Expand Up @@ -133,5 +134,6 @@ def set(cls, object_id, key, value, convert_json=False):

mapper(DataCache, data_cache_table)


def init_tables():
metadata.create_all(model.meta.engine)
2 changes: 1 addition & 1 deletion ckanext/report/plugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import ckanext.report.logic.auth.get as auth_get
import ckanext.report.logic.auth.update as auth_update


class ReportPlugin(p.SingletonPlugin):
p.implements(p.IRoutes, inherit=True)
p.implements(p.IConfigurer)
Expand Down Expand Up @@ -72,4 +73,3 @@ class TaglessReportPlugin(p.SingletonPlugin):
def register_reports(self):
import reports
return [reports.tagless_report_info]

Loading