diff --git a/base/constants.py b/base/constants.py index ec717b89..95ed7a30 100644 --- a/base/constants.py +++ b/base/constants.py @@ -39,6 +39,7 @@ class PRICES: GOOGLE_CLOUD_BUCKET = 'elegansvariation.org' GOOGLE_CLOUD_PROJECT_ID = 'andersen-lab' +GOOGLE_CLOUD_LOCATION = 'us-central1' # WI Strain Info Dataset GOOGLE_SHEETS = {"orders": "1BCnmdJNRjQR3Bx8fMjD_IlTzmh3o7yj8ZQXTkk6tTXM", diff --git a/base/models.py b/base/models.py index 9ec61bb3..07935a6d 100644 --- a/base/models.py +++ b/base/models.py @@ -383,6 +383,33 @@ def save(self, *args, **kwargs): super(markdown_ds, self).save(*args, **kwargs) + +class h2calc_ds(datastore_model): + """ + The Heritability Calculation Task Model - for creating and retrieving + data and status information about a heritability calculation task + executed in Google Cloud Run + """ + kind = 'h2calc' + kind = '{}{}'.format(config['DS_PREFIX'], kind) + + + def __init__(self, *args, **kwargs): + super(h2calc_ds, self).__init__(*args, **kwargs) + + def query_by_username(self, username, keys_only=False): + filters = [('username', '=', username)] + results = query_item(self.kind, filters=filters, keys_only=keys_only) + return results + + def save(self, *args, **kwargs): + now = arrow.utcnow().datetime + self.modified_on = now + if not self._exists: + self.created_on = now + super(h2calc_ds, self).save(*args, **kwargs) + + class data_report_ds(datastore_model): """ The Data Report model - for creating and retrieving diff --git a/base/queue.yaml b/base/queue.yaml new file mode 100644 index 00000000..bfa0d41c --- /dev/null +++ b/base/queue.yaml @@ -0,0 +1,6 @@ +queue: +- name: heritability-calc + rate: 1/s + retry_parameters: + task_retry_limit: 2 + task_age_limit: 1d diff --git a/base/templates/data_v2.html b/base/templates/data_v2.html index 950abdc2..d286941a 100644 --- a/base/templates/data_v2.html +++ b/base/templates/data_v2.html @@ -1,19 +1,24 @@ -{% extends "_layouts/default.html" %} {% block custom_head %} +{% extends "_layouts/default.html" %} + +{% block custom_head %} + + {% endblock %} {% block content %} +
{# /Download Tab #} - {% endblock %} + {% block script %} + + {% endblock %} diff --git a/base/templates/tools/h2_result_list.html b/base/templates/tools/h2_result_list.html new file mode 100644 index 00000000..dfe9b55f --- /dev/null +++ b/base/templates/tools/h2_result_list.html @@ -0,0 +1,50 @@ +{% extends "_layouts/default.html" %} + +{% block content %} + +
+
+
+ +
+ +
+
+ + + + + + + + + + + + {% for item in items %} + + {% if item %} + + + + + + {% endif %} + + {% endfor %} + +
Label Data Hash Status Created:
{{ item.label }} {{ item.data_hash }} {{ item.status }} {{ item.created_on }} + + View + +
+
+
+ + +{% endblock %} diff --git a/base/templates/tools/heritability_calculator.html b/base/templates/tools/heritability_calculator.html index 12ad4bcf..7dde73a1 100644 --- a/base/templates/tools/heritability_calculator.html +++ b/base/templates/tools/heritability_calculator.html @@ -2,6 +2,7 @@ {% block custom_head %} +
+ +
-

This tool will calculate the broad-sense heritability for your trait of interest using a set of C. elegans wild isolates. The broad-sense heritability is the amount of trait variance that comes from genetic differences in the assayed group of strains. Generally, it is the ratio of genetic variance to total (genetic plus environmental) variance.

-

To obtain the best estimate of heritability, please measure a set of at least five wild strains in three independent assays. These assays should use different nematode growths, synchronizations, bacterial food preparations, and any other experimental condition. You should measure trait variance across as many different experimental conditions (in one block) as you would typically encounter in a large experiment.

-

Please organize your data in a long format, where each row is an independent observation of one strain in one trait. The columns of the data set should be:

-
  1. AssayNumber - a numeric indicator of independent assays.
  2. Strain - one of the CeNDR isotype reference strain names.
  3. @@ -40,38 +40,94 @@
  4. Value - the measured output of the trait (e.g. 297 for BroodSize).
-
Use example data
-

NA values will not be used in broad-sense heritability calculations.

- +

NA values will not be used in broad-sense heritability calculations.

+ +
{# /col-md-12 #} +
{# /row #} + + +{% if hide_form == True %} + +
+
{# /col-md-3 #} +
+ + + New Calculation + +
{# /col-md-3 #} +
+ + + My Reports + +
{# /col-md-3 #} +
{# /col-md-3 #} -
{# /col-md-8 #} {# /row #} +{% else %} +
-
+ +
+
{#/ col-md-3 #} + +
{# /col-md-3 #} + + {# /col-md-3 #} + + {# /col-md-3 #} -
-
- -
-
- - Prepare your data according to the column headers (described above). Data should be pasted in the table below. - -
-
-
-
- -
{# /col-md-12 #} -
{# /row #} - -
- -
{#/ col-md-12 #} +
{# /row #} + +
+
+
+ + +
+
+
+ +
+
+
+
+
+
+
+
+ Prepare your data according to the column headers (described below). Data should be pasted in the table below. +
+
{# /col-md-12 #} +
{# /row #} + +
{# /col-md-12 #} {# /row #} + + + + +{% endif %} + + {% endblock %} {% block script %} @@ -82,28 +138,62 @@ ]; var noNAs = 0; function dataValidator(instance, td, row, col, prop, value, cellProperties) { - Handsontable.renderers.TextRenderer.apply(this, arguments); - if (row === 0) { - cellProperties.readOnly = true; + Handsontable.renderers.TextRenderer.apply(this, arguments); + if (row === 0) { + cellProperties.readOnly = true; } - if (row === 0) { - td.style.fontWeight = 'bold'; - td.style.backgroundColor = '#EAEAEA'; - } + if (row === 0) { + td.style.fontWeight = 'bold'; + td.style.backgroundColor = '#EAEAEA'; + } - if (['NA',].indexOf(String(value).trim()) >= 0) { - td.style.background = '#FC6666'; - td.style.fontWeight = 'bold'; - } + if (['NA',].indexOf(String(value).trim()) >= 0) { + td.style.background = '#FC6666'; + td.style.fontWeight = 'bold'; + } if (duplicate_rows.indexOf(row) > -1) { - td.style.background = '#0CEF13'; + td.style.background = 'RED'; } + + if (trait_names.includes(value)) { + td.style.background = 'RED'; + } + + if (unknown_strains.includes(value)) { + td.style.background = 'RED'; + } } -var isValid = false +{% autoescape off %} +var strain_list = {{ strain_list }}; +{% endautoescape %} + +var isValid = false; var duplicate_rows = []; +var trait_names = []; +var unknown_strains = []; + + +var trim_data = function(data) { + for (let i = 0; i < data.length; i++) { + for (let j = 0; j < data[i].length; j++) { + if (data[i][j]) { + data[i][j] = data[i][j].replace("\r\n", "").replace("\r", "").replace("\n", ""); + data[i][j] = data[i][j].trim() + } + } + var stringData = JSON.stringify(data[i]) + if(JSON.stringify(['','','','','']) == stringData || "[null,null,null,null,null]" == stringData) { + data.splice(i, 1); + i--; + } + } + return data; +} + + var validate_data = function(data){ // Performs error checks: // 1) Duplicates @@ -111,36 +201,67 @@ // 3) Trait Name pass = true + data = trim_data(data); // Check for duplicates - dup_search = _.map(data, (item, idx) => item.toString()); - dup_values = _.chain(dup_search).groupBy().filter(x => x.length > 1 && x[0] !== ",,,,").flatten().uniq().value() - duplicate_rows = _.map(dup_search, (item, idx) => _.indexOf(dup_values, item) > -1 ? idx : null).filter(x => x !== null) + occurences = [] + duplicate_rows = [] + for (let i = 0; i < data.length; i++) { + item = JSON.stringify(data[i]); + if (occurences[item]) { + duplicate_rows.push(i) + } + occurences[item] = true; + }; // if dups are present alert user $("#duplicate_error").text("") if (duplicate_rows.length > 0) { - $("#duplicate_error").text("Please check the data because duplicate rows are present. The duplicate rows are shown in green.") + $("#duplicate_error").text("Please check the data because duplicate rows are present. The duplicate rows are shown in red.") pass = false } - // Count number of strains - n_strains = _.uniq(_.pluck(data, 1).filter((x, idx) => idx > 0 && x !== null)).length + // Strain names + var strain_names = _.uniq(_.pluck(data, 1).filter((x, idx) => idx > 0 && x !== null && x.length > 0)); + unknown_strains = _.difference(strain_names, strain_list); + $("#strain_name_error").text(""); + if (unknown_strains.length > 0) { + $("#strain_name_error").text("Please check the data - Some of the strains were not recognized") + pass = false + } - $("#strain_count_error").text("") + // Count number of strains + n_strains = strain_names.length; + $("#strain_count_error").text(""); if (n_strains < 5) { $("#strain_count_error").text("Please check the data because fewer than five strains are present. Please measure trait values for at least five wild strains in at least three independent assays.") pass = false } // Trait Name - trait_count = _.uniq(_.pluck(data, 2).filter((x, idx) => idx > 0 && x !== null)).length + trait_names = _.uniq(_.pluck(data, 2).filter((x, idx) => idx > 0 && x !== null && x.length > 0)); + trait_count = trait_names.length; $("#trait_name_error").text("") - if (trait_count > 1) { + if (trait_count >= 2) { + pass = false $("#trait_name_error").text("Please check the data. The TraitName has multiple unique values. Only data for a single trait allowed.") + var most_common_trait = _.chain(_.pluck(data, 2)).countBy().pairs().max(_.last).head().value(); + const index = trait_names.indexOf(most_common_trait); + if (index !== -1) { + trait_names.splice(index, 1); + } + } else { + trait_names = [] + } + + // Data label + label_len = $("#calcLabel").val().length + $("#calc_label_error").text("") + if (label_len == 0) { + $("#calc_label_error").text("Please include a brief description of the data.") pass = false } - + return pass } @@ -171,40 +292,26 @@ } }); + +$("#calcLabel").on('change', function() { + onFormChange() +}) + hot.addHook("afterChange", function() { - isValid = validate_data(hot.getData()); + onFormChange() +}) + +function onFormChange() { + isValid = validate_data(hot.getData()); hot.render(); if (isValid) { document.getElementById('hcalc').disabled = false; - - // Fetch dataset statistics - $.ajax({type: "POST", - url: "{{ url_for('heritability.check_data') }}", - data: JSON.stringify(hot.getData()), - contentType: "application/json; charset=utf-8", - dataType: 'json', - success:function(result) { - $("#trait_summary").html(` -
- Input data summary: -