diff --git a/base/constants.py b/base/constants.py
index ec717b89..95ed7a30 100644
--- a/base/constants.py
+++ b/base/constants.py
@@ -39,6 +39,7 @@ class PRICES:
GOOGLE_CLOUD_BUCKET = 'elegansvariation.org'
GOOGLE_CLOUD_PROJECT_ID = 'andersen-lab'
+GOOGLE_CLOUD_LOCATION = 'us-central1'
# WI Strain Info Dataset
GOOGLE_SHEETS = {"orders": "1BCnmdJNRjQR3Bx8fMjD_IlTzmh3o7yj8ZQXTkk6tTXM",
diff --git a/base/models.py b/base/models.py
index 9ec61bb3..07935a6d 100644
--- a/base/models.py
+++ b/base/models.py
@@ -383,6 +383,33 @@ def save(self, *args, **kwargs):
super(markdown_ds, self).save(*args, **kwargs)
+
+class h2calc_ds(datastore_model):
+ """
+ The Heritability Calculation Task Model - for creating and retrieving
+ data and status information about a heritability calculation task
+ executed in Google Cloud Run
+ """
+ kind = 'h2calc'
+ kind = '{}{}'.format(config['DS_PREFIX'], kind)
+
+
+ def __init__(self, *args, **kwargs):
+ super(h2calc_ds, self).__init__(*args, **kwargs)
+
+ def query_by_username(self, username, keys_only=False):
+ filters = [('username', '=', username)]
+ results = query_item(self.kind, filters=filters, keys_only=keys_only)
+ return results
+
+ def save(self, *args, **kwargs):
+ now = arrow.utcnow().datetime
+ self.modified_on = now
+ if not self._exists:
+ self.created_on = now
+ super(h2calc_ds, self).save(*args, **kwargs)
+
+
class data_report_ds(datastore_model):
"""
The Data Report model - for creating and retrieving
diff --git a/base/queue.yaml b/base/queue.yaml
new file mode 100644
index 00000000..bfa0d41c
--- /dev/null
+++ b/base/queue.yaml
@@ -0,0 +1,6 @@
+queue:
+- name: heritability-calc
+ rate: 1/s
+ retry_parameters:
+ task_retry_limit: 2
+ task_age_limit: 1d
diff --git a/base/templates/data_v2.html b/base/templates/data_v2.html
index 950abdc2..d286941a 100644
--- a/base/templates/data_v2.html
+++ b/base/templates/data_v2.html
@@ -1,19 +1,24 @@
-{% extends "_layouts/default.html" %} {% block custom_head %}
+{% extends "_layouts/default.html" %}
+
+{% block custom_head %}
+
+
{% endblock %}
{% block content %}
+
@@ -256,9 +261,10 @@ Datasets
{# /Download Tab #}
-
{% endblock %}
+
{% block script %}
+
+
{% endblock %}
diff --git a/base/templates/tools/h2_result_list.html b/base/templates/tools/h2_result_list.html
new file mode 100644
index 00000000..dfe9b55f
--- /dev/null
+++ b/base/templates/tools/h2_result_list.html
@@ -0,0 +1,50 @@
+{% extends "_layouts/default.html" %}
+
+{% block content %}
+
+
+
+
+
+
+
+
+
+
+ {% for item in items %}
+
+ {% if item %}
+ {{ item.label }}
+ {{ item.data_hash }}
+ {{ item.status }}
+ {{ item.created_on }}
+
+
+ View
+
+
+ {% endif %}
+
+ {% endfor %}
+
+
+
+
+
+
+{% endblock %}
diff --git a/base/templates/tools/heritability_calculator.html b/base/templates/tools/heritability_calculator.html
index 12ad4bcf..7dde73a1 100644
--- a/base/templates/tools/heritability_calculator.html
+++ b/base/templates/tools/heritability_calculator.html
@@ -2,6 +2,7 @@
{% block custom_head %}
+
+
+
-
This tool will calculate the broad-sense heritability for your trait of interest using a set of C. elegans wild
isolates. The broad-sense heritability is the amount of trait variance that comes from genetic differences in the
assayed group of strains. Generally, it is the ratio of genetic variance to total (genetic plus environmental)
variance.
-
To obtain the best estimate of heritability, please measure a set of at least five wild strains in three
independent assays. These assays should use different nematode growths, synchronizations, bacterial food
preparations, and any other experimental condition. You should measure trait variance across as many
different experimental conditions (in one block) as you would typically encounter in a large experiment.
-
Please organize your data in a long format, where each row is an independent observation of one strain in one
trait. The columns of the data set should be:
-
AssayNumber - a numeric indicator of independent assays.
Strain - one of the CeNDR isotype reference strain names.
@@ -40,38 +40,94 @@
Value - the measured output of the trait (e.g. 297 for BroodSize).
-
Use example data
-
NA values will not be used in broad-sense heritability calculations.
-
+
NA values will not be used in broad-sense heritability calculations.
+
+
{# /col-md-12 #}
+
{# /row #}
+
+
+{% if hide_form == True %}
+
+
+
{# /col-md-3 #}
+
{# /col-md-3 #}
+
{# /col-md-3 #}
+
{# /col-md-3 #}
-
{# /col-md-8 #}
{# /row #}
+{% else %}
+
-
+
+
+
{#/ col-md-3 #}
+
+
{# /col-md-3 #}
+
+
{# /col-md-3 #}
+
+
{# /col-md-3 #}
-
-
-
-
-
-
-
Prepare your data according to the column headers (described above). Data should be pasted in the table below.
-
-
-
-
-
-
-
{# /col-md-12 #}
-
{# /row #}
-
-
- Calculate Heritability
-
{#/ col-md-12 #}
+
{# /row #}
+
+
+
+
+
+
+
+
+
+
+
+
+
Prepare your data according to the column headers (described below). Data should be pasted in the table below.
+
+
{# /col-md-12 #}
+
{# /row #}
+
+
{# /col-md-12 #}
{# /row #}
+
+
+
+
+{% endif %}
+
+
{% endblock %}
{% block script %}
@@ -82,28 +138,62 @@
];
var noNAs = 0;
function dataValidator(instance, td, row, col, prop, value, cellProperties) {
- Handsontable.renderers.TextRenderer.apply(this, arguments);
- if (row === 0) {
- cellProperties.readOnly = true;
+ Handsontable.renderers.TextRenderer.apply(this, arguments);
+ if (row === 0) {
+ cellProperties.readOnly = true;
}
- if (row === 0) {
- td.style.fontWeight = 'bold';
- td.style.backgroundColor = '#EAEAEA';
- }
+ if (row === 0) {
+ td.style.fontWeight = 'bold';
+ td.style.backgroundColor = '#EAEAEA';
+ }
- if (['NA',].indexOf(String(value).trim()) >= 0) {
- td.style.background = '#FC6666';
- td.style.fontWeight = 'bold';
- }
+ if (['NA',].indexOf(String(value).trim()) >= 0) {
+ td.style.background = '#FC6666';
+ td.style.fontWeight = 'bold';
+ }
if (duplicate_rows.indexOf(row) > -1) {
- td.style.background = '#0CEF13';
+ td.style.background = 'RED';
}
+
+ if (trait_names.includes(value)) {
+ td.style.background = 'RED';
+ }
+
+ if (unknown_strains.includes(value)) {
+ td.style.background = 'RED';
+ }
}
-var isValid = false
+{% autoescape off %}
+var strain_list = {{ strain_list }};
+{% endautoescape %}
+
+var isValid = false;
var duplicate_rows = [];
+var trait_names = [];
+var unknown_strains = [];
+
+
+var trim_data = function(data) {
+ for (let i = 0; i < data.length; i++) {
+ for (let j = 0; j < data[i].length; j++) {
+ if (data[i][j]) {
+ data[i][j] = data[i][j].replace("\r\n", "").replace("\r", "").replace("\n", "");
+ data[i][j] = data[i][j].trim()
+ }
+ }
+ var stringData = JSON.stringify(data[i])
+ if(JSON.stringify(['','','','','']) == stringData || "[null,null,null,null,null]" == stringData) {
+ data.splice(i, 1);
+ i--;
+ }
+ }
+ return data;
+}
+
+
var validate_data = function(data){
// Performs error checks:
// 1) Duplicates
@@ -111,36 +201,67 @@
// 3) Trait Name
pass = true
+ data = trim_data(data);
// Check for duplicates
- dup_search = _.map(data, (item, idx) => item.toString());
- dup_values = _.chain(dup_search).groupBy().filter(x => x.length > 1 && x[0] !== ",,,,").flatten().uniq().value()
- duplicate_rows = _.map(dup_search, (item, idx) => _.indexOf(dup_values, item) > -1 ? idx : null).filter(x => x !== null)
+ occurences = []
+ duplicate_rows = []
+ for (let i = 0; i < data.length; i++) {
+ item = JSON.stringify(data[i]);
+ if (occurences[item]) {
+ duplicate_rows.push(i)
+ }
+ occurences[item] = true;
+ };
// if dups are present alert user
$("#duplicate_error").text("")
if (duplicate_rows.length > 0) {
- $("#duplicate_error").text("Please check the data because duplicate rows are present. The duplicate rows are shown in green.")
+ $("#duplicate_error").text("Please check the data because duplicate rows are present. The duplicate rows are shown in red.")
pass = false
}
- // Count number of strains
- n_strains = _.uniq(_.pluck(data, 1).filter((x, idx) => idx > 0 && x !== null)).length
+ // Strain names
+ var strain_names = _.uniq(_.pluck(data, 1).filter((x, idx) => idx > 0 && x !== null && x.length > 0));
+ unknown_strains = _.difference(strain_names, strain_list);
+ $("#strain_name_error").text("");
+ if (unknown_strains.length > 0) {
+ $("#strain_name_error").text("Please check the data - Some of the strains were not recognized")
+ pass = false
+ }
- $("#strain_count_error").text("")
+ // Count number of strains
+ n_strains = strain_names.length;
+ $("#strain_count_error").text("");
if (n_strains < 5) {
$("#strain_count_error").text("Please check the data because fewer than five strains are present. Please measure trait values for at least five wild strains in at least three independent assays.")
pass = false
}
// Trait Name
- trait_count = _.uniq(_.pluck(data, 2).filter((x, idx) => idx > 0 && x !== null)).length
+ trait_names = _.uniq(_.pluck(data, 2).filter((x, idx) => idx > 0 && x !== null && x.length > 0));
+ trait_count = trait_names.length;
$("#trait_name_error").text("")
- if (trait_count > 1) {
+ if (trait_count >= 2) {
+ pass = false
$("#trait_name_error").text("Please check the data. The TraitName has multiple unique values. Only data for a single trait allowed.")
+ var most_common_trait = _.chain(_.pluck(data, 2)).countBy().pairs().max(_.last).head().value();
+ const index = trait_names.indexOf(most_common_trait);
+ if (index !== -1) {
+ trait_names.splice(index, 1);
+ }
+ } else {
+ trait_names = []
+ }
+
+ // Data label
+ label_len = $("#calcLabel").val().length
+ $("#calc_label_error").text("")
+ if (label_len == 0) {
+ $("#calc_label_error").text("Please include a brief description of the data.")
pass = false
}
-
+
return pass
}
@@ -171,40 +292,26 @@
}
});
+
+$("#calcLabel").on('change', function() {
+ onFormChange()
+})
+
hot.addHook("afterChange", function() {
- isValid = validate_data(hot.getData());
+ onFormChange()
+})
+
+function onFormChange() {
+ isValid = validate_data(hot.getData());
hot.render();
if (isValid) {
document.getElementById('hcalc').disabled = false;
-
- // Fetch dataset statistics
- $.ajax({type: "POST",
- url: "{{ url_for('heritability.check_data') }}",
- data: JSON.stringify(hot.getData()),
- contentType: "application/json; charset=utf-8",
- dataType: 'json',
- success:function(result) {
- $("#trait_summary").html(`
-
- Input data summary:
-
- Minimum: ${result['minimum']}
- Maximum: ${result['maximum']}
- 25% Quartile: ${result['25']}
- 50% Quartile: ${result['50']}
- 75% Quartile: ${result['75']}
- Variance: ${result['variance']}
- `)
- }
- });
-
} else {
$("#trait_summary").html("")
document.getElementById('hcalc').disabled = true;
}
-
-})
+}
// Enable setting example data
$("#set-example").on('click', function() {
@@ -225,16 +332,24 @@
// submit result
$("#hcalc").on("click", function(e) {
+ $("#hcalc").addClass("disabled")
if (isValid) {
+ var data = new FormData($('form#form-submit')[0]);
+ data.append('table_data', JSON.stringify(hot.getData()));
+ data.set('label', $("#calcLabel").val());
$.ajax({
type: "POST",
+ processData: false,
+ contentType: false,
+ dataType: 'json',
url: "{{ url_for('heritability.submit_h2') }}",
- data: JSON.stringify(hot.getData()),
- contentType: "application/json; charset=utf-8",
- dataType: 'json',
- success:function(result){
- window.location = `heritability/h2/${result.data_hash}`
- }
+ data: data,
+ success:function(result) {
+ window.location = `../heritability/h2/${result.id}`
+ },
+ error:function(error) {
+ $("#hcalc").removeClass("disabled")
+ }
})
}
});
diff --git a/base/templates/tools/heritability_results.html b/base/templates/tools/heritability_results.html
index 5293c734..b3d9e4fc 100644
--- a/base/templates/tools/heritability_results.html
+++ b/base/templates/tools/heritability_results.html
@@ -1,8 +1,8 @@
{% extends "_layouts/default.html" %}
{% block custom_head %}
-
+
@@ -11,6 +11,7 @@
+