From b023a80e0223741cabc7b805f2f8328049246241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6k=C3=A7en=20Eraslan?= Date: Mon, 7 Oct 2019 14:31:34 -0400 Subject: [PATCH 1/6] Fix versus_rest link --- docs/tutorials.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 95ee021..25c5ee9 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -32,7 +32,8 @@ Multiple tests per gene How to perform pairwise tests `pairwise `__. -How to perform group versus rest tests `versus_rest `__. +How to perform group versus rest tests `versus_rest +`__. Gene set enrichment: enrich From c27eead881f9857fafe3da82e8bbc8d76d618999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6k=C3=A7en=20Eraslan?= Date: Mon, 7 Oct 2019 14:33:44 -0400 Subject: [PATCH 2/6] Remove whitespace --- docs/tutorials.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 25c5ee9..e7b79e0 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -32,8 +32,7 @@ Multiple tests per gene How to perform pairwise tests `pairwise `__. -How to perform group versus rest tests `versus_rest -`__. +How to perform group versus rest tests `versus_rest `__. Gene set enrichment: enrich From efb773228ca887ccc2e071d36d08a4662642d558 Mon Sep 17 00:00:00 2001 From: Mario Picciani Date: Mon, 13 Jan 2020 16:13:54 +0100 Subject: [PATCH 3/6] bugfix: rm batch_size from constructor args in tf2 --- diffxpy/testing/tests.py | 2 +- diffxpy/unit_test/test_fit.py | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/diffxpy/testing/tests.py b/diffxpy/testing/tests.py index 1bdb85c..504be1e 100644 --- a/diffxpy/testing/tests.py +++ b/diffxpy/testing/tests.py @@ -162,7 +162,7 @@ def _fit( constructor_args = {} if quick_scale is not None: constructor_args["quick_scale"] = quick_scale - if batch_size is not None: + if batch_size is not None and backend is not "tf2": constructor_args["batch_size"] = batch_size # Backend-specific constructor arguments: if backend.lower() in ["tf1"]: diff --git a/diffxpy/unit_test/test_fit.py b/diffxpy/unit_test/test_fit.py index 27cb421..f746dae 100644 --- a/diffxpy/unit_test/test_fit.py +++ b/diffxpy/unit_test/test_fit.py @@ -17,7 +17,7 @@ def _test_model_fit( """ Test if de.wald() generates a uniform p-value distribution if it is given data simulated based on the null model. Returns the p-value - of the two-side Kolmgorov-Smirnov test for equality of the observed + of the two-side Kolmgorov-Smirnov test for equality of the observed p-value distribution and a uniform distribution. :param n_cells: Number of cells to simulate (number of observations per test). @@ -34,7 +34,7 @@ def _test_model_fit( raise ValueError("noise model %s not recognized" % noise_model) sim = Simulator(num_observations=n_cells, num_features=n_genes) - sim.generate_sample_description(num_batches=0, num_conditions=0) + sim.generate_sample_description(num_batches=1, num_conditions=1) sim.generate_params(rand_fn_scale=rand_fn_scale) sim.generate_data() @@ -77,7 +77,7 @@ def _test_model_fit_partition( raise ValueError("noise model %s not recognized" % noise_model) sim = Simulator(num_observations=n_cells, num_features=n_genes) - sim.generate_sample_description(num_batches=0, num_conditions=0) + sim.generate_sample_description(num_batches=1, num_conditions=1) sim.generate_params(rand_fn_scale=rand_fn_scale) sim.generate_data() @@ -121,7 +121,7 @@ def _test_residuals_fit( raise ValueError("noise model %s not recognized" % noise_model) sim = Simulator(num_observations=n_cells, num_features=n_genes) - sim.generate_sample_description(num_batches=0, num_conditions=0) + sim.generate_sample_description(num_batches=1, num_conditions=1) sim.generate() random_sample_description = pd.DataFrame({ @@ -209,23 +209,23 @@ def test_residuals_fit( noise_model="nb" ) - +""" class TestFitNorm(_TestFit, unittest.TestCase): - """ + Normal noise model unit tests that tests whether model fit relay works. - """ + def test_model_fit( self, n_cells: int = 2000, n_genes: int = 2 ): - """ + Test if model fit for "norm" noise model works. :param n_cells: Number of cells to simulate (number of observations per test). :param n_genes: Number of genes to simulate (number of tests). - """ + logging.getLogger("tensorflow").setLevel(logging.ERROR) logging.getLogger("batchglm").setLevel(logging.WARNING) logging.getLogger("diffxpy").setLevel(logging.WARNING) @@ -242,12 +242,12 @@ def test_model_fit_partition( n_cells: int = 2000, n_genes: int = 2 ): - """ + Test if partitioned model fit for "norm" noise model works. :param n_cells: Number of cells to simulate (number of observations per test). :param n_genes: Number of genes to simulate (number of tests). - """ + logging.getLogger("tensorflow").setLevel(logging.ERROR) logging.getLogger("batchglm").setLevel(logging.WARNING) logging.getLogger("diffxpy").setLevel(logging.WARNING) @@ -264,12 +264,12 @@ def test_residuals_fit( n_cells: int = 2000, n_genes: int = 2 ): - """ + Test if residual fit for "norm" noise model works. :param n_cells: Number of cells to simulate (number of observations per test). :param n_genes: Number of genes to simulate (number of tests). - """ + logging.getLogger("tensorflow").setLevel(logging.ERROR) logging.getLogger("batchglm").setLevel(logging.WARNING) logging.getLogger("diffxpy").setLevel(logging.WARNING) @@ -280,7 +280,7 @@ def test_residuals_fit( n_genes=n_genes, noise_model="norm" ) - +""" if __name__ == '__main__': unittest.main() From ffe8a10b1cdd0dbe5c32b99fdc17ab274e6f1bb7 Mon Sep 17 00:00:00 2001 From: Mario Picciani Date: Mon, 13 Jan 2020 16:14:40 +0100 Subject: [PATCH 4/6] make closed form gradient the default for batchglm --- diffxpy/pkg_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diffxpy/pkg_constants.py b/diffxpy/pkg_constants.py index 12dfea2..48cabe0 100644 --- a/diffxpy/pkg_constants.py +++ b/diffxpy/pkg_constants.py @@ -15,4 +15,4 @@ BATCHGLM_BACKEND = "tf1" BATCHGLM_FEATUREWISE = True -BATCHGLM_AUTOGRAD = True +BATCHGLM_AUTOGRAD = False From 5979cfe6c79f250b513ffb2730eff65c92da22cf Mon Sep 17 00:00:00 2001 From: Mario Picciani Date: Thu, 16 Jan 2020 23:30:17 +0100 Subject: [PATCH 5/6] added unittest for numpy/tf2 DE comparison --- diffxpy/unit_test/test_compare_numpy_tf2.py | 81 +++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 diffxpy/unit_test/test_compare_numpy_tf2.py diff --git a/diffxpy/unit_test/test_compare_numpy_tf2.py b/diffxpy/unit_test/test_compare_numpy_tf2.py new file mode 100644 index 0000000..8ddb4f1 --- /dev/null +++ b/diffxpy/unit_test/test_compare_numpy_tf2.py @@ -0,0 +1,81 @@ +import unittest +import logging +import diffxpy.api as de +import numpy as np + + +class TestBackendsNb(unittest.TestCase): + + """ + Negative binomial noise model unit tests that test whether the wald test results + in the same logfoldchange, p- and q-values and the coefficents are the same after + fitting when using the same simulated data. + """ + + def __init__(self, *args, **kwargs): + + super(TestBackendsNb, self).__init__(*args, **kwargs) + + from batchglm.api.models.numpy.glm_nb import Simulator + self.sim = Simulator(num_observations=10000, num_features=200) + self.sim.generate_sample_description(num_batches=0, num_conditions=4) + self.sim.generate_params() + self.sim.generate_data() + + logging.getLogger("tensorflow").setLevel(logging.ERROR) + logging.getLogger("batchglm").setLevel(logging.WARNING) + logging.getLogger("diffxpy").setLevel(logging.WARNING) + + self.numpy_results = de.test.wald( + data=self.sim.input_data, + sample_description=self.sim.sample_description, + factor_loc_totest="condition", + formula_loc="~ 1 + condition + batch", + noise_model='nb', + backend='numpy' + ) + _ = self.numpy_results.summary() + + self.tf2_results = de.test.wald( + data=self.sim.input_data, + sample_description=self.sim.sample_description, + factor_loc_totest="condition", + formula_loc="~ 1 + condition + batch", + noise_model='nb', + backend='tf2' + ) + _ = self.tf2_results.summary() + + """ + Test with numpy: + """ + + def test_coeff_similarity(self): + + a_var_max_diff = np.max(np.abs(self.tf2_results.model_estim.a_var-self.numpy_results.model_estim.a_var)) + b_var_max_diff = np.max(np.abs(self.tf2_results.model_estim.b_var-self.numpy_results.model_estim.b_var)) + assert a_var_max_diff < 1e-6 and b_var_max_diff < 1e-6, \ + ("a_var_max_diff: %f, b_var_max_diff: %f", a_var_max_diff, b_var_max_diff) + + return True + + def test_logfoldchange_similarity(self): + max_diff = np.max(np.abs(self.tf2_results.summary()['log2fc'].values-self.numpy_results.summary()['log2fc'].values)) + assert max_diff < 1e-12, ("log_fold_change difference: %f > 1e-12", max_diff) + + return True + + def test_pval_similarity(self): + max_diff = np.max(np.abs(self.tf2_results.pval-self.numpy_results.pval)) + assert max_diff < 1e-12, ("p-val difference: %f > 1e-12", max_diff) + + return True + + def test_qval_similarity(self): + max_diff = np.max(np.abs(self.tf2_results.summary()['qval'].values-self.numpy_results.summary()['qval'].values)) + assert max_diff < 1e-12, ("q-val difference: %f > 1e-12", max_diff) + + return True + +if __name__ == '__main__': + unittest.main() From c165888606a26aa02128bb36fb230719a0a91291 Mon Sep 17 00:00:00 2001 From: Mario Picciani Date: Fri, 17 Jan 2020 04:10:26 +0100 Subject: [PATCH 6/6] bugfix: string comparison with "is not" --- diffxpy/testing/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diffxpy/testing/tests.py b/diffxpy/testing/tests.py index 8a327b1..351fc8f 100644 --- a/diffxpy/testing/tests.py +++ b/diffxpy/testing/tests.py @@ -194,7 +194,7 @@ def _fit( constructor_args = {} if quick_scale is not None: constructor_args["quick_scale"] = quick_scale - if batch_size is not None and backend is not "tf2": + if batch_size is not None and backend != "tf2": constructor_args["batch_size"] = batch_size # Backend-specific constructor arguments: if backend.lower() in ["tf1"]: