From 118d6739eae5ba804cb5244b8c82632295d865d3 Mon Sep 17 00:00:00 2001 From: Will Handley Date: Thu, 12 Sep 2019 12:13:46 +0100 Subject: [PATCH 1/5] Added references --- Paper/HighDimensionalSampling.bib | 114 ++++++++++++++++++++++++++++++ Paper/HighDimensionalSampling.tex | 7 ++ 2 files changed, 121 insertions(+) diff --git a/Paper/HighDimensionalSampling.bib b/Paper/HighDimensionalSampling.bib index e525c6f..e9cf949 100644 --- a/Paper/HighDimensionalSampling.bib +++ b/Paper/HighDimensionalSampling.bib @@ -3232,3 +3232,117 @@ @article{Beringer:1900zz reportNumber = "SLAC-REPRINT-2014-001", SLACcitation = "%%CITATION = PHRVA,D86,010001;%%", } + +@article{Handley:2015fda, + author = "Handley, W. J. and Hobson, M. P. and Lasenby, A. N.", + title = "{PolyChord: nested sampling for cosmology}", + journal = "Mon. Not. Roy. Astron. Soc.", + volume = "450", + year = "2015", + number = "1", + pages = "L61-L65", + doi = "10.1093/mnrasl/slv047", + eprint = "1502.01856", + archivePrefix = "arXiv", + primaryClass = "astro-ph.CO", + SLACcitation = "%%CITATION = ARXIV:1502.01856;%%" +} + +@ARTICLE{2015MNRAS.453.4384H, + author = {{Handley}, W.~J. and {Hobson}, M.~P. and {Lasenby}, A.~N.}, + title = "{POLYCHORD: next-generation nested sampling}", + journal = "Mon. Not. Roy. Astron. Soc.", + keywords = {methods: data analysis, methods: statistical, Astrophysics - Instrumentation and Methods for Astrophysics}, + year = "2015", + month = "Nov", + volume = {453}, + number = {4}, + pages = {4384-4398}, + doi = {10.1093/mnras/stv1911}, +archivePrefix = {arXiv}, + eprint = {1506.00171}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2015MNRAS.453.4384H}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@article{Graff:2011gv, + author = "Graff, Philip and Feroz, Farhan and Hobson, Michael P. + and Lasenby, Anthony", + title = "{BAMBI: blind accelerated multimodal Bayesian inference}", + journal = "Mon. Not. Roy. Astron. Soc.", + volume = "421", + year = "2012", + pages = "169-180", + doi = "10.1111/j.1365-2966.2011.20288.x", + eprint = "1110.2997", + archivePrefix = "arXiv", + primaryClass = "astro-ph.IM", + SLACcitation = "%%CITATION = ARXIV:1110.2997;%%" +} + +@article{Feroz:2007kg, + author = "Feroz, Farhan and Hobson, M. P.", + title = "{Multimodal nested sampling: an efficient and robust + alternative to MCMC methods for astronomical data + analysis}", + journal = "Mon. Not. Roy. Astron. Soc.", + volume = "384", + year = "2008", + pages = "449", + doi = "10.1111/j.1365-2966.2007.12353.x", + eprint = "0704.3704", + archivePrefix = "arXiv", + primaryClass = "astro-ph", + SLACcitation = "%%CITATION = ARXIV:0704.3704;%%" +} +@article{Feroz:2008xx, + author = "Feroz, F. and Hobson, M. P. and Bridges, M.", + title = "{MultiNest: an efficient and robust Bayesian inference + tool for cosmology and particle physics}", + journal = "Mon. Not. Roy. Astron. Soc.", + volume = "398", + year = "2009", + pages = "1601-1614", + doi = "10.1111/j.1365-2966.2009.14548.x", + eprint = "0809.3437", + archivePrefix = "arXiv", + primaryClass = "astro-ph", + SLACcitation = "%%CITATION = ARXIV:0809.3437;%%" +} +@article{Feroz:2013hea, + author = "Feroz, F. and Hobson, M. P. and Cameron, E. and Pettitt, + A. N.", + title = "{Importance Nested Sampling and the MultiNest Algorithm}", + year = "2013", + eprint = "1306.2144", + archivePrefix = "arXiv", + primaryClass = "astro-ph.IM", + SLACcitation = "%%CITATION = ARXIV:1306.2144;%%" +} +@article{Graff:2013cla, + author = "Graff, Philip and Feroz, Farhan and Hobson, Michael P. + and Lasenby, Anthony N.", + title = "{SKYNET: an efficient and robust neural network training + tool for machine learning in astronomy}", + journal = "Mon. Not. Roy. Astron. Soc.", + volume = "441", + year = "2014", + number = "2", + pages = "1741-1759", + doi = "10.1093/mnras/stu642", + eprint = "1309.0790", + archivePrefix = "arXiv", + primaryClass = "astro-ph.IM", + SLACcitation = "%%CITATION = ARXIV:1309.0790;%%" +} + +@misc{pybambi, + author = {{Handley}, W.~J. and {Scott}, P. and {White}, M.~J}, + title = {pyBAMBI}, + month = Feb, + year = 2019, + doi = {10.5281/zenodo.2500877}, + url = {https://doi.org/10.5281/zenodo.2500877} +} + diff --git a/Paper/HighDimensionalSampling.tex b/Paper/HighDimensionalSampling.tex index 492b65e..252f4d3 100644 --- a/Paper/HighDimensionalSampling.tex +++ b/Paper/HighDimensionalSampling.tex @@ -59,6 +59,13 @@ \subsection{Nested sampling and pyBAMBI} \emph{Author: Will Handley} +\texttt{PolyChord}~\cite{2015MNRAS.453.4384H,Handley:2015fda} +\texttt{MultiNest}~\cite{Feroz:2007kg,Feroz:2008xx,Feroz:2013hea} +\texttt{BAMBI}~\cite{Graff:2011gv} +\texttt{SkyNet}~\cite{Graff:2013cla} +\texttt{pyBAMBI}~\cite{pybambi} + + \subsection{Sequential Neural Likelihoods} \emph{Author: Barry Dillon} From eadd2c972f50f1b23d2e7858dd1062f19f46f75c Mon Sep 17 00:00:00 2001 From: Will Handley Date: Thu, 12 Sep 2019 12:13:54 +0100 Subject: [PATCH 2/5] Added .gitignore file --- Paper/.gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 Paper/.gitignore diff --git a/Paper/.gitignore b/Paper/.gitignore new file mode 100644 index 0000000..ed1eeb0 --- /dev/null +++ b/Paper/.gitignore @@ -0,0 +1,8 @@ +HighDimensionalSampling.aux +HighDimensionalSampling.bbl +HighDimensionalSampling.blg +HighDimensionalSampling.fdb_latexmk +HighDimensionalSampling.fls +HighDimensionalSampling.log +HighDimensionalSampling.pdf +HighDimensionalSampling.toc From 2783cef2832e8f43958693acf61f6a66709c53e1 Mon Sep 17 00:00:00 2001 From: Will Handley Date: Wed, 27 Nov 2019 09:05:31 +0000 Subject: [PATCH 3/5] Added a nested sampling figure --- Paper/HighDimensionalSampling.bib | 19 +++++++++++++++++ Paper/HighDimensionalSampling.tex | 33 +++++++++++++++--------------- Paper/nested_sampling.pdf | Bin 0 -> 15360 bytes 3 files changed, 35 insertions(+), 17 deletions(-) create mode 100644 Paper/nested_sampling.pdf diff --git a/Paper/HighDimensionalSampling.bib b/Paper/HighDimensionalSampling.bib index e9cf949..175a28f 100644 --- a/Paper/HighDimensionalSampling.bib +++ b/Paper/HighDimensionalSampling.bib @@ -3233,6 +3233,18 @@ @article{Beringer:1900zz SLACcitation = "%%CITATION = PHRVA,D86,010001;%%", } +@article{Skilling:2006gxv, + author = "Skilling, John", + title = "{Nested sampling for general Bayesian computation}", + journal = "Bayesian Analysis", + volume = "1", + year = "2006", + number = "4", + pages = "833-859", + doi = "10.1214/06-BA127", + SLACcitation = "%%CITATION = INSPIRE-1670681;%%" +} + @article{Handley:2015fda, author = "Handley, W. J. and Hobson, M. P. and Lasenby, A. N.", title = "{PolyChord: nested sampling for cosmology}", @@ -3346,3 +3358,10 @@ @misc{pybambi url = {https://doi.org/10.5281/zenodo.2500877} } +@article{aeons, + author = {{Handley}, Will, and others}, + title = {AEONS: Approximate end of nested sampling}, + journal = {In preparation}, + year = {2019} +} + diff --git a/Paper/HighDimensionalSampling.tex b/Paper/HighDimensionalSampling.tex index 252f4d3..c391bba 100644 --- a/Paper/HighDimensionalSampling.tex +++ b/Paper/HighDimensionalSampling.tex @@ -1,21 +1,6 @@ -\documentclass[12pt]{JHEP3} +\documentclass[11pt]{article} +\usepackage{jheppub} \bibliographystyle{JHEP} -\usepackage{multicol} -\usepackage[dvips]{epsfig,graphics} -\usepackage{graphicx} -\usepackage{cite} - -%% %%%%%%%%% Graphics macros %%%%%%%%%%%%%% - \newlength{\wth} - \setlength{\wth}{2.6in} - -\newcommand{\twographst}[2]{% - \unitlength=1.1in - \begin{picture}(5.8,2.6)(-0.15,0) - \put(0,0){\epsfig{file=#1, width=\wth}} - \put(2.7,0){\epsfig{file=#2, width=\wth}} - \end{picture} -} \title{A comparison of sampling algorithms for particle astrophysics applications} \author{The DarkMachines High Dimensional Sampling group: ...} @@ -23,6 +8,8 @@ \keywords{Sampling, Dark Matter} \begin{document} \section{Introduction} +\maketitle + The core of the scientific method in the physical sciences is the extraction of the parameters of theories that can explain a variety of observations. In the modern era, the increasing size of observed datasets, combined with the increasing complexity of physical theories, has introduced a substantial computational complexity in the elucidation of correct physical theories. This is exacerbated by the fact that we have passed the era where one experiment is likely to unambiguously determine the next theory of particle physics or cosmology, and we must instead combine clues from many different branches of observation. At the same time, one can argue that substantial increases in computing power have made scientists more ambitious in the scope of both the observations and the theoretical calculations that can be used to describe the physical world. It is often true that one can define a likelihood for a set of model parameters by, for example, simulating the results of a variety of observations, and using standard statistics to define the likelihood terms that result from each individual experiment. These can then be combined into a composite likelihood, and one can use this along with a Bayesian or frequentist statistical framework to define and obtain the ``best'' set of parameters of a given model, and the associated uncertainties. One can also use the same approach to perform model comparison. The difficulty is that the likelihood function is rarely known analytically, and can only be obtained by sampling. The simplest approach of naive random sampling is deficient for two reasons. Firstly, it is known to concentrate samples at the boundary of the space once the number of parameters is moderately large, leading to biased inferences. Secondly, it is highly inefficient in most physical examples, in which the high likelihood regions of the parameter space usually occupy a very small region of the total multidimensional volume. The past decades have thus seen the development of a series of novel sampling procedures, many of which have been utilised in particle astrophyics applications {\bf (MJW: Need a citation strategy here. I suggest a very generous citation of both the original techniques and particle astrophysics global fits that have used them)}. @@ -56,8 +43,20 @@ \subsection{Active learning} \emph{Author: Bob Stienen} \subsection{Nested sampling and pyBAMBI} +%\printinunitsof{in}\prntlen{\textwidth} \emph{Author: Will Handley} +\begin{figure} + \centerline{% + \includegraphics{nested_sampling.pdf} + } + \caption{Nested sampling. Live points are in orange, dead points in blue. Posterior peaks are marked with a cross.} +\end{figure} + +Nested sampling~\cite{Skilling:2006gxv} is an alternative sampling strategy which simultaneously computes the Bayesian evidence whilst generating samples from the posterior distribution. +It achieves this by initially generating $n_\mathrm{live}\sim10^2-10^4$ samples from the prior. These points are then evolved these by deleting the points with the lowest likelihood, repopulating with new points drawn from the prior but at higher likelihood. The live points steadily evolve in an exponentially decreasing volume of the prior, tightening around the peak(s) of the likelihood. + +The interplay between prior sampling with the likelihood entering via a constrained bound allows nested sampling to compute evidences. Furthermore, the set of discarded points may be weighted to produce posterior samples at any thermodynamic temperature, since nested sampling is an athermal algorithm~\cite{aeons}. \texttt{PolyChord}~\cite{2015MNRAS.453.4384H,Handley:2015fda} \texttt{MultiNest}~\cite{Feroz:2007kg,Feroz:2008xx,Feroz:2013hea} diff --git a/Paper/nested_sampling.pdf b/Paper/nested_sampling.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3657fee33c4413088eebda580f69378d827c0949 GIT binary patch literal 15360 zcmd^m1zTLpwk-(+4Q|0g1HrYCMiL+d3l`j68VJ_7yIXJw0Rq9D;O_1cJXrAH?)Db@ z?0wHZcfa@kz~TE+tg4!GjyXrw>b08Gpq3L6V+ONuqEZ*lg9__W!5}se%)kuw%^MJ_ zoW2Rv4#Wo(!himj0BM6* z6;1VxK3bW8Apde#lQl4d8rp-n{|Nr?bii!Dz+VLjSy{pC?EnbCUl>RQY6bLo{}n*o z7Up0LdMr~|5yUD2b^K@uRS*ZDfu54BzLlM|zAe`jK>h#X^O54ek@=Y0 zU&)Jov;g=5v5Hv$go;27VMYM|(oicCds6`IG03B%oxLqo-xAe1W%BDlgnOUO;>8`_ z*;DPo8z{X{E8Nq@7q-g*%lI;iG{zRePj{y;4#Kwgy-ZXlB+!N4j(StcsG69p_k|A- zV(YGj)U-5q&85MX&k67EylNgCZw_uB9&EyD(k?GXU_NP{9*iI7(H_nQdc3+4_xWq& zamWMCdc0iRZqi^aH%RxFOP673iw1R@$#-eGO9ph;=68n!o0t@fo104l-JTDf#y8V* zkrKwWJ#)V=jHwLruFj_g7w&iVyp}vKFeR3VV9Poi$Syf*uLb=zjDsrzj)3`-otDFc z3nUNT+w+^WYTXv*miwKwrTh4yae^+*j~8RcU#QQqVH^wN?>~miUDw_nP0tl`=5PaXf(H!?YNW4s`6m1-Ipg=Np1$k?p4WAJonHq;OpGG#j5 z*|bbY8JkkXZw-|+t%Y^$-U@WNtvoR^wNY_m-qJC7@1WHDPWCJ61Fwe6qFuF%EN|eb z>(JSGY7albtL(G=;@hU&8@+|}iSD`$$Hd`zsFKoQDQQJg?`iXk7&#yjL7n_WSI(7v z75O5%UO`+cFWgi5B1+?z#N45=Nqy>%a0(~K%O;RA#o3f_;$k4j>zg^>&Nx8~3$&*x z#$orKnU)pRI+w}rg~*aGtI~%SO$RAl!}LJXy?PkEH!tkE{lPW~w&G)k7)2!cD{JzN z{c27l>fApBH6xf3#;%6>u+%GOmri;$Hd72CS$G^#CiK{MVT&dHZiSz=l9=!Yro0Y8 zC2BSxD!uPuKXlwy3`!=BfM44f?bxWqShn{(LlZRJu6ogLzD$#|^Zi2S6VS5bj_SzawDtqnHm>6E^*H$;6Jn#uIHs&m z)Tb$)s1(J9_3%XfW_FVUvL?=!>F8*DCPBfe8$J6TIBC_ZgXe~kv!W4#j?&Li18yWs zJXTLl+LK?Y=-8xLe4tYko7OvX6WZ<{EK5Z1F<4JAv}&g9d?9uV-Vq{Ol4>$kQsd)k z#qA2MAjbCS3`uF$t>2&= zd$wYNm+TFqPU#4zdquk>(u1}86R#Zdu}CI8dF{dxpVAKLFHcA?^Sj7b>?<%W_MOv4 z8A+lodl+@c+fz!2%$(dwurN=e5YlttR9@EANsV0Fa^j+QG#gAs+qjFtN~$SHz`um8 zKP!Te`~w`ih3ahsM3bxR<0>Rgs~CTAU@6%I8K&UAbudUK0x_W!q{NVnxHo5F?;>xw zSo`<3|H53~X>6-qj!4Ox@!c~)$n@ir2br*GwzGIoW91+%wwpiMcne#cx6tE`RgmGD zZU1J6zQDD5fKuZv^rUQA04f0Q9sZz?qVp?qSpI3*n4Dk!t$LK7T{P}SY%Dpcm#|VB z+Oq|z;QEHsi-_qST=-=4*Ys*;3Q=u{&KUt`-G*gkdZZStHz9k_tM3*BfoavujyYmq)bxACM1`8hyeut~a!TDt=g}0c3C{U(OPXXsG zk7aPqt?h$yqsC~B_?D#?pwIUD!%eJV}s zZ1Ni%@cQrpU+{Z59uyY6UY!?|A&@DOa6d?kolYn(-@Qxi6M_f?qfc;p)l7>&YU%nEiny;#h~6qYnfdB`Kx}AD!PHC^y2Zyn ze+5Bp9q=;lBU_rp0!hZp z9>9{Qh`qZZqg;uJ=3UO=L9e;nGQ)@&yL6|3XWAa5nqF-A?ka z-}hUrFtz4L*>$~^Zch&ewqxFN!!P7g?`yooHj&Uq2_N>;?oYiQrpX^pZ_@6M&U?Hx zPr_ucI%)*nW%=&6smZw3lp~IJoCNQGO_J);k9aln;}k@8F@zAy2jBC0p(c8dTnT!r z*?x;`la-KrT}6)jHByEs>8IRF->;zg`%7^N>F221aq%kF3K^^q$JMv%2VSl>P7zu- z*_X-~q>)5xkucma@sxMrGXj>PTAXnQtr_%%l#1y^#8Vue8nv}P{sTqZ?3XFZ+JR{m z1v|UBMcdU0iYgA;1N4RU%)9?vKzo{Nar7*O9#6`N@YTNFMzjikRws*8N*-nqAFZw0xX-RD) z%1+3|iZ^}Fxcyup9LcdTj# zRZdg%y^!by7xQ_p%j4Ti`zkxq(|YY)imZ^o1W~z)hLT+()$aB8gDDzcKG0J%e)%c< z&iS?bEQ*CBZK#LIXWN7(lI3`KVhxD5pc4Fz8f#~`$x+-DpIhVJFM78|GQpqfwj4eR z$AybWc)!`32j*-zPBaEa5Ab{S*{mIB$vg(RV45>f?^m_9qIR#E&1w}2rNu&gn9UOs zXx3VVAESOq?)-2Jx<8Y9{#s z-+L%lOBF74%0~T(TELpplbPWowI^zp+?TnOo;(%uOxe0`cb>pjDBlZdgDPK6R<{fs zmX>Y^tEUie21^9Tw9$V?@qD#rZ8`GRq_nIYGjTr&!BxnHrCu(rr>ZF3yqUY2F>f`)4zl;gl*iAP8DwxH_KX+3H@$wu6?$o@U@>{ z$1Y@)gxney@=-ETMNZ_?mjx@oAJ9HXYLH zMvhsGH$2QN@)UmWO9>x@j=I{P4_T047LeA5cFxXq+u=j~qJkLubFgw5`eUuPD4+pB z0Z1}!g#CLx5;OR|Xq{7QjnYpc{-j8&p@iEB@yw|NVTqXihzFZ!Mw=RG;SIrO-ExxsQs@jiX==JZOUfi>D#H<_v{I&L zs4no%?lAdDU+fmp*NrGk%}`k*qS@9Ta>c{V zw1hO}68E6r^4IjW#Uzn-fZAKWB7AD)u~&M>!Ht4f7(4U%$eSwnodpbeYzCyAdN!Y} z*bsDmFlhGjKqW>T^=Lh8%i66c@^qPW9RUq%rJ+uV?KZnk~>UB2U15)NH^t2t=?;GI z0QV$2MZ<`r8FUe&1{1%(jMud$;K09It}i!JLZvtzERegY$h0#+_r_Nrg?GwpM(aH` z{(kX|lgcFPZNIfdPej0|JGwGzP$}B4*UXtRc!Av!D_c>=Zmpj)-s%|w2} z{-~wsLE4r0wlxSh;Vb;CF>FqGj@DObxjvy}JBy(sT?BTnr)<$Y$HP?qOyrLL0nTZT zhM~RfO!O`LSmQ6k+>UOirJjW&N{e@&D+`{J41K~$ajq~VS8~0j#D{B zddlDg9{V9#!2avKkdP2BBkrN;I?PiC4 z)Ga^!ex9@v15yQ&A84;l+TBhSW5{nGp8F8=Q z(3UwXI22xqCnfbeirDSiSNF;{YS@=Hd{FwxIA8Ph+K3aGQv_TdW+Uetf&Wx0_=xdi zt7yMYi*=m@L(l@sOq=OPYdy}Wnh&}GaX243DG~|0bUVkyBnwlE<%p5!X?4OV&3sdN zeHheYnmnFVqW*XumIs^kuD3UJlpT$<_$p&NM$-I(`Ernrg4elhM(bn*@>vf@ZZNfw zxPKzw-bpvn-TvKT&LGv{b3E}|Y+sX`>}X0NgW4DbnJ;5MMM(zQ~Ux~Qv$vN{@X&3lF_qmDyCMnv!J}^@q6uIJqXu*tc(>UnRxPv zdv?@X>+s)Fpx5Ni4)&8sI;1q70=;#8bPQ?;>&V`Xp=HD$UVIaLwrtabv$VobF;zrI zE$03s{wD03R#DNxempFeuu~`x!;p*jD_`}fTYHg6tz;}{#~6(LnoJ4R?qW6O;mqin zkE^`I$!YI0hJKxVXl^Tm3zAeHq)b1+CW)P)sfn#G!^YaGJjL;YA}8GXZ|(QP7&v&8 z*TsS&2;R?L$_{-}h;H%z@y(RaiAcs?EI()$jT<|$UuH$s3Emn`SVV-{P1TOZlfIzB zSfdIwZPbLi57DSf)$=}s>?F1Qgr~u)<`EmeCAJ!BWQg=R8=mZ4XmKymTQmN-@5|@-aBK2*+Xz9*N*=mqu`7+=OhunJeKW0l$SW(V!0dvdjK!ufv@5$P zuW_qtT$;&Cj)6!6tle?nUp4P8rIx!&e0IAznbX8m^T;^G#u4ziMoSAEkzKtg(@o5Ipbz&%O6$C8U32J8MF=F+Tvi>n16_N7kwf#>b< z>!y*kb>BsCYnRA0-KcdL!IodEK?jN+&mFfsmT0bJr4BkS)HVmrU3C|=R`Hu)L~a=u z%zh)M&Cts)i#5ZZ85fr3ykj?0RVlwc<`h0Wjz=%ngbR$_9ODw#WnC0^V4gOg?M9vg zcQ$+zY~jG%bqR9B1q5=>xhS?v`|aW4w1wolRG&!I1XCWwZGz1u0Cw?hKv%opXYrzv zc^dGg7k6E96rpup$`SYsVC(`I+Z{fOHG6$P6Cp5uqjCmhk?Mc@_F!%bNUFHp7QpWD z0@BMl2gWcSfQD=oz*XJ=GN|?gLOI?7L9M#KZ+YCb1I-+P&e+X)2&`r9Mf7?^N`*~%Pf<_Z9_x&YGNPBT!ArnAh!+{j(bVvXP` zfW+O01c1Z==}*Q1>?G6H%K%Ljm9nIfP$;{_O{wRQyntL|eKFN~reEARu-8#v@> zg84-O&N*0sU#zwEkDF?K!5=q}a4(qyH(OfO(`K?%9iZ>;1e7%(09*;%0C9>pft)ds zOSU{@1s*A5o~Lda!7lPYqFLKF5G`fvl)ON6M(R9Z*1-A|mDm@50z0G?jNOB@)zi!s zUpu~W|8 zf*AlWf%0a7K_4T;RH83$f3*-d8ipGWG~7eeBF+3L_A+a8%2!FfHDq_uy%%vjwXcr% zP-4s}mk2~jgWY%R9DV%t@bc101ZLi4n{``z(Dp>4r7z8bT)^-lhP1dDNTD6%44jsIw?ABW37(lyJLlLi}cTe3H4?dFk9i(io@{m zR(pn6{q^cYyK!N&&QL-j;(Vedw6~dIeC}@@X&CoiCd|uMa?HfbSE`e6)oru~ql+%% zJnCo|t*ma}s<@+Kvh;EGkU9Qa+{Fd=wHq>zEI#N|z+M)@krV1B%D)v$)A>&wA zu}@Ytpfd5*@c`!!!UXwr>w%JFIcjXS-Vz_VmrieSau8Fw>P0c^rNL$@1fR#h)_<3% zvmgXv-9*q4s4IM%mYyg`HsVuYa(aNz*q6xabLaS^pg{!{6tHw!H_pO0jzl(QhOL@V zM%oXa5y{#QDz^j8-aOEy{p=;vqZKI2=!3&@k?)p!aBN<$det>=8=%;gTvV8{zmhu; z){qqbRgRxZWudx@P;``yrsXt+Jp+`;=egW#mfnP9TfFNbTsS1An$p0>%B`e09j4CoS*1Ge%0biBwQV zF0v*%#hd>T2oZ9KdZWN`rGTMKB<_S{Bk>k7+(7|#^nG7=QQs%SVT%RY0Xw}N0ovX~ zp3v1vkyn!*UT;k7M5!f|uGylpHhnGGY`^*!#6}lK|0sS zqd+D7Ad<-IVkf@3f3zq3k&Ka=VU=z>F3oQJ6l=@$BiX_&-We%F^`*6nG6Qv4_zwbQ z(H}B7`w@(z5-U80+vs+I>8MqvMV`Z@z7Nr3?|DFheAT{s2cq4j>H_^Q5bmOp<*Y>V zNd&b7u~IKO-e9Gsy%RT(E~^@c?_%uWF+WCB#eOCFwnsra!0?4SXWi9D_yO$D?eGPN zUp5*NwjJYcqqT1aE!^t@t}M9tU`}z(wZqLNY2(lbuw^O;3>^yY^MPYX_E&f=ilB8BcA`0@SW&>}sY|FSz~12;ZAwoHb5}w|y85C{=^u_2fHnBU zIEF(jp$z-CvbK82UUau{1HwddL#PJoH>GX^jr|OU6WTAw-DslF_>1zQ=!~e0Yquwt zZX_KC71xWQ1y%*m3-VExq6Mi^9Kg7u#pOhz1=P*nR(B~UXk2d&GPA#g{TP-kMbyr` z@6ys%eaR#Wa`G3Fv_)3SnQta``aGuIrq(9#$@p@9d*0O88uNXT8v10{_es|}gbb`g z@cdCm295R?a^#@SR@3xC!Hxwyd4m{`tBQTRgC&&qoVX94W#EI1UJ_OezGHH~F|{YP z%@n*syaX?=xniC_GkZW@ugND^tO#C>;UbD99uB?S+ToFGnM*?+92}gssMyExZHpIi zDnC@2Y6`IRM@PxJpl{~C8O#|wc7C>_E^}a87LsO>WMYV4WuXp^DYk!j5owg26oH5c za+M-9VHl?CjS7sO!FSSRJ`D)PSdWfR-id6tmzhF}Z~Ub6Ht7DdSta#kQuGu1{#b&l z2(`UW9^izqZ40{5!Nn3K{87e0RRQe)g6!b|LJG6#n*8J-<(;C9Up~3|Po$q?xvbB0kHw4c3Z5rdX1{We@H{<@rdC@`VO& z^JR#clQ2WrZ}mN-Cd*{9fH-X{1{C4}5qu$PQLPEu_5nE(S#hd$=gdVkIm2UeWq0&1 zffO8c^VGH+bVFyA@%0m|F8S&wW zJErg%!Izy1kcjeVVH7v->b7S{W@#!i#92w?i%7duv2@-gcnoNS1*V;|C09L647_vh zb1wS|n)>7UQhO*|}~ z>H8;T#7ZmrC<^gA zScxYHqtiIlEe^k^Ltu4g_ z-p_7+)rvO$YKYy{#DQFhs7xo!6b{&*-?7*_7Zx^}Dcw_E*-|3|_a#ZUZ@j z1EK>opkF$IE#LC4kFK7P3%aLm8G#6Y{z2(V83suucW6r^hyduEz)FWZ8F~rnB=?o@TRJ(Dj+}x|a_? zP8?N12}65_n9Ruy@bv(6{1uS*T$>^g@fL`)5UyKev5(dQWLAAx0JTfIEQi{~#{huW z!PCsig$6G*Al!P^Krys}RTfsXE@rdyoF_nt3mRa0X95U5^qEu>a^V2PDbaLZD6dPq zfevikhXd0M5(1SLR^l$eU{VZ#xFJ!Z2}zr>nJuqF?E`Xq-2>p5HADiIiCWbb0_nj6 zK)7?Q8Vf7aM-1EoVW9d5z|#x?Ag93@z_7Y;0#fMQo+z&q*ervdprimhWM9K;w)_hC zPNNB_LE^8nVCfiPPA2hx1E8bYn9aK5lmLM(M*tv_ZlF|m;7bj2^6NKE7PIqMi)FwP zo&yaB-ebyVjQ|z!zB>s+wB&98oy_ylfgCC|z<%e+R}Dx_Jh>)>yX}$5vps<0zGYy6 z_%70+b`@y=os)#A0AH_x3a>~pfI3QSoq=pc)`4u3H-MPHcNBn(!ehY6@f5Jn?Q8?c zN(I#cpc%nHwbv=R!0759NveGY)|6!(Sdh76fT;QeAOKkeu#>o-J>qW)R9WmBI0IyU zSqI7^W6`a)uyXVGYpLEj0M(u30W~ETkO90*rT}}T3s|bOa@NN|58w`v+Gu&*))BzY*K6R4?{Ph2`+*9+?ll9vRNnw#Tq-~a z6Y0lG{25?=JmGQuyB|F*dx2;rCP22b=gR;M`1!zAfLs7O2Pq&3*eMvm(pz<30-lor zk27Zg5ZeA@#vg%AAfTQKgtPPA1w1`QflEg|Aku3-{z{7yUt0j)t^^1|Hmn2eJhD*$ zDV3wqV%A-f9k{7#s(=P~iG8FfLKV0VBmlAn^s689#|5mkrhN~(Z%wm1v$?)hk!Uo_ z#%ygwly&BNWU&?=V%D|~HKy{>rP(S3Vf61w$dbjmmF$=M*~XR_BCV&+^V-W{ROXyIL!#5L!l8oYzf4z{0}Q%6}cla8~%r%~I*^*CB&t*vJJ3 z?o>m)3wu$_xt506W{*#PdaRPxFJaccVEPPBF4k zaZ_s1Vu~h=ti)Zp^qR}L%^5#6Md=eU1K&=~^~WS1sUQ(+w5sdg|Y^nAYbgQ^~cbVj=PX3{aK7+iiq-%#;nJ;6joXWLcP-RrdG5_0p&wdR)tg7Ty^F9 zmlvu-S;%g_66I2jljdP#!uVQYThKvARUt?7it}s1lyaS-T%P%QcM;?;&!dBhi-ygJ zxaZ0)C7Pz`HRm^ z_v)@iG#%eWHuG2&u5&jwEHBIIuKJ3VC!<|uRS2;0MU@bAnkMv;6mo>B zDKDFH&Box88m_<`ZQYlIW#S*7{|dZ@za63q?}~el4Lwa7H6tn_Br>1>{4Vts#6V@P zY`eGlweWcq`fz~ro_q@xO%->;HPi^LHk4QgCr?}=D=F@^ZDHZ|)YnjwE`N9XG5T+N ztnCW7UqN`)wi@OvKa?iYcY~hJjGDgcuo6mV5x^GOdqyi0_jxw{`-=HQkauh6%1ala zCL8H-Iz58gBRD$Qqq%ocQ7nP6p;?j5SovVM!O8FW=V5$nh`Y3U{@1~+iBa4x9Dma!i$qh2TYUIw+q0cV)oM9$ zlZl9gcNsNY-7m|b$e`mr%6v#;?4W2Uo6EhZLn1wzw~=iXR21R7J8XhsM3fp{j4=EI zABmjOzb9-N?alWNdGqMVIJ17ouLue$N59*&2gFV47ETX;->b|}ZoIciQ$8cM!Bik5 zQbXA(Nm%A9uQ_jBQy+BTvfhGqWl>mMx>B97C;dkJ79SX|T$B{@A|KYN+joYKH>W49 z9-(|FgW$eG6LF7Zp^o_3;*Ok|&c+CaWGsYyVs9d#t6CN=2sF4Uw z53UK)$fpWr7^I42WD_ojTF*6h` z(#e7VhGs~Ky*z7R&Gb3ysnr3haT_Ua0b8@d@YxndjZNi38>!XmjN|GxB;xFOWQ~ml zrEUEu!_QG2mZUd_rnlVoQ|xADWnrjQcU3c*8OaLX2^V-Ix-LR(Po3U$OExv(_L|wR z;uVvq8nM7glIxA_$CN9qrRinkX2)H~CCg zuxwXYB2`i5pWK_$+N`1AZ=mBMxITkCLoLSuXSCimdc8}&vXv00O#j7$;E1celHm)^ zn8DHwE!D|fk-Q=oQG;9G8_Ua3pM<0O3$*9qvIVPSxStzT?&BNh0}xcBuv<8H`PY+79UQTXn#OE*CV3F*wZu5Z*z}zB#}6}mMF}eQh4LmWsXEiX z)BVT@{XeCX40NE)z-5eCQ+aD~|p_S0ns1(lR_S48T>7`{TYYtu&7 za^rn?TLQ|!nJ!~e*`WD(3g>v8N=aMEV&VRRj?c47z zT-;qTDbyzuIs2yeJLgF{^0cnWLT$!R!>M>EYNhBxJ5rsBCOX4yN$6dG6vm4CZ!1D_ zLu7a8`b%CrC+M>0yWm`?~qq=kC_j>99654GG~RN>=oO-7v^Wu6Rti_ zb!u=?7@FNEzrmBJR-MLUbcmQh0wcxosq%kxr^>S!-f*rPwaQ{)CDgzKj{4m~g1>%L zWuPDK)n%%ng{q=aH-mb5$nY}UO?MW0CS+i5QyDPAs<<4KQFMR1mcFKqQXe~Krp=d@ zJ=aF&=qWL+4Co;f&!85C)-;hxs2o)auw#>LE+FR^jMuYgqYk{5ioIsai2kWXen&z~ zD&pWo$V?a76y6pgIl<#@bbKbJEuu}WeAPwyfk7pR?BWcO8I-`35kf!2ApY#BlZ;=r zO8WQOeudr;onNN6h6o$m{TK~)XEgTQ%;Ty2Gv04PP?%|Xm> zsEE&>u-M&psc4(1&QURr_!Fq?OjBgM8xQif1=UkMJB13Sw=7s;Iq`3=TPhb4&$m4n zB7+HCeCVb7E%su1m3}!VMlvcD(u+ zB>|7-=$(U_4-Q>|OT7Uf1ic0&&2MfmZyu-a=d9K=O6qM(%3W_dF3?In&sbW2WNWjs z4qhJ(9B|uSRovd5KRgjN!0vz4Ec|(3{!jJ79|!3lPuzpq*|^#M(lk6O7XCbj|8Hf% zqom=lv-v{ce<=rqfU!SM^8Z`=z-p)u0zaP3XBA@q%MASgHv8N0|3_)WKX4xf5C0f^ zls){Fj0_tH{HI7m1`GoKDYuYe2Z8@oVaRZRz<;PNK&&!=QsEB;1#rm)0{@|&02B=n z5cm&S!ryZFBjjHY@BSrs0E8l3pg)?&i^qR}&f)*7Q{Z4nWmR-Au>V6mA_lXy{ELUb z^c#PdFjz)`F}y*~MK4l)(cFK82A4$WbaW*;KTJX>^Vy~9Fq~A^ftV9IYAdM|V-C9xM|H}sX+g@P(FQ(z|yZ^tKhQHmR z|INGMZ=Xs3%{2V|Q~1C5rue%A@V}Ra$4BsA&r4yL10eka9{N%rjUM%ff2a}_9-o%~ zmi^iN{bBoe;h!*|=(GO_vl4j}Qqqa=v9p1B*?7U+UkkF}98$UxuF z9C#)Amp}dAGIO?t8l$p-*x69o{^th-;o{=p0vUt;XYBEv;_(M$_2(Fv69W0i7&{jP z`274&J9aLff7(HK{%Oa}_D?%*cHj>Dk9a`m9~f+G9K665{a<$MZ2x-<%+AFLJnjF( z4$RKO{*PQZ*m(Zo%fZ0~XfprPmjm+n2KK+kc)0&}&R|Y3Cm~Zh?q1sja=n4fE?{Pux?DcK!|5#2iCl3!B MDmAsJtQhM50g!Vf+W-In literal 0 HcmV?d00001 From c3e69b67f5ff7738abc8a1436602eca4485bd6d6 Mon Sep 17 00:00:00 2001 From: Will Handley Date: Wed, 27 Nov 2019 09:47:44 +0000 Subject: [PATCH 4/5] Added text --- Paper/HighDimensionalSampling.bib | 52 +++++++++++++++++++++++++++++++ Paper/HighDimensionalSampling.tex | 13 +++----- 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/Paper/HighDimensionalSampling.bib b/Paper/HighDimensionalSampling.bib index 175a28f..7c8958b 100644 --- a/Paper/HighDimensionalSampling.bib +++ b/Paper/HighDimensionalSampling.bib @@ -3365,3 +3365,55 @@ @article{aeons year = {2019} } +@article{neal2003, + author = "Neal, Radford M.", + doi = "10.1214/aos/1056562461", + fjournal = "The Annals of Statistics", + journal = "Ann. Statist.", + month = "06", + number = "3", + pages = "705--767", + publisher = "The Institute of Mathematical Statistics", + title = "Slice sampling", + url = "https://doi.org/10.1214/aos/1056562461", + volume = "31", + year = "2003" +} +@ARTICLE{Higson:2019, + author = {{Higson}, Edward and {Handley}, Will and {Hobson}, Mike and + {Lasenby}, Anthony}, + title = "{Dynamic nested sampling: an improved algorithm for parameter estimation and evidence calculation}", + journal = {Statistics and Computing}, + keywords = {Statistics - Computation, Astrophysics - Instrumentation and Methods for Astrophysics, Physics - Data Analysis, Statistics and Probability, Statistics - Methodology, Statistics - Computation, Astrophysics - Instrumentation and Methods for Astrophysics, Physics - Data Analysis, Statistics and Probability, Statistics - Methodology}, + year = "2019", + month = "Sep", + volume = {29}, + number = {5}, + pages = {891-913}, + doi = {10.1007/s11222-018-9844-0}, +archivePrefix = {arXiv}, + eprint = {1704.03459}, + primaryClass = {stat.CO}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2019S&C....29..891H}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + +@Article{Brewer2011, +author="Brewer, Brendon J. +and P{\'a}rtay, Livia B. +and Cs{\'a}nyi, G{\'a}bor", +title="Diffusive nested sampling", +journal="Statistics and Computing", +year="2011", +month="Oct", +day="01", +volume="21", +number="4", +pages="649--656", +abstract="We introduce a general Monte Carlo method based on Nested Sampling (NS), for sampling complex probability distributions and estimating the normalising constant. The method uses one or more particles, which explore a mixture of nested probability distributions, each successive distribution occupying ∼e−1 times the enclosed prior mass of the previous distribution. While NS technically requires independent generation of particles, Markov Chain Monte Carlo (MCMC) exploration fits naturally into this technique. We illustrate the new method on a test problem and find that it can achieve four times the accuracy of classic MCMC-based Nested Sampling, for the same computational effort; equivalent to a factor of 16 speedup. An additional benefit is that more samples and a more accurate evidence value can be obtained simply by continuing the run for longer, as in standard MCMC.", +issn="1573-1375", +doi="10.1007/s11222-010-9198-8", +url="https://doi.org/10.1007/s11222-010-9198-8" +} + diff --git a/Paper/HighDimensionalSampling.tex b/Paper/HighDimensionalSampling.tex index ad08a2f..6245826 100644 --- a/Paper/HighDimensionalSampling.tex +++ b/Paper/HighDimensionalSampling.tex @@ -50,19 +50,16 @@ \subsection{Nested sampling and pyBAMBI} \centerline{% \includegraphics{nested_sampling.pdf} } - \caption{Nested sampling. Live points are in orange, dead points in blue. Posterior peaks are marked with a cross.} + \caption{Nested sampling. Live points are in orange, dead points in blue. Posterior peaks are marked with a cross.\label{fig:nested_sampling}} \end{figure} Nested sampling~\cite{Skilling:2006gxv} is an alternative sampling strategy which simultaneously computes the Bayesian evidence whilst generating samples from the posterior distribution. -It achieves this by initially generating $n_\mathrm{live}\sim10^2-10^4$ samples from the prior. These points are then evolved these by deleting the points with the lowest likelihood, repopulating with new points drawn from the prior but at higher likelihood. The live points steadily evolve in an exponentially decreasing volume of the prior, tightening around the peak(s) of the likelihood. +It achieves this by initially generating $n_\mathrm{live}\sim10^2-10^4$ samples from the prior. These points are then evolved these by deleting the points with the lowest likelihood and repopulating with new points drawn from the prior but at higher likelihood. The live points steadily evolve in an exponentially decreasing volume of the prior, tightening around the peak(s) of the likelihood (Figure~\ref{fig:nested_sampling}). The set of discarded ``dead'' points can then be used as posterior samples when appropriately weighted, and the same weighting may be used to compute the Bayesian evidence. The interplay between prior sampling with the likelihood entering via a constrained bound is what allows nested sampling to compute evidences. Furthermore, the set of discarded points may be weighted to produce posterior samples at any thermodynamic temperature, since nested sampling is an athermal algorithm~\cite{aeons}. -The interplay between prior sampling with the likelihood entering via a constrained bound allows nested sampling to compute evidences. Furthermore, the set of discarded points may be weighted to produce posterior samples at any thermodynamic temperature, since nested sampling is an athermal algorithm~\cite{aeons}. +Historically the procedure by which nested sampling repopulates the set of live points is the main distinguishing feature between nested sampling algorithms. For example, +\texttt{MultiNest}~\cite{Feroz:2007kg,Feroz:2008xx,Feroz:2013hea} achieves this by constructing a hull of intersecting ellipsoids and then rejection sampling within them, whilst \texttt{PolyChord}~\cite{2015MNRAS.453.4384H,Handley:2015fda} utilises a slice-sampling~\cite{neal2003} approach in order to avoid the exponential inefficiency scaling with dimensionality associated with rejection sampling. Adjustments to the meta-algorithm of nested sampling have been suggested and implemented, for example dynamic~\cite{Higson:2019} and diffusive~\cite{Brewer2011} nested sampling. -\texttt{PolyChord}~\cite{2015MNRAS.453.4384H,Handley:2015fda} -\texttt{MultiNest}~\cite{Feroz:2007kg,Feroz:2008xx,Feroz:2013hea} -\texttt{BAMBI}~\cite{Graff:2011gv} -\texttt{SkyNet}~\cite{Graff:2013cla} -\texttt{pyBAMBI}~\cite{pybambi} +Another extension to nested sampling is the \texttt{BAMBI}~\cite{Graff:2011gv} algorithm (blind accelerated multimodal Bayesian inference). \texttt{BAMBI} was designed as a means to reduce the time spent on likelihood calls by training a neural network to act as a proxy for the true likelihood. Periodically a neural network is trained and tested on the current set of live and dead points. When the neural network forms a sufficiently accurate proxy, it is swapped with the true (substantially slower) likelihood. At the end of this process one is left with a set of trained neural networks that can also be used for further analyses. \texttt{BAMBI}~\cite{Graff:2011gv} was originally implemented in an early version of \texttt{MultiNest}, and has since been updated for later versions and other nested sampling software in \texttt{pyBAMBI}~\cite{pybambi}. In this update, the interface is in pure python, with the additional functionality implemented via dumper function interfaces, with the neural network code \texttt{TensorFlow} replacing \texttt{SkyNet}~\cite{Graff:2013cla}. \subsection{Sequential Neural Likelihoods} From bb5d9537ff4df07fe529188f3e4af569868d4fe9 Mon Sep 17 00:00:00 2001 From: Will Handley Date: Wed, 27 Nov 2019 09:50:00 +0000 Subject: [PATCH 5/5] Updated gitignore and added script to produce plot --- Paper/.gitignore | 1 + Paper/nested_sampling.py | 80 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 Paper/nested_sampling.py diff --git a/Paper/.gitignore b/Paper/.gitignore index ed1eeb0..c5c7045 100644 --- a/Paper/.gitignore +++ b/Paper/.gitignore @@ -6,3 +6,4 @@ HighDimensionalSampling.fls HighDimensionalSampling.log HighDimensionalSampling.pdf HighDimensionalSampling.toc +HighDimensionalSampling.out diff --git a/Paper/nested_sampling.py b/Paper/nested_sampling.py new file mode 100644 index 0000000..686567b --- /dev/null +++ b/Paper/nested_sampling.py @@ -0,0 +1,80 @@ +from anesthetic import NestedSamples +import matplotlib.pyplot as plt +from scipy.special import logsumexp +import numpy +import tqdm + +def loglikelihood(x): + """Example non-trivial loglikelihood + + - Constrained zero-centered correlated parameters x0 and x1, + - half-constrained x2 (exponential). + - unconstrained x3 between 0 and 1 + - x4 is a slanted top-hat distribution between 2 and 4 + + """ + x0, x1 = x[:] + sigma0, sigma1 = 0.1, 0.1 + mu0, mu1 = 0.7, 0.7 + eps = 0.9 + x0 = (x0-mu0)/sigma0 + x1 = (x1-mu1)/sigma1 + + logl_A = -numpy.log(2*numpy.pi*sigma0*sigma1*(1-eps**2)**0.5) - (x0**2 - 2*eps*x0*x1 + x1**2)/(1-eps**2)/2 + + x0, x1 = x[:] + sigma0, sigma1 = 0.1, 0.1 + mu0, mu1 = 0.3, 0.3 + eps = -0.9 + x0 = (x0-mu0)/sigma0 + x1 = (x1-mu1)/sigma1 + + logl_B = -numpy.log(2*numpy.pi*sigma0*sigma1*(1-eps**2)**0.5) - (x0**2 - 2*eps*x0*x1 + x1**2)/(1-eps**2)/2 + + return logsumexp([logl_B,logl_A]) - numpy.log(2) + + +def ns_sim(ndims=2, nlive=100, ndead=700): + """Brute force Nested Sampling run""" + numpy.random.seed(0) + low=(0, 0) + high=(1,1) + live_points = numpy.random.uniform(low=low, high=high, size=(nlive, ndims)) + live_likes = numpy.array([loglikelihood(x) for x in live_points]) + live_birth_likes = numpy.ones(nlive) * -numpy.inf + + dead_points = [] + dead_likes = [] + birth_likes = [] + for _ in tqdm.tqdm(range(ndead)): + i = numpy.argmin(live_likes) + Lmin = live_likes[i] + dead_points.append(live_points[i].copy()) + dead_likes.append(live_likes[i]) + birth_likes.append(live_birth_likes[i]) + live_birth_likes[i] = Lmin + while live_likes[i] <= Lmin: + live_points[i, :] = numpy.random.uniform(low=low, high=high, size=ndims) + live_likes[i] = loglikelihood(live_points[i]) + return dead_points, dead_likes, birth_likes, live_points, live_likes, live_birth_likes + +data, logL, logL_birth, live, live_logL, live_logL_birth = ns_sim() + +ns = NestedSamples(data=data, logL=logL, logL_birth=logL_birth) + +fig, axes = plt.subplots(1,4, sharex=True, sharey=True, figsize=(5.95,5.95/3.8)) + +for i, ax in enumerate(axes): + j = i*100 + points = ns.iloc[:j] + ax.plot(points[0], points[1], '.', ms=4, label='dead points') + points = ns.live_points(ns.logL[j]) + ax.plot(points[0], points[1], '.', ms=4, label='live points') + ax.plot(0.7, 0.7, 'k+') + ax.plot(0.3, 0.3, 'k+') + +ax.xaxis.set_ticks([]) +ax.yaxis.set_ticks([]) + +fig.tight_layout() +fig.savefig('nested_sampling.pdf')