Entropy signature and folder structure cleanup

.github/PULL_REQUEST_TEMPLATE/new_entropy_template.md

@@ -13,7 +13,7 @@ project [devdocs](https://juliadynamics.github.io/Entropies.jl/dev/devdocs/).
 Ticking the boxes below will help us provide good feedback and speed up the review process.
 Partial PRs are welcome too, and we're happy to help if you're stuck on something.
 
-- [ ] The new entropy subtypes `Entropy` or `IndirectEntropy`.
+- [ ] The new entropy (estimator) subtypes `Entropy` (`EntropyEstimator`).
 - [ ] The new entropy has an informative docstring, which is referenced in
     `docs/src/entropies.md`.
 - [ ] Relevant sources are cited in the docstring.

docs/src/examples.md

@@ -137,9 +137,9 @@ for (i, r) in enumerate(rs)
     lyaps[i] = lyapunov(ds, N_lyap)
 
     x = trajectory(ds, N_ent) # time series
-    hperm = entropy(x, SymbolicPermutation(; m, τ))
-    hwtperm = entropy(x, SymbolicWeightedPermutation(; m, τ))
-    hampperm = entropy(x, SymbolicAmplitudeAwarePermutation(; m, τ))
+    hperm = entropy(SymbolicPermutation(; m, τ), x)
+    hwtperm = entropy(SymbolicWeightedPermutation(; m, τ), x)
+    hampperm = entropy(SymbolicAmplitudeAwarePermutation(; m, τ), x)
 
     hs_perm[i] = hperm; hs_wtperm[i] = hwtperm; hs_ampperm[i] = hampperm
 end
@@ -173,7 +173,7 @@ using DynamicalSystemsBase, CairoMakie, Distributions
 N = 500
 D = Dataset(sort([rand(𝒩) for i = 1:N]))
 x, y = columns(D)
-p = probabilities(D, NaiveKernel(1.5))
+p = probabilities(NaiveKernel(1.5), D)
 fig, ax = scatter(D[:, 1], D[:, 2], zeros(N);
     markersize=8, axis=(type = Axis3,)
 )
@@ -301,9 +301,9 @@ des = zeros(length(windows))
 pes = zeros(length(windows))
 
 m, c = 2, 6
-est_de = Dispersion(encoding = GaussianCDFEncoding(c), m = m, τ = 1)
+est_de = Dispersion(c = c, m = m, τ = 1)
 for (i, window) in enumerate(windows)
-    des[i] = entropy_normalized(Renyi(), y[window], est_de)
+    des[i] = entropy_normalized(Renyi(), est_de, y[window])
 end
 
 fig = Figure()
@@ -344,8 +344,8 @@ for N in (N1, N2)
     local w = trajectory(Systems.lorenz(), N÷10; Δt = 0.1, Ttr = 100)[:, 1] # chaotic
 
     for q in (x, y, z, w)
-        h = entropy(q, PowerSpectrum())
-        n = entropy_normalized(q, PowerSpectrum())
+        h = entropy(PowerSpectrum(), q)
+        n = entropy_normalized(PowerSpectrum(), q)
         println("entropy: $(h), normalized: $(n).")
     end
 end

docs/src/index.md

@@ -40,7 +40,11 @@ Thus, any of the implemented [probabilities estimators](@ref probabilities_estim
 
     These names are common place, and so in Entropies.jl we provide convenience functions like [`entropy_wavelet`](@ref). However, it should be noted that these functions really aren't anything more than 2-lines-of-code wrappers that call [`entropy`](@ref) with the appropriate [`ProbabilitiesEstimator`](@ref).
 
-    There are only a few exceptions to this rule, which are quantities that are able to compute Shannon entropies via alternate means, without explicitly computing some probability distributions. These are `IndirectEntropy` instances, such as [`Kraskov`](@ref).
+    In addition to `ProbabilitiesEstimators`, we also provide [`EntropyEstimator`](@ref)s, 
+    which compute entropies via alternate means, without explicitly computing some 
+    probability distribution. For example, [`Kraskov`](@ref) estimator computes Shannon 
+    entropy via a nearest neighbor algorithm, while the [`Zhu`](@ref) estimator computes
+    Shannon entropy using order statistics.
 
 ### Other complexity measures
 

src/deprecations.jl

@@ -8,6 +8,14 @@ function probabilities(x::Vector_or_Dataset, ε::Union{Real, Vector{<:Real}})
     probabilities(x, ValueHistogram(ε))
 end
 
+function probabilities(x, est::ProbabilitiesEstimator)
+    @warn """
+    `probabilities(x, est::ProbabilitiesEstimator)`
+    is deprecated, use `probabilities(est::ProbabilitiesEstimator, x) instead`.
+    """
+    return probabilities(est, x)
+end
+
 export genentropy, permentropy
 
 function permentropy(x; τ = 1, m = 3, base = MathConstants.e)

src/entropies/convenience_definitions.jl

@@ -23,26 +23,26 @@ for the weighted/amplitude-aware versions.
 """
 function entropy_permutation(x; base = 2, kwargs...)
     est = SymbolicPermutation(; kwargs...)
-    entropy(Shannon(; base), x, est)
+    entropy(Shannon(; base), est, x)
 end
 
 """
-    entropy_spatial_permutation(x, stencil, periodic = true; kwargs...)
+    entropy_spatial_permutation(x, stencil; periodic = true; kwargs...)
 
 Compute the spatial permutation entropy of `x` given the `stencil`.
 Here `x` must be a matrix or higher dimensional `Array` containing spatial data.
 This function is just a convenience call to:
 
 ```julia
 est = SpatialSymbolicPermutation(stencil, x, periodic)
-entropy(Renyi(;kwargs...), x, est)
+entropy(Renyi(;kwargs...), est, x)
 ```
 
 See [`SpatialSymbolicPermutation`](@ref) for more info, or how to encode stencils.
 """
-function entropy_spatial_permutation(x, stencil, periodic = true; kwargs...)
+function entropy_spatial_permutation(x, stencil; periodic = true, kwargs...)
     est = SpatialSymbolicPermutation(stencil, x, periodic)
-    entropy(Renyi(;kwargs...), x, est)
+    entropy(Renyi(;kwargs...), est, x)
 end
 
 """
@@ -52,14 +52,14 @@ Compute the wavelet entropy. This function is just a convenience call to:
 
 ```julia
 est = WaveletOverlap(wavelet)
-entropy(Shannon(base), x, est)
+entropy(Shannon(base), est, x)
 ```
 
 See [`WaveletOverlap`](@ref) for more info.
 """
 function entropy_wavelet(x; wavelet = Wavelets.WT.Daubechies{12}(), base = 2)
     est = WaveletOverlap(wavelet)
-    entropy(Shannon(; base), x, est)
+    entropy(Shannon(; base), est, x)
 end
 
 """
@@ -69,12 +69,12 @@ Compute the dispersion entropy. This function is just a convenience call to:
 
 ```julia
 est = Dispersion(kwargs...)
-entropy(Shannon(base), x, est)
+entropy(Shannon(base), est, x)
 ```
 
 See [`Dispersion`](@ref) for more info.
 """
 function entropy_dispersion(x; base = 2, kwargs...)
     est = Dispersion(kwargs...)
-    entropy(Shannon(; base), x, est)
+    entropy(Shannon(; base), est, x)
 end

src/entropies/entropies.jl

@@ -3,4 +3,4 @@ include("tsallis.jl")
 include("curado.jl")
 include("streched_exponential.jl")
 include("convenience_definitions.jl")
-include("direct_entropies/direct_entropies.jl")
+include("estimators/estimators.jl")

src/entropies/direct_entropies/nearest_neighbors/KozachenkoLeonenko.jl → src/entropies/estimators/nearest_neighbors/KozachenkoLeonenko.jl

@@ -1,32 +1,38 @@
 export KozachenkoLeonenko
 
 """
-    KozachenkoLeonenko <: IndirectEntropy
+    KozachenkoLeonenko <: EntropyEstimator
     KozachenkoLeonenko(; k::Int = 1, w::Int = 1, base = 2)
 
-An indirect entropy estimator used in [`entropy`](@ref)`(KozachenkoLeonenko(), x)` to
-estimate the Shannon entropy of `x` (a multi-dimensional `Dataset`) to the given
-`base` using nearest neighbor searches using the method from Kozachenko &
-Leonenko[^KozachenkoLeonenko1987], as described in Charzyńska and Gambin[^Charzyńska2016].
+The `KozachenkoLeonenko` estimator computes the [`Shannon`](@ref) [`entropy`](@ref) of `x`
+(a multi-dimensional `Dataset`) to the given `base`, based on nearest neighbor searches
+using the method from Kozachenko & Leonenko (1987)[^KozachenkoLeonenko1987], as described in
+Charzyńska and Gambin[^Charzyńska2016].
 
 `w` is the Theiler window, which determines if temporal neighbors are excluded
 during neighbor searches (defaults to `0`, meaning that only the point itself is excluded
 when searching for neighbours).
 
 In contrast to [`Kraskov`](@ref), this estimator uses only the *closest* neighbor.
 
+See also: [`entropy`](@ref).
+
 [^Charzyńska2016]: Charzyńska, A., & Gambin, A. (2016). Improvement of the k-NN entropy
     estimator with applications in systems biology. Entropy, 18(1), 13.
 [^KozachenkoLeonenko1987]: Kozachenko, L. F., & Leonenko, N. N. (1987). Sample estimate of
     the entropy of a random vector. Problemy Peredachi Informatsii, 23(2), 9-16.
 """
-@Base.kwdef struct KozachenkoLeonenko{B} <: IndirectEntropy
+@Base.kwdef struct KozachenkoLeonenko{B} <: EntropyEstimator
     w::Int = 1
     base::B = 2
 end
 
-function entropy(e::KozachenkoLeonenko, x::AbstractDataset{D, T}) where {D, T}
-    (; w, base) = e
+function entropy(e::Renyi, est::KozachenkoLeonenko, x::AbstractDataset{D, T}) where {D, T}
+    e.q == 1 || throw(ArgumentError(
+        "Renyi entropy with q = $(e.q) not implemented for $(typeof(est)) estimator"
+    ))
+    (; w, base) = est
+
     N = length(x)
     ρs = maximum_neighbor_distances(x, w, 1)
     # The estimated entropy has "unit" [nats]

src/entropies/direct_entropies/nearest_neighbors/Kraskov.jl → src/entropies/estimators/nearest_neighbors/Kraskov.jl

@@ -1,31 +1,34 @@
 export Kraskov
 
 """
-    Kraskov <: IndirectEntropy
+    Kraskov <: EntropyEstimator
     Kraskov(; k::Int = 1, w::Int = 1, base = 2)
 
-An indirect entropy used in [`entropy`](@ref)`(Kraskov(), x)` to estimate the Shannon
-entropy of `x` (a multi-dimensional `Dataset`) to the given
-`base` using `k`-th nearest neighbor searches as in [^Kraskov2004].
+The `Kraskov` estimator computes the [`Shannon`](@ref) [`entropy`](@ref) of `x`
+(a multi-dimensional `Dataset`) to the given `base`, using the `k`-th nearest neighbor
+searches method from [^Kraskov2004].
 
 `w` is the Theiler window, which determines if temporal neighbors are excluded
 during neighbor searches (defaults to `0`, meaning that only the point itself is excluded
 when searching for neighbours).
 
-See also: [`KozachenkoLeonenko`](@ref).
+See also: [`entropy`](@ref), [`KozachenkoLeonenko`](@ref).
 
 [^Kraskov2004]:
     Kraskov, A., Stögbauer, H., & Grassberger, P. (2004).
     Estimating mutual information. Physical review E, 69(6), 066138.
 """
-Base.@kwdef struct Kraskov{B} <: IndirectEntropy
+Base.@kwdef struct Kraskov{B} <: EntropyEstimator
     k::Int = 1
     w::Int = 1
     base::B = 2
 end
 
-function entropy(e::Kraskov, x::AbstractDataset{D, T}) where {D, T}
-    (; k, w, base) = e
+function entropy(e::Renyi, est::Kraskov, x::AbstractDataset{D, T}) where {D, T}
+    e.q == 1 || throw(ArgumentError(
+        "Renyi entropy with q = $(e.q) not implemented for $(typeof(est)) estimator"
+    ))
+    (; k, w, base) = est
     N = length(x)
     ρs = maximum_neighbor_distances(x, w, k)
     # The estimated entropy has "unit" [nats]

src/entropies/direct_entropies/nearest_neighbors/Zhu.jl → src/entropies/estimators/nearest_neighbors/Zhu.jl

@@ -1,38 +1,43 @@
 export Zhu
 
 """
-    Zhu <: IndirectEntropy
-    Zhu(k = 1, w = 0, base = MathConstants.e)
+    Zhu <: EntropyEstimator
+    Zhu(k = 1, w = 0)
 
-The `Zhu` indirect entropy estimator (Zhu et al., 2015)[^Zhu2015] estimates the Shannon
-entropy of `x` (a multi-dimensional `Dataset`) to the given `base`, by approximating
-probabilities within hyperrectangles surrounding each point `xᵢ ∈ x` using
+The `Zhu` estimator (Zhu et al., 2015)[^Zhu2015] computes the [`Shannon`](@ref)
+[`entropy`](@ref) of `x` (a multi-dimensional `Dataset`), by
+approximating probabilities within hyperrectangles surrounding each point `xᵢ ∈ x` using
 using `k` nearest neighbor searches.
 
-This estimator is an extension to [`KozachenkoLeonenko`](@ref).
-
 `w` is the Theiler window, which determines if temporal neighbors are excluded
 during neighbor searches (defaults to `0`, meaning that only the point itself is excluded
 when searching for neighbours).
 
+This estimator is an extension to [`KozachenkoLeonenko`](@ref).
+
+See also: [`entropy`](@ref).
+
 [^Zhu2015]:
     Zhu, J., Bellanger, J. J., Shu, H., & Le Bouquin Jeannès, R. (2015). Contribution to
     transfer entropy estimation via the k-nearest-neighbors approach. Entropy, 17(6),
     4173-4201.
 """
-Base.@kwdef struct Zhu{B} <: IndirectEntropy
+Base.@kwdef struct Zhu <: EntropyEstimator
     k::Int = 1
     w::Int = 0
-    base::B = MathConstants.e
 end
 
-function entropy(e::Zhu, x::AbstractDataset{D, T}) where {D, T}
-    (; k, w, base) = e
+function entropy(e::Renyi, est::Zhu, x::AbstractDataset{D, T}) where {D, T}
+    e.q == 1 || throw(ArgumentError(
+        "Renyi entropy with q = $(e.q) not implemented for $(typeof(est)) estimator"
+    ))
+    (; k, w) = est
+
     N = length(x)
     tree = KDTree(x, Euclidean())
     nn_idxs = bulkisearch(tree, x, NeighborNumber(k), Theiler(w))
     h = digamma(N) + mean_logvolumes(x, nn_idxs, N) - digamma(k) + (D - 1) / k
-    return h / log(base, MathConstants.e)
+    return h / log(e.base, MathConstants.e)
 end
 
 function mean_logvolumes(x, nn_idxs, N::Int)

src/entropies/direct_entropies/nearest_neighbors/ZhuSingh.jl → src/entropies/estimators/nearest_neighbors/ZhuSingh.jl

@@ -1,16 +1,16 @@
 using DelayEmbeddings: minmaxima
 using SpecialFunctions: digamma
-using Entropies: Entropy, IndirectEntropy
+using Entropies: Entropy, EntropyEstimator
 using Neighborhood: KDTree, Chebyshev, bulkisearch, Theiler, NeighborNumber
 
 export ZhuSingh
 
 """
-    ZhuSingh <: IndirectEntropy
-    ZhuSingh(k = 1, w = 0, base = MathConstants.e)
+    ZhuSingh <: EntropyEstimator
+    ZhuSingh(k = 1, w = 0)
 
-The `ZhuSingh` indirect entropy estimator (Zhu et al., 2015)[^Zhu2015] estimates the Shannon
-entropy of `x` (a multi-dimensional `Dataset`) to the given `base`.
+The `ZhuSingh` estimator (Zhu et al., 2015)[^Zhu2015] computes the [`Shannon`](@ref)
+[`entropy`](@ref) of `x` (a multi-dimensional `Dataset`).
 
 Like [`Zhu`](@ref), this estimator approximates probabilities within hyperrectangles
 surrounding each point `xᵢ ∈ x` using using `k` nearest neighbor searches. However,
@@ -21,6 +21,8 @@ This estimator is an extension to the entropy estimator in Singh et al. (2003).
 during neighbor searches (defaults to `0`, meaning that only the point itself is excluded
 when searching for neighbours).
 
+See also: [`entropy`](@ref).
+
 [^Zhu2015]:
     Zhu, J., Bellanger, J. J., Shu, H., & Le Bouquin Jeannès, R. (2015). Contribution to
     transfer entropy estimation via the k-nearest-neighbors approach. Entropy, 17(6),
@@ -30,24 +32,22 @@ when searching for neighbours).
     neighbor estimates of entropy. American journal of mathematical and management
     sciences, 23(3-4), 301-321.
 """
-Base.@kwdef struct ZhuSingh{B} <: IndirectEntropy
+Base.@kwdef struct ZhuSingh <: EntropyEstimator
     k::Int = 1
     w::Int = 0
-    base::B = MathConstants.e
-
-    function ZhuSingh(k::Int, w::Int, base::B) where B
-        new{B}(k, w, base)
-    end
 end
 
-function entropy(e::ZhuSingh, x::AbstractDataset{D, T}) where {D, T}
-    (; k, w, base) = e
+function entropy(e::Renyi, est::ZhuSingh, x::AbstractDataset{D, T}) where {D, T}
+    e.q == 1 || throw(ArgumentError(
+        "Renyi entropy with q = $(e.q) not implemented for $(typeof(est)) estimator"
+    ))
+    (; k, w) = est
     N = length(x)
     tree = KDTree(x, Euclidean())
     nn_idxs = bulkisearch(tree, x, NeighborNumber(k), Theiler(w))
     mean_logvol, mean_digammaξ = mean_logvolumes_and_digamma(x, nn_idxs, N, k)
     h = digamma(N) + mean_logvol - mean_digammaξ
-    return h / log(base, MathConstants.e)
+    return h / log(e.base, MathConstants.e)
 end
 
 function mean_logvolumes_and_digamma(x, nn_idxs, N::Int, k::Int)