Skip to content

Commit

Permalink
Merge pull request #2241 from AllenInstitute/bugfix/2241-adaptive-sup…
Browse files Browse the repository at this point in the history 
…ra-dascale-calculation

PSQ_DAScale/PSQ_DS_ADAPT: Fix DAScale estimation after fill-in
  • Loading branch information
@t-b
t-b authored Sep 18, 2024
2 parents 1a6cb1f + 4549ee5 commit 831972d
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 49 deletions.
108 changes: 67 additions & 41 deletions Packages/MIES/MIES_AnalysisFunctions_PatchSeq.ipf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,10 +185,11 @@ static Constant PSQ_DA_FALLBACK_DASCALE_RANGE_FAC = 1.5
///@{
static Constant PSQ_DS_FI_SLOPE = 0x1
static Constant PSQ_DS_FI_SLOPE_REACHED_PASS = 0x2
static Constant PSQ_DS_SWEEP_PASS = 0x3
static Constant PSQ_DS_SWEEP = 0x4
static Constant PSQ_DS_APFREQ = 0x5
static Constant PSQ_DS_DASCALE = 0x6
static Constant PSQ_DS_FI_OFFSET = 0x3
static Constant PSQ_DS_SWEEP_PASS = 0x4
static Constant PSQ_DS_SWEEP = 0x5
static Constant PSQ_DS_APFREQ = 0x6
static Constant PSQ_DS_DASCALE = 0x7
///@}

/// @brief Fills `s` according to the analysis function type
Expand Down Expand Up @@ -2264,8 +2265,11 @@ static Function PSQ_DS_CreateSurveyPlotForUser(string device, variable sweepNo,
fromRhSuAd = !!fromRhSuAd
endif

[WAVE apfreq, emptySCI] = PSQ_DS_GetLabnotebookData(device, sweepNo, headstage, PSQ_DS_APFREQ, filterPassing = 1, fromRhSuAd = fromRhSuAd)
[WAVE DAScale, emptySCI] = PSQ_DS_GetLabnotebookData(device, sweepNo, headstage, PSQ_DS_DASCALE, filterPassing = 1, fromRhSuAd = fromRhSuAd)
WAVE numericalValues = GetLBNumericalValues(device)
WAVE textualValues = GetLBTextualValues(device)

[WAVE apfreq, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_APFREQ, filterPassing = 1, fromRhSuAd = fromRhSuAd)
[WAVE DAScale, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_DASCALE, filterPassing = 1, fromRhSuAd = fromRhSuAd)

/// PSQ_FMT_LBN_DA_AT_RSA_FREQ
/// PSQ_FMT_LBN_DA_AT_RSA_DASCALE
Expand All @@ -2288,7 +2292,7 @@ static Function PSQ_DS_CreateSurveyPlotForUser(string device, variable sweepNo,
sprintf line, "daScaleWithoutFirst = [%s]\r", NumericWaveToList(DAScale, ",", format = PERCENT_F_MAX_PREC, trailSep = 0)
str += line

[WAVE fitSlopes, emptySCI] = PSQ_DS_GetLabnotebookData(device, sweepNo, headstage, PSQ_DS_FI_SLOPE, filterPassing = 1, fromRhSuAd = fromRhSuAd)
[WAVE fitSlopes, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_FI_SLOPE, filterPassing = 1, fromRhSuAd = fromRhSuAd)

/// PSQ_FMT_LBN_DA_AT_RSA_FI_SLOPES
///
Expand Down Expand Up @@ -2359,7 +2363,7 @@ static Function PSQ_DS_AreFitResultsValid(string device, variable sweepNo, varia
return validFit
End

static Function [variable daScaleStepMinNorm, variable daScaleStepMaxNorm] PSQ_DS_GetDAScaleStepsNorm(string device, variable sweepNo, variable headstage)
Function [variable daScaleStepMinNorm, variable daScaleStepMaxNorm] PSQ_DS_GetDAScaleStepsNorm(string device, variable sweepNo, variable headstage)

string key

Expand Down Expand Up @@ -2528,15 +2532,16 @@ static Function [variable maxSlope, WAVE fitSlopes, WAVE DAScales] PSQ_DS_Calcul
string key
variable idx, emptySCI

[WAVE fitSlopes, emptySCI] = PSQ_DS_GetLabnotebookData(device, sweepNo, headstage, PSQ_DS_FI_SLOPE)
[WAVE DAScales, emptySCI] = PSQ_DS_GetLabnotebookData(device, sweepNo, headstage, PSQ_DS_DASCALE)
WAVE numericalValues = GetLBNumericalValues(device)
WAVE textualValues = GetLBTextualValues(device)

[WAVE fitSlopes, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_FI_SLOPE)
[WAVE DAScales, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_DASCALE)

if(IsNaN(fitSlope))
return [NaN, fitSlopes, DAScales]
endif

WAVE numericalValues = GetLBNumericalValues(device)

maxSlope = max(WaveMax(fitSlopes), fitSlope)

WAVE maxSlopeLBN = LBN_GetNumericWave()
Expand Down Expand Up @@ -2603,14 +2608,10 @@ End
/// - Check if the resulting fit slope is smaller than `slopePercentage`
///
/// @retval futureDAScales future DAScale values including the historic ones, @see PSQ_DS_GatherFutureDAScalesAndFrequency
/// @retval fitOffset offset of the linear curve fit
/// @retval fitSlope slope of the linear curve fit
/// @retval DAScale current DAScale value
/// @retval apfreq current AP frequency value
static Function [WAVE futureDAScales, variable fitOffset, variable fitSlope, variable DAScale, variable apfreq] PSQ_DS_EvaluateAdaptiveThresholdSweep(string device, variable sweepNo, variable headstage, STRUCT PSQ_DS_DAScaleParams &cdp)
static Function [WAVE futureDAScales] PSQ_DS_EvaluateAdaptiveThresholdSweep(string device, variable sweepNo, variable headstage, STRUCT PSQ_DS_DAScaleParams &cdp)

string key, errMsg
variable maxSlope, validFit
variable maxSlope, validFit, fitOffset, fitSlope

WAVE textualValues = GetLBTextualValues(device)
WAVE numericalValues = GetLBNumericalValues(device)
Expand Down Expand Up @@ -2642,23 +2643,11 @@ static Function [WAVE futureDAScales, variable fitOffset, variable fitSlope, var
fitSlope = NaN
endif

if(WaveExists(apfreqs))
apfreq = apfreqs[Inf]
else
apfreq = NaN
endif

if(WaveExists(DAScales))
DAScale = DAScales[Inf]
else
DAScale = NaN
endif

[maxSlope, WAVE fitSlopesAll, WAVE DAScalesAll] = PSQ_DS_CalculateMaxSlopeAndWriteToLabnotebook(device, sweepNo, headstage, fitSlope)

PSQ_DS_CalculateReachedFinalSlopeAndWriteToLabnotebook(device, sweepNo, fitSlopesAll, DAScalesAll, fitSlope, maxSlope, cdp.slopePercentage, validFit)

return [futureDAScales, fitOffset, fitSlope, DAScale, apfreq]
return [futureDAScales]
End

/// @brief Determine from the POST_SWEEP_EVENT if the set is already finished
Expand Down Expand Up @@ -2725,6 +2714,8 @@ static Function [string currentSCI, string RhSuAd, variable headstageContingency
return [PSQ_FMT_LBN_DA_fI_SLOPE, PSQ_FMT_LBN_DA_AT_RSA_fI_SLOPES, HCM_DEPEND]
case PSQ_DS_FI_SLOPE_REACHED_PASS:
return [PSQ_FMT_LBN_DA_fI_SLOPE_REACHED_PASS, PSQ_FMT_LBN_DA_AT_RSA_fI_SLOPES_PASS, HCM_INDEP]
case PSQ_DS_FI_OFFSET:
return [PSQ_FMT_LBN_DA_AT_FI_OFFSET, PSQ_FMT_LBN_DA_AT_RSA_FI_OFFSETS, HCM_DEPEND]
case PSQ_DS_SWEEP_PASS:
return [PSQ_FMT_LBN_SWEEP_PASS, PSQ_FMT_LBN_DA_AT_RSA_SWEEPS, HCM_INDEP]
case PSQ_DS_SWEEP:
Expand All @@ -2739,7 +2730,7 @@ static Function [string currentSCI, string RhSuAd, variable headstageContingency
End

/// @brief Return some labnotebook entries for all passing sweeps from RhSuAd and the current SCI
static Function [WAVE data, variable emptySCI] PSQ_DS_GetLabnotebookData(string device, variable sweepNo, variable headstage, variable type, [variable filterPassing, variable beforeSweepQCResult, variable fromRhSuAd])
static Function [WAVE data, variable emptySCI] PSQ_DS_GetLabnotebookData(WAVE numericalValues, WAVE textualValues, variable sweepNo, variable headstage, variable type, [variable filterPassing, variable beforeSweepQCResult, variable fromRhSuAd])

string key, keyCurrentSCI, keyRhSuAd
variable hcmMode, idx
Expand All @@ -2762,9 +2753,6 @@ static Function [WAVE data, variable emptySCI] PSQ_DS_GetLabnotebookData(string
filterPassing = !!filterPassing
endif

WAVE numericalValues = GetLBNumericalValues(device)
WAVE textualValues = GetLBTextualValues(device)

[keyCurrentSCI, keyRhSuAd, hcmMode] = PSQ_DS_GetLabnotebookKeyConstants(type)

if(fromRhSuAd)
Expand Down Expand Up @@ -2910,13 +2898,13 @@ static Function PSQ_DS_AdaptiveIsFinished(string device, variable sweepNo, varia
endif
endif

[WAVE sweepPassed, emptySCI] = PSQ_DS_GetLabnotebookData(device, sweepNo, headstage, PSQ_DS_SWEEP_PASS, fromRhSuAd = fromRhSuAd)
[WAVE sweepPassed, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_SWEEP_PASS, fromRhSuAd = fromRhSuAd)

if(DimSize(sweepPassed, ROWS) < numSweepsWithSaturation)
return 0
endif

[WAVE fitSlopeReached, emptySCI] = PSQ_DS_GetLabnotebookData(device, sweepNo, headstage, PSQ_DS_FI_SLOPE_REACHED_PASS, fromRhSuAd = fromRhSuAd)
[WAVE fitSlopeReached, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_FI_SLOPE_REACHED_PASS, fromRhSuAd = fromRhSuAd)

// slope is determined between sweeps so we have always one sweep more than slopes
ASSERT(DimSize(sweepPassed, ROWS) == (DimSize(fitSlopeReached, ROWS) + 1), "Unmatched sweepPassed and fitSlopeReached waves")
Expand Down Expand Up @@ -2960,8 +2948,8 @@ static Function [WAVE futureDAScales, WAVE apfreq, WAVE DAScales] PSQ_DS_GatherF
Make/FREE/D/N=0 futureDAScalesHistoric
endif

[WAVE DAScales, emptySCI] = PSQ_DS_GetLabnotebookData(device, sweepNo, headstage, PSQ_DS_DASCALE, filterPassing = 1, beforeSweepQCResult = 1)
[WAVE apfreq, emptySCI] = PSQ_DS_GetLabnotebookData(device, sweepNo, headstage, PSQ_DS_APFREQ, filterPassing = 1, beforeSweepQCResult = 1)
[WAVE DAScales, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_DASCALE, filterPassing = 1, beforeSweepQCResult = 1)
[WAVE apfreq, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_APFREQ, filterPassing = 1, beforeSweepQCResult = 1)

if(!emptySCI)
WAVE/D/Z futureDAScalesFromLastSweep = PSQ_DS_GatherOvershootCorrection(cdp, apfreq, DAScales, futureDAScalesHistoric)
Expand Down Expand Up @@ -3219,6 +3207,43 @@ static Function [variable daScaleStepMinNorm, variable daScaleStepMaxNorm] PSQ_D
return [daScaleMinStepWidth, daScaleMaxStepWidth]
End

static Function [variable fitOffset, variable fitSlope, variable DAScale, variable apfreq] PSQ_DS_GetValuesOfLargestDAScale(WAVE numericalValues, WAVE/T textualValues, variable sweepNo, variable headstage)

variable emptySCI, offset, i, numEntries, maxValue, maxLoc

[WAVE apfreqAll, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_APFREQ, filterPassing = 1)
ASSERT(!emptySCI, "Unexpected emptySCI")
[WAVE fitSlopeAll, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_FI_SLOPE, filterPassing = 1)
ASSERT(!emptySCI, "Unexpected emptySCI")
[WAVE fitOffsetAll, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_FI_OFFSET, filterPassing = 1)
ASSERT(!emptySCI, "Unexpected emptySCI")
[WAVE DAScaleAll, emptySCI] = PSQ_DS_GetLabnotebookData(numericalValues, textualValues, sweepNo, headstage, PSQ_DS_DASCALE, filterPassing = 1)
ASSERT(!emptySCI, "Unexpected emptySCI")

// get the largest value with the highest index
maxValue = -Inf
numEntries = DimSize(DAScaleAll, ROWS)
for(i = numEntries - 1; i >= 0; i -= 1)
if(DAScaleAll[i] > maxValue)
maxValue = DAScaleAll[i]
maxLoc = i
endif
endfor

// highest DAScale/APFreq is at maxloc
apfreq = apfreqAll[maxLoc]
DAScale = DAScaleAll[maxLoc]

// find the rest by indexing relative from the end as we have fewer fitOffset/fitSlope entries in total
// but a 1:1 correspondence in the SCI data which is at the end
offset = DimSize(DAScaleAll, ROWS) - maxLoc

fitOffset = fitOffsetAll[DimSize(fitOffsetAll, ROWS) - offset]
fitSlope = fitSlopeAll[DimSize(fitSlopeAll, ROWS) - offset]

return [fitOffset, fitSlope, DAScale, apfreq]
End

static Structure PSQ_DS_DAScaleParams
variable absFrequencyMinDistance, maxFrequencyChangePercent
variable daScaleStepMinNorm, daScaleStepMaxNorm, slopePercentage
Expand Down Expand Up @@ -3676,7 +3701,7 @@ Function PSQ_DAScale(device, s)
variable sweepPassed, setPassed, length, minLength, reachedFinalSlope, fitOffset, fitSlope, apfreq, enoughFIPointsPassedQC
variable minimumSpikeCount, maximumSpikeCount, daScaleModifierParam, measuredAllFutureDAScales, fallbackDAScaleRangeFac
variable sweepsInSet, passesInSet, acquiredSweepsInSet, multiplier, asyncAlarmPassed, supraStimsetCycle
variable daScaleStepMinNorm, daScaleStepMaxNorm, maxSlope, validFit
variable daScaleStepMinNorm, daScaleStepMaxNorm, maxSlope, validFit, emptySCI
string msg, stimset, key, opMode, offsetOp, textboxString, str, errMsg
variable daScaleOffset
variable finalSlopePercent = NaN
Expand Down Expand Up @@ -3952,7 +3977,7 @@ Function PSQ_DAScale(device, s)
cdp.daScaleStepMinNorm = daScaleStepMinNorm
cdp.daScaleStepMaxNorm = daScaleStepMaxNorm

[WAVE futureDAScales, fitOffset, fitSlope, DAScale, apfreq] = PSQ_DS_EvaluateAdaptiveThresholdSweep(device, s.sweepNo, s.headstage, cdp)
[WAVE futureDAScales] = PSQ_DS_EvaluateAdaptiveThresholdSweep(device, s.sweepNo, s.headstage, cdp)

// prepare wave for setting DAScale; DAScalesIndex is incremented when required below
WAVE/ZZ DAScales = futureDAScales
Expand Down Expand Up @@ -4155,6 +4180,7 @@ Function PSQ_DAScale(device, s)

if(sweepPassed)
if(measuredAllFutureDAScales)
[fitOffset, fitSlope, DAScale, apfreq] = PSQ_DS_GetValuesOfLargestDAScale(numericalValues, textualValues, s.sweepNo, s.headstage)

dascale = PSQ_DS_CalculateDAScale(cdp, fitOffset, fitSlope, DAScale, apfreq)
Make/FREE/D DAScaleNew = {dascale}
Expand Down
2 changes: 1 addition & 1 deletion Packages/MIES/MIES_Constants.ipf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ Constant RESULTS_VERSION = 3
Constant PSQ_PIPETTE_BATH_VERSION = 4
Constant PSQ_ACC_RES_SMOKE_VERSION = 2
Constant PSQ_CHIRP_VERSION = 13
Constant PSQ_DA_SCALE_VERSION = 8
Constant PSQ_DA_SCALE_VERSION = 11
Constant PSQ_RAMP_VERSION = 6
Constant PSQ_RHEOBASE_VERSION = 5
Constant PSQ_SQUARE_PULSE_VERSION = 4
Expand Down
2 changes: 1 addition & 1 deletion Packages/doc/dot/patch-seq-dascale.dot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ QC and f-I slope QC passing?"];
n109 -> n100 [label=Yes];
n103 [label="Measured all future DAScale values?"];
n100 -> n103 [label=Yes];
n106 [label="Calculate new DAScale value by\n extrapolating fit slope and offset\n and using MaximumChangePercent - 2 as frequency distance"];
n106 [label="Calculate new DAScale value by\n extrapolating fit slope and offset\n and using MaximumChangePercent - 2 as frequency distance.\n Uses the fit slopes and offsets from the\n passing sweep with the highest DAScale searching from the back."];
n103 -> n106 [label=Yes];
n108 [label="Use the next DAScale value"];
n103 -> n108 [label=No];
Expand Down
11 changes: 5 additions & 6 deletions Packages/tests/HardwareAnalysisFunctions/UTF_PatchSeqDAScale_Adapt.ipf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1534,14 +1534,14 @@ static Function PS_DS_AD10_REENTRY([string str])
CHECK_EQUAL_WAVES(entries[%dascaleFromRhSuAd], DAScalesFromRhSuAd, mode = WAVE_DATA)
CHECK_EQUAL_WAVES(entries[%sweepPassFromRhSuAd], sweepPassedFRomRhSuAd, mode = WAVE_DATA)

Make/FREE/D maxSlopeRef = {9.374999999999962e-10, 3.124999999999997e-09, 6.510416666666681e-09}
Make/FREE/D fiSlopeRef = {9.374999999999962e-10, 3.124999999999997e-09, 6.510416666666681e-09}
Make/FREE/D fiOffsetRef = {-21.49999999999984, -129.9999999999999, -314.1666666666805}
Make/FREE/D maxSlopeRef = {9.374999999999962e-10, 3.125000000000005e-09, 6.510416666666915e-09}
Make/FREE/D fiSlopeRef = {9.374999999999962e-10, 3.125000000000005e-09, 6.510416666666915e-09}
Make/FREE/D fiOffsetRef = {-21.49999999999984, -130.0000000000003, -314.1666666666805}
Make/FREE/T futureDAScalesRef = {"4.96;5.44;", "4.96;5.44;5.6704;", "4.96;5.44;5.6704;5.82246399999999;"}

Make/FREE/D fiSlopesFromRhSuAdRef = {1e-10, 2e-10, 3e-10}
Make/FREE/D fiOffsetsFromRhSuAdRef = {9, 7, 4}
Make/FREE/D DAScalesRef = {4.96, 5.440000000000003, 5.670400000000001}
Make/FREE/D DAScalesRef = {4.96, 5.440000000000003, 5.6704}

CHECK_EQUAL_WAVES(entries[%maxSlope], maxSlopeRef, mode = WAVE_DATA, tol = 1e-24)
CHECK_EQUAL_WAVES(entries[%fiSlope], fiSlopeRef, mode = WAVE_DATA, tol = 1e-24)
Expand Down Expand Up @@ -1768,7 +1768,6 @@ static Function PS_DS_AD12_REENTRY([string str])
Make/FREE/D fiSlopeRef = {3.416149068322985e-10, 3.416149068322985e-10, -2.376451525789897e-10, -4.078983962177433e-12}
Make/FREE/D fiOffsetRef = {2.043478260869549, 2.043478260869549, 40.9697542533081, 21.34276443867624}
Make/FREE/T futureDAScalesRef = {"6.72;8.40318181818182;", "6.72;8.40318181818182;", "6.72;8.40318181818182;5.95159090909091;", "6.72;8.40318181818182;5.95159090909091;"}

Make/FREE/D fiSlopesFromRhSuAdRef = {2e-10, 2e-10, 1e-10}
Make/FREE/D fiOffsetsFromRhSuAdRef = {8, 8, 10.5}
Make/FREE/D DAScalesRef = {6.719999999999999, 8.403181818181814, 8.403181818181814, 5.951590909090905}
Expand Down Expand Up @@ -2094,7 +2093,7 @@ static Function PS_DS_AD15_REENTRY([string str])
Make/FREE/D fiOffsetRef = {0.5555555555555558, 0.5555555555555558, 0.5555555555555558}
Make/FREE/T futureDAScalesRef = {"10;40;", \
"10;40;21;", \
"10;40;21;51;"}
"10;40;21;70;"}

Make/FREE/D fiSlopesFromRhSuAdRef = {0, PSQ_DS_SKIPPED_FI_SLOPE, PSQ_DS_SKIPPED_FI_SLOPE}
Make/FREE/D fiOffsetsFromRhSuAdRef = {1, PSQ_DS_SKIPPED_FI_SLOPE, PSQ_DS_SKIPPED_FI_SLOPE}
Expand Down

0 comments on commit 831972d

Please sign in to comment.