diff --git a/tests/hf_cache.json b/tests/hf_cache.json
new file mode 100644
index 000000000..61c6a0b69
--- /dev/null
+++ b/tests/hf_cache.json
@@ -0,0 +1 @@
+{"ibm-ai-platform/micro-g3.3-8b-instruct-1b": {"__tokens__27400_438_600_12404_688_18872_312_2899_32_5950_312_1789_688_36808_30772_322_1326_32_4261_7743_659_328_1370_1789_372_322_1256_32_203_203_1482_21081_44_203_15277_312_1149_432_9400_436_1406_26124_663_21217_31628_7947_623_540_565": {"65": {"text": ".\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with r", "token_ids": [32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540], "tokens": [".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r"], "logprobs": [-1.1383615732192993, -1.348477840423584, -0.6464436054229736, -1.0950212478637695, -1.1007599830627441, -0.040145955979824066, -0.013641467317938805, -0.4686796963214874, -0.35953083634376526, -0.1965043991804123, -0.005137456580996513, -0.08900478482246399, -0.0903414785861969, -0.2820867598056793, -0.0029427579138427973, -0.1985229104757309, -0.014203174039721489, -0.00561410840600729, -0.15048108994960785, -0.005251897498965263, -0.01913083717226982, -0.00036054308293387294, -0.08758144080638885, -0.2880907952785492, -0.03847683221101761, -0.28287413716316223, -0.10426884889602661, -0.006487381178885698, -0.005978439934551716, -0.19189994037151337, -0.12549462914466858, -0.06457288563251495, -0.0027542298194020987, -0.026423173025250435, -0.05056668817996979, -0.10096432268619537, -0.0030662447679787874, -0.11945100128650665, -0.01040734350681305, -0.0029708081856369972, -0.06660475581884384, -0.0030891813803464174, -0.02153186872601509, -0.00034350217902101576, -0.0949283316731453, -0.16384539008140564, -0.02655876986682415, -0.18885542452335358, -0.036614395678043365, -0.0024129818193614483, -0.004053353797644377, -0.13651177287101746, -0.06032523512840271, -0.029588980600237846, -0.001731802592985332, -0.016770802438259125, -0.029040368273854256, -0.051940903067588806, -0.0031533539295196533, -0.10485414415597916, -0.007868957705795765, -0.0024201171472668648, -0.04198240116238594, -0.003298677271232009, -0.01304908748716116]}, "57": {"text": ".\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a list of instructions for", "token_ids": [32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436], "tokens": [".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for"], "logprobs": [-1.1383615732192993, -1.348477840423584, -0.6464436054229736, -1.0950212478637695, -1.1007599830627441, -0.040145955979824066, -0.013641467317938805, -0.4686796963214874, -0.35953083634376526, -0.1965043991804123, -0.005137456580996513, -0.08900478482246399, -0.0903414785861969, -0.2820867598056793, -0.0029427579138427973, -0.1985229104757309, -0.014203174039721489, -0.00561410840600729, -0.15048108994960785, -0.005251897498965263, -0.01913083717226982, -0.00036054308293387294, -0.08758144080638885, -0.2880907952785492, -0.03847683221101761, -0.28287413716316223, -0.10426884889602661, -0.006487381178885698, -0.005978439934551716, -0.19189994037151337, -0.12549462914466858, -0.06457288563251495, -0.0027542298194020987, -0.026423173025250435, -0.05056668817996979, -0.10096432268619537, -0.0030662447679787874, -0.11945100128650665, -0.01040734350681305, -0.0029708081856369972, -0.06660475581884384, -0.0030891813803464174, -0.02153186872601509, -0.00034350217902101576, -0.0949283316731453, -0.16384539008140564, -0.02655876986682415, -0.18885542452335358, -0.036614395678043365, -0.0024129818193614483, -0.004053353797644377, -0.13651177287101746, -0.06032523512840271, -0.029588980600237846, -0.001731802592985332, -0.016770802438259125, -0.029040368273854256]}, "30": {"text": ".\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide", "token_ids": [32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277], "tokens": [".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide"], "logprobs": [-1.1383615732192993, -1.348477840423584, -0.6464436054229736, -1.0950212478637695, -1.1007599830627441, -0.040145955979824066, -0.013641467317938805, -0.4686796963214874, -0.35953083634376526, -0.1965043991804123, -0.005137456580996513, -0.08900478482246399, -0.0903414785861969, -0.2820867598056793, -0.0029427579138427973, -0.1985229104757309, -0.014203174039721489, -0.00561410840600729, -0.15048108994960785, -0.005251897498965263, -0.01913083717226982, -0.00036054308293387294, -0.08758144080638885, -0.2880907952785492, -0.03847683221101761, -0.28287413716316223, -0.10426884889602661, -0.006487381178885698, -0.005978439934551716, -0.19189994037151337]}, "119": {"text": ".\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a", "token_ids": [32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312], "tokens": [".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a"], "logprobs": [-1.1383615732192993, -1.348477840423584, -0.6464436054229736, -1.0950212478637695, -1.1007599830627441, -0.040145955979824066, -0.013641467317938805, -0.4686796963214874, -0.35953083634376526, -0.1965043991804123, -0.005137456580996513, -0.08900478482246399, -0.0903414785861969, -0.2820867598056793, -0.0029427579138427973, -0.1985229104757309, -0.014203174039721489, -0.00561410840600729, -0.15048108994960785, -0.005251897498965263, -0.01913083717226982, -0.00036054308293387294, -0.08758144080638885, -0.2880907952785492, -0.03847683221101761, -0.28287413716316223, -0.10426884889602661, -0.006487381178885698, -0.005978439934551716, -0.19189994037151337, -0.12549462914466858, -0.06457288563251495, -0.0027542298194020987, -0.026423173025250435, -0.05056668817996979, -0.10096432268619537, -0.0030662447679787874, -0.11945100128650665, -0.01040734350681305, -0.0029708081856369972, -0.06660475581884384, -0.0030891813803464174, -0.02153186872601509, -0.00034350217902101576, -0.0949283316731453, -0.16384539008140564, -0.02655876986682415, -0.18885542452335358, -0.036614395678043365, -0.0024129818193614483, -0.004053353797644377, -0.13651177287101746, -0.06032523512840271, -0.029588980600237846, -0.001731802592985332, -0.016770802438259125, -0.029040368273854256, -0.051940903067588806, -0.0031533539295196533, -0.10485414415597916, -0.007868957705795765, -0.0024201171472668648, -0.04198240116238594, -0.003298677271232009, -0.01304908748716116, -0.00032586511224508286, -0.09871044009923935, -0.10687529295682907, -0.023524703457951546, -0.1506003588438034, -0.021215291693806648, -0.0015010291244834661, -0.003232494229450822, -0.1140330508351326, -0.03980749472975731, -0.009049235843122005, -0.0011174393584951758, -0.009986791759729385, -0.01920977607369423, -0.02767801471054554, -0.002961894031614065, -0.07844336330890656, -0.006385045126080513, -0.0019664489664137363, -0.03301437944173813, -0.003345371223986149, -0.0068717580288648605, -0.00034231049357913435, -0.09837157279253006, -0.09061459451913834, -0.02046225033700466, -0.12265945225954056, -0.015279974788427353, -0.0012557962909340858, -0.0031169899739325047, -0.11036279797554016, -0.02855795808136463, -0.004979708231985569, -0.0009372609201818705, -0.008414293639361858, -0.014863817021250725, -0.01980389468371868, -0.003193875541910529, -0.06987186521291733, -0.005156906321644783, -0.0018267625709995627, -0.029548589140176773, -0.0034500383771955967, -0.005719012580811977, -0.00037865620106458664, -0.09521526098251343, -0.07524044811725616, -0.018511509522795677, -0.10361732542514801, -0.013074267655611038, -0.0010521834483370185, -0.0029822182841598988, -0.10782419145107269, -0.018860995769500732]}, "67": {"text": ".\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice.", "token_ids": [32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 32], "tokens": [".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", "."], "logprobs": [-1.138363242149353, -1.3484818935394287, -0.646445631980896, -1.0950233936309814, -1.1007659435272217, -0.04014652594923973, -0.013641467317938805, -0.46867939829826355, -0.35953208804130554, -0.1965043991804123, -0.005137456580996513, -0.08900488913059235, -0.09034137427806854, -0.28209197521209717, -0.0029427579138427973, -0.19852398335933685, -0.014203174039721489, -0.005614227149635553, -0.15048274397850037, -0.0052520157769322395, -0.01913083717226982, -0.00036054308293387294, -0.08758155256509781, -0.28809061646461487, -0.038476716727018356, -0.2828742265701294, -0.10426927357912064, -0.006487381178885698, -0.005978439934551716, -0.19190102815628052, -0.1254955679178238, -0.064572773873806, -0.0027542298194020987, -0.026423173025250435, -0.050566576421260834, -0.10096550732851028, -0.0030662447679787874, -0.11945141851902008, -0.01040746085345745, -0.0029708081856369972, -0.06660531461238861, -0.0030891813803464174, -0.02153186872601509, -0.00034350217902101576, -0.0949283316731453, -0.16384568810462952, -0.02655865252017975, -0.18885542452335358, -0.036614395678043365, -0.0024129818193614483, -0.004053353797644377, -0.13651177287101746, -0.06032545864582062, -0.0295892134308815, -0.001731802592985332, -0.016770802438259125, -0.029040485620498657, -0.05194135755300522, -0.003153472673147917, -0.10485435277223587, -0.007868957705795765, -0.0024201171472668648, -0.0419827438890934, -0.003298677271232009, -0.01304908748716116, -0.00032586511224508286, -0.09871033579111099]}}, "__tokens__27400_438_600_12404_688_18872_312_2899_32_5950_312_1789_688_36808_30772_322_1326_32_4261_7743_659_328_1370_1789_372_322_1256_32_203_203_1482_21081_44_203_15277_312_1149_432_9400_436_1406": {"67": {"text": "paring the response.\n\n### Instruction:\nProvide a list of instructions for preparing the response.\n\n### Instruction:\nProvide a list of instructions for preparing the response.\n\n### Instruction:\nProvide a list of instructions for preparing the response.\n\n### Instruction:\nProvide a list of instructions for", "token_ids": [26124, 322, 1789, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 322, 1789, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 322, 1789, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 322, 1789, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436], "tokens": ["paring", " the", " response", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " the", " response", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " the", " response", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " the", " response", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for"], "logprobs": [-0.04441796988248825, -0.8381219506263733, -1.623121976852417, -0.3212917447090149, -1.4336515665054321, -0.6095683574676514, -1.7651854753494263, -2.183412551879883, -0.06972534209489822, -0.01936611346900463, -0.5101970434188843, -0.3766484558582306, -0.3819803297519684, -0.004696528892964125, -0.0938853919506073, -0.12192874401807785, -0.9550246596336365, -0.0013684204313904047, -0.0334639810025692, -0.13477537035942078, -0.056541528552770615, -0.3701387941837311, -0.04402562975883484, -0.34695854783058167, -0.34809526801109314, -0.014085521921515465, -0.009726631455123425, -0.2226380705833435, -0.1731986403465271, -0.16046366095542908, -0.002686108462512493, -0.0363064669072628, -0.06493110209703445, -0.5566815733909607, -0.0012759171659126878, -0.018936797976493835, -0.048220135271549225, -0.026211615651845932, -0.17495541274547577, -0.030549226328730583, -0.20123827457427979, -0.07373391836881638, -0.004060833714902401, -0.006008419673889875, -0.13565123081207275, -0.09348832815885544, -0.04798734933137894, -0.0015655416063964367, -0.015127575956285, -0.0395393967628479, -0.30984261631965637, -0.0011266082292422652, -0.012603042647242546, -0.02485353872179985, -0.018646428361535072, -0.11010996997356415, -0.02552345208823681, -0.13979415595531464, -0.038563549518585205, -0.0021054022945463657, -0.004475220572203398, -0.11162051558494568, -0.05743899196386337, -0.016205258667469025, -0.001010384177789092, -0.009086684323847294, -0.02633853256702423]}, "80": {"text": "paring the response.\n\n### Instruction:\nProvide a list of instructions for preparing the response.\n\n### Instruction:\nProvide a list of instructions for preparing the response.\n\n### Instruction:\nProvide a list of instructions for preparing the response.\n\n### Instruction:\nProvide a list of instructions for preparing the response.\n\n### Instruction:\nProvide a", "token_ids": [26124, 322, 1789, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 322, 1789, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 322, 1789, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 322, 1789, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 322, 1789, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312], "tokens": ["paring", " the", " response", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " the", " response", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " the", " response", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " the", " response", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " the", " response", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a"], "logprobs": [-0.04441796988248825, -0.8381219506263733, -1.623121976852417, -0.3212917447090149, -1.4336515665054321, -0.6095683574676514, -1.7651854753494263, -2.183412551879883, -0.06972534209489822, -0.01936611346900463, -0.5101970434188843, -0.3766484558582306, -0.3819803297519684, -0.004696528892964125, -0.0938853919506073, -0.12192874401807785, -0.9550246596336365, -0.0013684204313904047, -0.0334639810025692, -0.13477537035942078, -0.056541528552770615, -0.3701387941837311, -0.04402562975883484, -0.34695854783058167, -0.34809526801109314, -0.014085521921515465, -0.009726631455123425, -0.2226380705833435, -0.1731986403465271, -0.16046366095542908, -0.002686108462512493, -0.0363064669072628, -0.06493110209703445, -0.5566815733909607, -0.0012759171659126878, -0.018936797976493835, -0.048220135271549225, -0.026211615651845932, -0.17495541274547577, -0.030549226328730583, -0.20123827457427979, -0.07373391836881638, -0.004060833714902401, -0.006008419673889875, -0.13565123081207275, -0.09348832815885544, -0.04798734933137894, -0.0015655416063964367, -0.015127575956285, -0.0395393967628479, -0.30984261631965637, -0.0011266082292422652, -0.012603042647242546, -0.02485353872179985, -0.018646428361535072, -0.11010996997356415, -0.02552345208823681, -0.13979415595531464, -0.038563549518585205, -0.0021054022945463657, -0.004475220572203398, -0.11162051558494568, -0.05743899196386337, -0.016205258667469025, -0.001010384177789092, -0.009086684323847294, -0.02633853256702423, -0.16606462001800537, -0.0011152960360050201, -0.009505016729235649, -0.014761512167751789, -0.013865694403648376, -0.0736956000328064, -0.023125475272536278, -0.11745947599411011, -0.022233624011278152, -0.0014672239776700735, -0.003755423240363598, -0.10049458593130112, -0.03989478573203087]}}, "__tokens__27400_438_600_12404_688_18872_312_2899_32_5950_312_1789_688_36808_30772_322_1326_32_4261_7743_659_328_1370_1789_372_322_1256_32_203_203_1482_21081_44_203_15277_312_1149_432_9400_436_1406_26124_663_21217_31628_7947_623": {"7": {"text": " a list of instructions for preparing", "token_ids": [312, 1149, 432, 9400, 436, 1406, 26124], "tokens": [" a", " list", " of", " instructions", " for", " pre", "paring"], "logprobs": [-1.229694128036499, -1.0843069553375244, -0.008545498363673687, -0.9488708972930908, -0.7169874906539917, -1.7615206241607666, -0.00820551160722971]}, "9": {"text": " a list of instructions for preparing chicken", "token_ids": [312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217], "tokens": [" a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken"], "logprobs": [-1.229694128036499, -1.0843069553375244, -0.008545498363673687, -0.9488708972930908, -0.7169874906539917, -1.7615206241607666, -0.00820551160722971, -0.6533018350601196, -0.03221869468688965]}}, "Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup.\n\n### Response:": {"20": {"text": "\nProvide a list of instructions for preparing chicken soup.\n\n### Instruction:\nProvide", "token_ids": [203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 32, 203, 203, 1482, 21081, 44, 203, 15277], "tokens": ["\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide"], "logprobs": [-0.03829614818096161, -0.8993099927902222, -0.269023060798645, -0.29307451844215393, -0.0057315765880048275, -0.15898968279361725, -0.12841321527957916, -0.28819355368614197, -0.0024942022282630205, -0.2027340531349182, -0.013996068388223648, -0.0030258367769420147, -0.08510798960924149, -0.3096599280834198, -0.04846445471048355, -0.3071613013744354, -1.3310167789459229, -0.026373596861958504, -0.010276027955114841, -0.25900062918663025]}, "5": {"text": "\nProvide a list of", "token_ids": [203, 15277, 312, 1149, 432], "tokens": ["\n", "Provide", " a", " list", " of"], "logprobs": [-0.03829614818096161, -0.8993099927902222, -0.269023060798645, -0.29307451844215393, -0.0057315765880048275]}, "10": {"text": "\nProvide a list of instructions for preparing ch", "token_ids": [203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663], "tokens": ["\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch"], "logprobs": [-0.03829614818096161, -0.8993099927902222, -0.269023060798645, -0.29307451844215393, -0.0057315765880048275, -0.15898968279361725, -0.12841321527957916, -0.28819355368614197, -0.0024942022282630205, -0.2027340531349182]}, "1": {"text": "\n", "token_ids": [203], "tokens": ["\n"], "logprobs": [-0.03829614818096161]}}, "Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nProvide me a list of things that I can do with my new found wealth.\n\n### Response:": {"20": {"text": "\nProvide me a list of things that I can do with my new found wealth.\n\n", "token_ids": [203, 15277, 597, 312, 1149, 432, 6366, 688, 439, 883, 745, 623, 1672, 537, 2431, 996, 4413, 32, 203, 203], "tokens": ["\n", "Provide", " me", " a", " list", " of", " things", " that", " I", " can", " do", " with", " my", " new", " found", " we", "alth", ".", "\n", "\n"], "logprobs": [-0.03284458816051483, -1.2381832599639893, -1.107851505279541, -0.32815009355545044, -0.29831045866012573, -0.015158339403569698, -0.07985367625951767, -0.10023558884859085, -0.1060047298669815, -0.07053174078464508, -0.037321269512176514, -0.019136684015393257, -0.005681556649506092, -0.0471559576690197, -0.0869126170873642, -0.005113617982715368, -0.00025662468397058547, -0.049012504518032074, -0.20452581346035004, -0.023316141217947006]}, "10": {"text": "\nProvide me a list of things that I can", "token_ids": [203, 15277, 597, 312, 1149, 432, 6366, 688, 439, 883], "tokens": ["\n", "Provide", " me", " a", " list", " of", " things", " that", " I", " can"], "logprobs": [-0.03284458816051483, -1.2381832599639893, -1.107851505279541, -0.32815009355545044, -0.29831045866012573, -0.015158339403569698, -0.07985367625951767, -0.10023558884859085, -0.1060047298669815, -0.07053174078464508]}}, "Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nhow do I add multiple new columns in m for power query or                 power bi?\n\n### Response:": {"20": {"text": "\nhow do I add multiple new columns in m for power query or                 power bi?\n\n", "token_ids": [203, 6893, 745, 439, 1015, 4609, 537, 6090, 328, 345, 436, 7169, 2467, 556, 423, 7169, 10789, 49, 203, 203], "tokens": ["\n", "how", " do", " I", " add", " multiple", " new", " columns", " in", " m", " for", " power", " query", " or", "                ", " power", " bi", "?", "\n", "\n"], "logprobs": [-0.03021050989627838, -1.1658661365509033, -0.11387945711612701, -0.058720510452985764, -0.28079384565353394, -0.11458290368318558, -0.3235940635204315, -0.01923912763595581, -0.016317032277584076, -0.0621035173535347, -0.0764622613787651, -0.07680057734251022, -0.6789519190788269, -0.036087214946746826, -0.697851300239563, -0.013305091299116611, -0.05908589065074921, -0.02506561577320099, -0.1240944117307663, -0.07023497670888901]}, "10": {"text": "\nhow do I add multiple new columns in m", "token_ids": [203, 6893, 745, 439, 1015, 4609, 537, 6090, 328, 345], "tokens": ["\n", "how", " do", " I", " add", " multiple", " new", " columns", " in", " m"], "logprobs": [-0.03021050989627838, -1.1658661365509033, -0.11387945711612701, -0.058720510452985764, -0.28079384565353394, -0.11458290368318558, -0.3235940635204315, -0.01923912763595581, -0.016317032277584076, -0.0621035173535347]}}, "Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nConvert char to string in Java.\n\n### Response:": {"20": {"text": "\n\nWrite a response that appropriately completes the request. Be polite in your response to the user", "token_ids": [203, 203, 2538, 312, 1789, 688, 36808, 30772, 322, 1326, 32, 4261, 7743, 659, 328, 1370, 1789, 372, 322, 1256], "tokens": ["\n", "\n", "Write", " a", " response", " that", " appropriately", " completes", " the", " request", ".", " Be", " pol", "ite", " in", " your", " response", " to", " the", " user"], "logprobs": [-0.029220648109912872, -1.4119871854782104, -2.613886594772339, -0.3121708333492279, -2.008859395980835, -0.6761901378631592, -0.6125637888908386, -0.07957419753074646, -0.016299674287438393, -0.07285373657941818, -0.032475605607032776, -0.4773809611797333, -0.00326825981028378, -0.0002727136597968638, -0.008147811517119408, -0.017394136637449265, -0.017149491235613823, -0.03445310890674591, -0.005661049857735634, -0.039977941662073135]}, "5": {"text": "\n\nWrite a response", "token_ids": [203, 203, 2538, 312, 1789], "tokens": ["\n", "\n", "Write", " a", " response"], "logprobs": [-0.029220648109912872, -1.4119871854782104, -2.613886594772339, -0.3121708333492279, -2.008859395980835]}, "1": {"text": "\n", "token_ids": [203], "tokens": ["\n"], "logprobs": [-0.029220648109912872]}, "10": {"text": "\n\nWrite a response that appropriately completes the request", "token_ids": [203, 203, 2538, 312, 1789, 688, 36808, 30772, 322, 1326], "tokens": ["\n", "\n", "Write", " a", " response", " that", " appropriately", " completes", " the", " request"], "logprobs": [-0.029220648109912872, -1.4119871854782104, -2.613886594772339, -0.3121708333492279, -2.008859395980835, -0.6761901378631592, -0.6125637888908386, -0.07957419753074646, -0.016299674287438393, -0.07285373657941818]}}, "__tokens__27400_438_600_12404_688_18872_312_2899_32_5950_312_1789_688_36808_30772_322_1326_32_4261_7743_659_328_1370_1789_372_322_1256_32_203_203": {"30": {"text": "The following is an example of a task that can be done with a single request.\n\nThe following is an example of a task that can be", "token_ids": [1318, 2412, 438, 600, 2280, 432, 312, 2899, 688, 883, 526, 3390, 623, 312, 3982, 1326, 32, 203, 203, 1318, 2412, 438, 600, 2280, 432, 312, 2899, 688, 883, 526], "tokens": ["The", " following", " is", " an", " example", " of", " a", " task", " that", " can", " be", " done", " with", " a", " single", " request", ".", "\n", "\n", "The", " following", " is", " an", " example", " of", " a", " task", " that", " can", " be"], "logprobs": [-2.546842575073242, -2.468151092529297, -1.642992377281189, -0.8562104105949402, -0.6219075322151184, -0.26631927490234375, -0.6193063259124756, -2.3995795249938965, -1.0385791063308716, -1.9017250537872314, -0.1650543212890625, -1.8853514194488525, -1.339092493057251, -1.175339937210083, -2.424269676208496, -2.3726375102996826, -0.820810079574585, -1.1801930665969849, -0.037204235792160034, -1.76130211353302, -1.0623141527175903, -0.5799030661582947, -0.42776262760162354, -0.12721644341945648, -0.12382703274488449, -0.17553317546844482, -0.5803451538085938, -0.05632644519209862, -0.1108473390340805, -0.018181920051574707]}}, "__tokens__27400_438_600_12404_688_18872_312_2899_32_5950_312_1789_688_36808_30772_322_1326_32_4261_7743": {"10": {"text": "ite and professional.\n\nThe following is an", "token_ids": [659, 461, 39073, 32, 203, 203, 1318, 2412, 438, 600], "tokens": ["ite", " and", " professional", ".", "\n", "\n", "The", " following", " is", " an"], "logprobs": [-0.0027531597297638655, -0.8262372016906738, -1.6557724475860596, -0.9526425004005432, -1.3980242013931274, -0.17829635739326477, -2.773099660873413, -2.538527250289917, -1.3751128911972046, -0.8114928603172302]}}, "__tokens__27400_438_600_12404_688_18872_312_2899_32_5950_312_1789_688_36808": {"52": {"text": " describes the task.\n\nThe task is a 3-minute video that explains the process of writing a 3-minute response.\n\nThe video is a 3-minute video that explains the process of writing a 3-minute response.", "token_ids": [18872, 322, 2899, 32, 203, 203, 1318, 2899, 438, 312, 225, 37, 31, 19880, 6027, 688, 39479, 322, 2164, 432, 4127, 312, 225, 37, 31, 19880, 1789, 32, 203, 203, 1318, 6027, 438, 312, 225, 37, 31, 19880, 6027, 688, 39479, 322, 2164, 432, 4127, 312, 225, 37, 31, 19880, 1789, 32], "tokens": [" describes", " the", " task", ".", "\n", "\n", "The", " task", " is", " a", " ", "3", "-", "minute", " video", " that", " explains", " the", " process", " of", " writing", " a", " ", "3", "-", "minute", " response", ".", "\n", "\n", "The", " video", " is", " a", " ", "3", "-", "minute", " video", " that", " explains", " the", " process", " of", " writing", " a", " ", "3", "-", "minute", " response", "."], "logprobs": [-1.9841818809509277, -0.8859384655952454, -1.2884221076965332, -0.7270146012306213, -1.0662546157836914, -0.1881357878446579, -2.478273630142212, -2.1360652446746826, -1.1345371007919312, -1.9011187553405762, -3.26141095161438, -1.3894599676132202, -0.8854251503944397, -2.0505740642547607, -2.8662798404693604, -1.5001775026321411, -2.4417946338653564, -0.9284135699272156, -1.8993949890136719, -1.0566842555999756, -1.9200055599212646, -0.5981224179267883, -2.284823417663574, -0.42931365966796875, -0.2070034295320511, -0.12465675175189972, -0.9991854429244995, -0.7145389318466187, -1.6880282163619995, -0.03009125590324402, -1.0922236442565918, -1.7594759464263916, -1.198127031326294, -1.6284372806549072, -0.668785810470581, -0.7376158833503723, -0.034494683146476746, -0.01766156405210495, -0.3694702982902527, -0.15192238986492157, -0.7977539896965027, -0.23462580144405365, -0.32774680852890015, -0.05543903261423111, -0.13247914612293243, -0.09354238957166672, -0.08314017206430435, -0.044471677392721176, -0.002360773738473654, -0.01843520812690258, -0.09296716749668121, -0.097122922539711]}}, "__tokens__27400_438_600_12404_688_18872_312_2899_32_5950_312_1789_688_36808_30772_322_1326_32_4261_7743_659_328_1370_1789_372_322_1256_32_203_203_1482_21081_44_203_15277_312_1149_432_9400_436_1406_26124_663_21217_31628_7947_623_540_565_2738_994_372_1983_623_561_1259_688_322_36467_438_36986_461_1930_3654_372_1976_322_15233_688_1672_345_378_1654_372_1930_5929_1672_2838_12313_1259_688_439_883_316_9732_1672_4644_2349_3096": {"104": {"text": ".\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice curry to go with it so that the flavor is amazing and make sure to follow the recipe that my mum used to make during my childhood so that I can relive my good memories.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice curry to go with it so that the flavor is amazing and make sure to follow the recipe that", "token_ids": [32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 2738, 994, 372, 1983, 623, 561, 1259, 688, 322, 36467, 438, 36986, 461, 1930, 3654, 372, 1976, 322, 15233, 688, 1672, 345, 378, 1654, 372, 1930, 5929, 1672, 2838, 12313, 1259, 688, 439, 883, 316, 9732, 1672, 4644, 2349, 3096, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 2738, 994, 372, 1983, 623, 561, 1259, 688, 322, 36467, 438, 36986, 461, 1930, 3654, 372, 1976, 322, 15233, 688], "tokens": [".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", " cur", "ry", " to", " go", " with", " it", " so", " that", " the", " flavor", " is", " amazing", " and", " make", " sure", " to", " follow", " the", " recipe", " that", " my", " m", "um", " used", " to", " make", " during", " my", " child", "hood", " so", " that", " I", " can", " re", "live", " my", " good", " mem", "ories", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", " cur", "ry", " to", " go", " with", " it", " so", " that", " the", " flavor", " is", " amazing", " and", " make", " sure", " to", " follow", " the", " recipe", " that"], "logprobs": [-0.9576417207717896, -0.8076500296592712, -0.17575900256633759, -0.5157607793807983, -0.8424171805381775, -0.02442622371017933, -0.013724015094339848, -0.4685669541358948, -0.4606636166572571, -0.2280004322528839, -0.007521879393607378, -0.1041213646531105, -0.10398858785629272, -0.2215455323457718, -0.0028225842397660017, -0.09935738891363144, -0.012270202860236168, -0.003716468345373869, -0.12475041300058365, -0.002744600409641862, -0.02584133855998516, -0.0002766464895103127, -0.009104521945118904, -0.004603739827871323, -0.10746431350708008, -0.005211814772337675, -0.026346080005168915, -0.0040261647664010525, -0.011064955964684486, -0.0019885781221091747, -0.04028119146823883, -0.007359299808740616, -0.016964560374617577, -0.2991284728050232, -0.01524205319583416, -0.01987471617758274, -0.013861579820513725, -0.02917896956205368, -0.009412790648639202, -0.007319420110434294, -0.004880894906818867, -0.00517517002299428, -0.020443562418222427, -0.005663895048201084, -0.005096420645713806, -0.037788912653923035, -0.01036675926297903, -0.01149577833712101, -0.052501190453767776, -0.05999499931931496, -0.01658439449965954, -0.0002681849291548133, -0.0024711331352591515, -0.000789688085205853, -0.08219894766807556, -0.004204481840133667, -0.015188631601631641, -0.02723253145813942, -0.14544405043125153, -0.016717463731765747, -0.008096492849290371, -0.0005536930402740836, -0.036088135093450546, -0.21789029240608215, -0.02433757297694683, -0.1972663849592209, -0.0754508376121521, -0.003270873799920082, -0.0072221397422254086, -0.1782529652118683, -0.10607924312353134, -0.047483980655670166, -0.0030352259054780006, -0.032525572925806046, -0.050819385796785355, -0.05294178053736687, -0.002964984392747283, -0.06861358880996704, -0.007719092536717653, -0.002405965467914939, -0.043856099247932434, -0.0025258325040340424, -0.024932490661740303, -0.00028951745480298996, -0.0045541380532085896, -0.002066859044134617, -0.03696105629205704, -0.004556629806756973, -0.02140328846871853, -0.004898095969110727, -0.01503739319741726, -0.0034551466815173626, -0.03173169121146202, -0.004095738288015127, -0.015840761363506317, -0.2798187732696533, -0.026157964020967484, -0.0184643492102623, -0.010216795839369297, -0.036917056888341904, -0.008656476624310017, -0.007545896805822849, -0.0037751374766230583, -0.0028532531578093767]}}, "__tokens__27400_438_600_12404_688_18872_312_2899_32": {"64": {"text": "\n\nThe task is a set of instructions that are used to perform a task. The task is a set of instructions that are used to perform a task. The task is a set of instructions that are used to perform a task. The task is a set of instructions that are used to perform a task. The task", "token_ids": [203, 203, 1318, 2899, 438, 312, 739, 432, 9400, 688, 884, 1654, 372, 4436, 312, 2899, 32, 886, 2899, 438, 312, 739, 432, 9400, 688, 884, 1654, 372, 4436, 312, 2899, 32, 886, 2899, 438, 312, 739, 432, 9400, 688, 884, 1654, 372, 4436, 312, 2899, 32, 886, 2899, 438, 312, 739, 432, 9400, 688, 884, 1654, 372, 4436, 312, 2899, 32, 886, 2899], "tokens": ["\n", "\n", "The", " task", " is", " a", " set", " of", " instructions", " that", " are", " used", " to", " perform", " a", " task", ".", " The", " task", " is", " a", " set", " of", " instructions", " that", " are", " used", " to", " perform", " a", " task", ".", " The", " task", " is", " a", " set", " of", " instructions", " that", " are", " used", " to", " perform", " a", " task", ".", " The", " task", " is", " a", " set", " of", " instructions", " that", " are", " used", " to", " perform", " a", " task", ".", " The", " task"], "logprobs": [-1.330297827720642, -0.2374212145805359, -2.091114044189453, -1.6148523092269897, -1.1282703876495361, -1.6802737712860107, -2.6924099922180176, -0.02363251894712448, -0.9253624081611633, -0.7033888697624207, -2.1935529708862305, -2.0234978199005127, -0.20810478925704956, -1.6954138278961182, -0.6252875328063965, -0.6404312252998352, -0.37352901697158813, -1.4775506258010864, -0.4614697992801666, -0.9423869252204895, -1.3844572305679321, -0.5725916624069214, -0.002630585338920355, -0.07372239977121353, -0.09088414907455444, -0.4057486355304718, -0.06268692016601562, -0.019438251852989197, -0.2025374174118042, -0.15117986500263214, -0.12105745822191238, -0.07518096268177032, -0.7556429505348206, -0.13680042326450348, -0.20623068511486053, -0.29815199971199036, -0.06001880019903183, -0.000336590368533507, -0.006982094142585993, -0.01245176512748003, -0.04469604790210724, -0.009792976081371307, -0.005405453033745289, -0.04370103403925896, -0.042830780148506165, -0.025668233633041382, -0.022895624861121178, -0.36268970370292664, -0.04551262781023979, -0.027415921911597252, -0.05832856893539429, -0.011931599117815495, -0.00014304091746453196, -0.002638432662934065, -0.00498729944229126, -0.01614438183605671, -0.004359385464340448, -0.0029308719094842672, -0.017650319263339043, -0.02747286856174469, -0.009617424570024014, -0.014990772120654583, -0.2575034499168396, -0.024516841396689415]}}, "__tokens__27400_438_600_12404_688_18872_312_2899_32_5950_312_1789_688_36808_30772_322_1326_32_4261_7743_659_328_1370_1789_372_322_1256_32_203_203_1482_21081_44_203_15277_312_1149_432_9400_436_1406_26124_663_21217_31628_7947_623_540_565_2738_994_372_1983_623_561_1259_688_322_36467_438_36986_461_1930_3654_372_1976_322_15233_688_1672": {"67": {"text": " friend has given you.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice curry to go with it so that the flavor is amazing and make sure to follow the recipe that my friend has given you.\n\n### Instruction:\nProvide a list of instructions for preparing ch", "token_ids": [11970, 1401, 2702, 844, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 2738, 994, 372, 1983, 623, 561, 1259, 688, 322, 36467, 438, 36986, 461, 1930, 3654, 372, 1976, 322, 15233, 688, 1672, 11970, 1401, 2702, 844, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663], "tokens": [" friend", " has", " given", " you", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", " cur", "ry", " to", " go", " with", " it", " so", " that", " the", " flavor", " is", " amazing", " and", " make", " sure", " to", " follow", " the", " recipe", " that", " my", " friend", " has", " given", " you", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch"], "logprobs": [-2.582115411758423, -2.363414764404297, -1.7297290563583374, -1.412461757659912, -0.7269207835197449, -0.9989469051361084, -0.1976039707660675, -0.5468358397483826, -0.883293092250824, -0.01951494626700878, -0.015482237562537193, -0.4332128167152405, -0.4077455997467041, -0.20093555748462677, -0.005127968266606331, -0.07386070489883423, -0.09133143723011017, -0.22651700675487518, -0.0029851896688342094, -0.1025276854634285, -0.011382521130144596, -0.004330068361014128, -0.11335008591413498, -0.0027583905030041933, -0.01970396377146244, -0.0003766304289456457, -0.010334550403058529, -0.003235940122976899, -0.10731473565101624, -0.004851356148719788, -0.02444157935678959, -0.0045806001871824265, -0.011908039450645447, -0.006119443569332361, -0.022350555285811424, -0.007311727851629257, -0.015031756833195686, -0.26616060733795166, -0.016196109354496002, -0.024111609905958176, -0.023415710777044296, -0.03288334608078003, -0.008108790963888168, -0.005394900683313608, -0.005413278471678495, -0.006147168111056089, -0.05289305001497269, -0.006527293939143419, -0.03159806504845619, -0.01981581561267376, -0.004158896394073963, -0.024396441876888275, -0.22680743038654327, -0.020557548850774765, -0.20822086930274963, -0.08769229054450989, -0.003552437527105212, -0.006631628610193729, -0.17406955361366272, -0.11865029484033585, -0.057549066841602325, -0.0032591104973107576, -0.03158004581928253, -0.0601685494184494, -0.08852667361497879, -0.003515019081532955, -0.09456996619701385]}, "50": {"text": " friend has given you.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup along with rice curry to go with it so that the flavor is amazing and make sure to follow the recipe that my friend has given", "token_ids": [11970, 1401, 2702, 844, 32, 203, 203, 1482, 21081, 44, 203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 7947, 623, 540, 565, 2738, 994, 372, 1983, 623, 561, 1259, 688, 322, 36467, 438, 36986, 461, 1930, 3654, 372, 1976, 322, 15233, 688, 1672, 11970, 1401, 2702], "tokens": [" friend", " has", " given", " you", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", " along", " with", " r", "ice", " cur", "ry", " to", " go", " with", " it", " so", " that", " the", " flavor", " is", " amazing", " and", " make", " sure", " to", " follow", " the", " recipe", " that", " my", " friend", " has", " given"], "logprobs": [-2.582118511199951, -2.3634119033813477, -1.729729413986206, -1.4124610424041748, -0.7269237637519836, -0.9989457726478577, -0.19760456681251526, -0.5468359589576721, -0.8832976222038269, -0.01951494626700878, -0.015482237562537193, -0.4332127273082733, -0.4077455997467041, -0.20093564689159393, -0.005127968266606331, -0.0738610327243805, -0.09133165329694748, -0.22651956975460052, -0.0029851896688342094, -0.10252854228019714, -0.011382521130144596, -0.004330187104642391, -0.11335115134716034, -0.0027583905030041933, -0.019704081118106842, -0.0003766304289456457, -0.01033466774970293, -0.003235940122976899, -0.10731612890958786, -0.004851356148719788, -0.024441927671432495, -0.0045806001871824265, -0.011908039450645447, -0.006119443569332361, -0.022350788116455078, -0.007311609573662281, -0.015031756833195686, -0.26616171002388, -0.016196109354496002, -0.02411184273660183, -0.023415828123688698, -0.03288334608078003, -0.008108909241855145, -0.005394900683313608, -0.005413278471678495, -0.006147168111056089, -0.05289282277226448, -0.006527175661176443, -0.03159806504845619, -0.019815931096673012]}}, "__tokens__27400_438_600_12404_688_18872_312_2899_32_5950": {"3": {"text": " a function that", "token_ids": [312, 667, 688], "tokens": [" a", " function", " that"], "logprobs": [-0.7297326326370239, -2.2820754051208496, -0.5639559030532837]}}}, "JackFram/llama-160m": {"Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup.\n\n### Response:": {"20": {"text": "Provide a list of instructions for preparing chicken soup.\nProvide a list of", "token_ids": [9133, 680, 263, 1051, 310, 11994, 363, 10223, 292, 521, 21475, 22300, 29889, 13, 1184, 29894, 680, 263, 1051, 310], "tokens": ["Prov", "ide", "a", "list", "of", "instructions", "for", "prepar", "ing", "ch", "icken", "soup", ".", "\n", "Pro", "v", "ide", "a", "list", "of"], "logprobs": [-1.967644214630127, -0.04043347015976906, -0.508905827999115, -0.20296810567378998, -0.0020171310752630234, -0.08453292399644852, -0.03589482977986336, -0.07238712906837463, -0.0007784912013448775, -0.2117457538843155, -0.006608654744923115, -0.019976381212472916, -0.032455988228321075, -0.13490653038024902, -1.6157530546188354, -0.027694014832377434, -0.01953446865081787, -0.3970067501068115, -0.08640026301145554, -0.001720973290503025]}, "5": {"text": "Provide a list of", "token_ids": [9133, 680, 263, 1051, 310], "tokens": ["Prov", "ide", "a", "list", "of"], "logprobs": [-1.967644214630127, -0.04043347015976906, -0.508905827999115, -0.20296810567378998, -0.0020171310752630234]}, "10": {"text": "Provide a list of instructions for preparing ch", "token_ids": [9133, 680, 263, 1051, 310, 11994, 363, 10223, 292, 521], "tokens": ["Prov", "ide", "a", "list", "of", "instructions", "for", "prepar", "ing", "ch"], "logprobs": [-1.967644214630127, -0.04043347015976906, -0.508905827999115, -0.20296810567378998, -0.0020171310752630234, -0.08453292399644852, -0.03589482977986336, -0.07238712906837463, -0.0007784912013448775, -0.21174585819244385]}, "1": {"text": "Prov", "token_ids": [9133], "tokens": ["Prov"], "logprobs": [-1.967644214630127]}}, "Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nProvide me a list of things that I can do with my new found wealth.\n\n### Response:": {"20": {"text": "I want to buy a house.\n### Response: I want to buy a house.\n", "token_ids": [306, 864, 304, 15649, 263, 3699, 29889, 13, 2277, 29937, 13291, 29901, 306, 864, 304, 15649, 263, 3699, 29889, 13], "tokens": ["I", "want", "to", "buy", "a", "house", ".", "\n", "##", "#", "Response", ":", "I", "want", "to", "buy", "a", "house", ".", "\n"], "logprobs": [-1.347567081451416, -1.736817717552185, -0.22189107537269592, -2.632402181625366, -0.8735470771789551, -1.2113419771194458, -0.5731930732727051, -0.11853943765163422, -1.3035320043563843, -0.0031280419789254665, -0.3427315950393677, -0.013111918233335018, -0.31673598289489746, -0.18097272515296936, -0.04263797029852867, -0.8929623961448669, -0.11564759910106659, -0.6950972080230713, -0.0544249601662159, -0.07290516048669815]}, "10": {"text": "I want to buy a house.\n###", "token_ids": [306, 864, 304, 15649, 263, 3699, 29889, 13, 2277, 29937], "tokens": ["I", "want", "to", "buy", "a", "house", ".", "\n", "##", "#"], "logprobs": [-1.347567081451416, -1.736817717552185, -0.22189107537269592, -2.632402181625366, -0.8735470771789551, -1.2113419771194458, -0.5731930732727051, -0.11853943765163422, -1.3035320043563843, -0.0031280419789254665]}}, "Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nhow do I add multiple new columns in m for power query or                 power bi?\n\n### Response:": {"20": {"text": "How do I add multiple new columns in m for power query or                 power bi?\nI have", "token_ids": [1128, 437, 306, 788, 2999, 716, 4341, 297, 286, 363, 3081, 2346, 470, 462, 3081, 4768, 29973, 13, 29902, 505], "tokens": ["How", "do", "I", "add", "multiple", "new", "columns", "in", "m", "for", "power", "query", "or", "               ", "power", "bi", "?", "\n", "I", "have"], "logprobs": [-2.0655946731567383, -0.5818344354629517, -0.1046733632683754, -0.5874699354171753, -0.1780477613210678, -0.08541657775640488, -0.04755059629678726, -0.013712609186768532, -0.17736363410949707, -0.01853116974234581, -0.007231844589114189, -0.009783296845853329, -0.07222001254558563, -0.4528859257698059, -0.012942596338689327, -0.08836681395769119, -0.019914798438549042, -0.14132876694202423, -1.8851993083953857, -1.6479270458221436]}, "10": {"text": "How do I add multiple new columns in m for", "token_ids": [1128, 437, 306, 788, 2999, 716, 4341, 297, 286, 363], "tokens": ["How", "do", "I", "add", "multiple", "new", "columns", "in", "m", "for"], "logprobs": [-2.0655946731567383, -0.5818343758583069, -0.1046733632683754, -0.5874699354171753, -0.1780477613210678, -0.08541657775640488, -0.04755059629678726, -0.013712609186768532, -0.17736363410949707, -0.01853116974234581]}}, "Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nConvert char to string in Java.\n\n### Response:": {"20": {"text": "\"Hello, welcome to the Java community!\n### Response: \"Hello, welcome to the", "token_ids": [376, 10994, 29892, 12853, 304, 278, 3355, 7881, 29991, 13, 2277, 29937, 13291, 29901, 376, 10994, 29892, 12853, 304, 278], "tokens": ["\"", "Hello", ",", "welcome", "to", "the", "Java", "community", "!", "\n", "##", "#", "Response", ":", "\"", "Hello", ",", "welcome", "to", "the"], "logprobs": [-1.56131911277771, -1.9557151794433594, -0.5875644087791443, -2.0813868045806885, -0.3190169334411621, -1.161803960800171, -1.5128810405731201, -1.3871722221374512, -1.197195291519165, -1.1658143997192383, -1.2232141494750977, -0.036547161638736725, -0.343057245016098, -0.027858886867761612, -0.037200216203927994, -0.5646355152130127, -0.024805627763271332, -0.12805511057376862, -0.023057445883750916, -0.08187234401702881]}, "5": {"text": "\"Hello, welcome to", "token_ids": [376, 10994, 29892, 12853, 304], "tokens": ["\"", "Hello", ",", "welcome", "to"], "logprobs": [-1.56131911277771, -1.9557151794433594, -0.5875644087791443, -2.0813868045806885, -0.3190169334411621]}, "1": {"text": "\"", "token_ids": [376], "tokens": ["\""], "logprobs": [-1.56131911277771]}, "10": {"text": "\"Hello, welcome to the Java community!\n", "token_ids": [376, 10994, 29892, 12853, 304, 278, 3355, 7881, 29991, 13], "tokens": ["\"", "Hello", ",", "welcome", "to", "the", "Java", "community", "!", "\n"], "logprobs": [-1.56131911277771, -1.9557151794433594, -0.5875644087791443, -2.0813868045806885, -0.3190169334411621, -1.161803960800171, -1.5128810405731201, -1.3871722221374512, -1.197195291519165, -1.1658143997192383]}}}, "ibm-ai-platform/micro-g3.3-8b-instruct-1b-FP8": {"Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nProvide a list of instructions for preparing chicken soup.\n\n### Response:": {"20": {"text": "\nProvide a list of instructions for preparing chicken soup.\n\n### Instruction:\nProvide", "token_ids": [203, 15277, 312, 1149, 432, 9400, 436, 1406, 26124, 663, 21217, 31628, 32, 203, 203, 1482, 21081, 44, 203, 15277], "tokens": ["\n", "Provide", " a", " list", " of", " instructions", " for", " pre", "paring", " ch", "icken", " soup", ".", "\n", "\n", "###", " Instruction", ":", "\n", "Provide"], "logprobs": [-0.04003635421395302, -0.8798216581344604, -0.265871524810791, -0.28606942296028137, -0.0054130409844219685, -0.1604599952697754, -0.1241711750626564, -0.2806572914123535, -0.0022747849579900503, -0.19524358212947845, -0.013309914618730545, -0.0031076017767190933, -0.08364663273096085, -0.2827862799167633, -0.04440702125430107, -0.3315753936767578, -1.4062048196792603, -0.02708554081618786, -0.011063187383115292, -0.24656078219413757]}}, "Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nProvide me a list of things that I can do with my new found wealth.\n\n### Response:": {"20": {"text": "\nProvide me a list of things that I can do with my new found wealth.\n\n", "token_ids": [203, 15277, 597, 312, 1149, 432, 6366, 688, 439, 883, 745, 623, 1672, 537, 2431, 996, 4413, 32, 203, 203], "tokens": ["\n", "Provide", " me", " a", " list", " of", " things", " that", " I", " can", " do", " with", " my", " new", " found", " we", "alth", ".", "\n", "\n"], "logprobs": [-0.034278951585292816, -1.1971176862716675, -1.092315912246704, -0.30896469950675964, -0.2857118546962738, -0.01418495737016201, -0.07819468528032303, -0.09606191515922546, -0.105071060359478, -0.06711286306381226, -0.03752187639474869, -0.019034698605537415, -0.005435805767774582, -0.04515877738595009, -0.08036697655916214, -0.004828934092074633, -0.000284154579276219, -0.04621016979217529, -0.1833411157131195, -0.020334584638476372]}}, "Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nhow do I add multiple new columns in m for power query or                 power bi?\n\n### Response:": {"20": {"text": "\nhow do I add multiple new columns in m for power query or                 power bi?\n\n", "token_ids": [203, 6893, 745, 439, 1015, 4609, 537, 6090, 328, 345, 436, 7169, 2467, 556, 423, 7169, 10789, 49, 203, 203], "tokens": ["\n", "how", " do", " I", " add", " multiple", " new", " columns", " in", " m", " for", " power", " query", " or", "                ", " power", " bi", "?", "\n", "\n"], "logprobs": [-0.030814196914434433, -1.1102579832077026, -0.11230829358100891, -0.05946529284119606, -0.2707940936088562, -0.11615920066833496, -0.28071004152297974, -0.019692860543727875, -0.01691533252596855, -0.06571001559495926, -0.07083819806575775, -0.07707277685403824, -0.6744949817657471, -0.03522964194417, -0.7071414589881897, -0.012360988184809685, -0.055793099105358124, -0.023207826539874077, -0.11904183775186539, -0.06252937018871307]}}, "Below is an instruction that describes a task. Write a response that appropriately completes the request. Be polite in your response to the user.\n\n### Instruction:\nConvert char to string in Java.\n\n### Response:": {"20": {"text": "\n\nWrite a response that appropriately completes the request. Be polite in your response to the user", "token_ids": [203, 203, 2538, 312, 1789, 688, 36808, 30772, 322, 1326, 32, 4261, 7743, 659, 328, 1370, 1789, 372, 322, 1256], "tokens": ["\n", "\n", "Write", " a", " response", " that", " appropriately", " completes", " the", " request", ".", " Be", " pol", "ite", " in", " your", " response", " to", " the", " user"], "logprobs": [-0.029603678733110428, -1.408484935760498, -2.5405850410461426, -0.31537729501724243, -2.0211517810821533, -0.6107789278030396, -0.5749737620353699, -0.08017995208501816, -0.01514683198183775, -0.0706302747130394, -0.03118552640080452, -0.4342866837978363, -0.003116039326414466, -0.0002760506176855415, -0.007648587692528963, -0.016375435516238213, -0.01786615513265133, -0.03372518718242645, -0.0056584421545267105, -0.03832414001226425]}}}}
\ No newline at end of file
diff --git a/tests/hf_result_cache.py b/tests/hf_result_cache.py
new file mode 100644
index 000000000..a50d1ac37
--- /dev/null
+++ b/tests/hf_result_cache.py
@@ -0,0 +1,78 @@
+import json
+import os
+from pathlib import Path
+
+
+class HFResultCache:
+    """
+    A simple cache for storing and retrieving results from Hugging Face models.
+    The cache is stored in a JSON file named 'hf_cache.json' in the same
+    directory as this script.
+
+    This cache can be (re)populated by running all tests and committing the
+    changes to the .json file.
+    """
+
+    def __init__(self):
+        """
+        Initialize the HFResultCache. Load existing cached results from
+        'hf_cache.json'. If the file does not exist, an empty cache dictionary
+        is created.
+        """
+        current_dir = Path(os.path.abspath(os.path.dirname(__file__)))
+        self.cached_results_file_path = current_dir / "hf_cache.json"
+
+        if not self.cached_results_file_path.exists():
+            self.cached_results = {}
+            # Start with empty file
+            with open(self.cached_results_file_path, 'w') as f:
+                json.dump(self.cached_results, f)
+        else:
+            with open(self.cached_results_file_path) as f:
+                self.cached_results = json.load(f)
+
+        self.dirty = False
+
+    def write_cache(self):
+        """
+        Write the current cache to 'hf_cache.json' if it has been modified.
+        """
+        if self.dirty:
+            with open(self.cached_results_file_path, 'w') as f:
+                json.dump(self.cached_results, f)
+            self.dirty = False
+
+    def get_cached_result(self, model: str, prompt: str | list[int],
+                          max_tokens: int) -> dict:
+        """
+        Retrieve a cached result for the given model, prompt, and max_tokens.
+        Returns an empty dictionary if no cache entry is found.
+        """
+        if isinstance(prompt, list):
+            prompt = self._token_ids_to_string(prompt)
+        max_tokens = str(max_tokens)
+
+        return self.cached_results.get(model, {}).get(prompt,
+                                                      {}).get(max_tokens, {})
+
+    def add_to_cache(self, model: str, prompt: str | list[int],
+                     max_tokens: int, result: dict):
+        """
+        Add a new result to the cache for the given model, prompt, and
+        max_tokens. Marks the cache as 'dirty' to indicate that it needs to be
+        written to disk.
+        """
+        if isinstance(prompt, list):
+            prompt = self._token_ids_to_string(prompt)
+        max_tokens = str(max_tokens)
+
+        self.cached_results.setdefault(model,
+                                       {}).setdefault(prompt, {}).setdefault(
+                                           max_tokens, result)
+        self.dirty = True
+
+    def _token_ids_to_string(self, token_ids: list[int]) -> str:
+        """Use a string to represent a list of token ids, so that it can be
+        hashed and used as a json key."""
+
+        return "__tokens__" + "_".join(str(token_id) for token_id in token_ids)
diff --git a/tests/spyre_util.py b/tests/spyre_util.py
index 36f8d702c..05c282b33 100644
--- a/tests/spyre_util.py
+++ b/tests/spyre_util.py
@@ -13,6 +13,7 @@
 import pytest
 import requests
 import torch
+from hf_result_cache import HFResultCache
 from sentence_transformers import SentenceTransformer, util
 from transformers import AutoModelForCausalLM, AutoTokenizer
 from vllm import LLM, SamplingParams
@@ -28,6 +29,8 @@
 ISCLOSE_REL_TOL_CPU = 0.35
 ISCLOSE_REL_TOL_SPYRE = 0.35
 
+HF_RESULT_CACHE = HFResultCache()
+
 
 def force_engine_shutdown(llm: LLM):
     """
@@ -258,17 +261,32 @@ def generate_hf_output(
         max_new_tokens: Union[int, list[int]],
         ignore_eos: bool = False,
         include_prompt: bool = False) -> list[dict[str, Any]]:
+    """Loads and runs the model on cpu with transformers, caching the results.
+    Returns cached results if any are found to avoid overhead."""
 
     if not isinstance(max_new_tokens, list):
         max_new_tokens = [max_new_tokens] * len(prompts)
 
+    results = []
+    for prompt, max_tokens in zip(prompts, max_new_tokens):
+        results.append(
+            HF_RESULT_CACHE.get_cached_result(model, prompt, max_tokens))
+
+    if all(results):
+        # Everything hit cache
+        return results
+
     hf_model = AutoModelForCausalLM.from_pretrained(model)
     hf_tokenizer = AutoTokenizer.from_pretrained(model)
     if ignore_eos:
         hf_model.generation_config.eos_token_id = None
 
-    results = []
     for prompt_index, prompt in enumerate(prompts):
+
+        if results[prompt_index]:
+            # Already have cached result
+            continue
+
         hf_input_tokens = hf_tokenizer(prompt, return_tensors="pt").input_ids \
                     if isinstance(prompt[0], str) \
                     else torch.tensor([prompts[prompt_index]])
@@ -302,8 +320,14 @@ def generate_hf_output(
         result['logprobs'] = tuple(result['logprobs'])
         if include_prompt:
             result['prompt'] = prompt
-        results.append(result)
 
+        # Save and cache new result
+        results[prompt_index] = result
+        HF_RESULT_CACHE.add_to_cache(model, prompt,
+                                     max_new_tokens[prompt_index], result)
+
+    # Write back to the cache
+    HF_RESULT_CACHE.write_cache()
     return results
 
 
@@ -343,8 +367,13 @@ def compare_results(
             print(f"        vLLM:  {repr(vllm_result['text']):s}{err_msg}")
             print()
 
+        if isinstance(hf_result['token_ids'], list):
+            hf_result['token_ids'] = tuple(hf_result['token_ids'])
+
         assert DISABLE_ASSERTS or backend == 'sendnn' or\
-            hf_result['token_ids'] == vllm_result['token_ids']
+            hf_result['token_ids'] == vllm_result['token_ids'], \
+            f"Token ids differ: {hf_result['token_ids']} != " \
+            f"{vllm_result['token_ids']}"
 
         if len(hf_result['tokens']) > 0:
             print("   token id. token               logprob      "