Skip to content

Commit

Permalink
optimize data storage and loading
Browse files Browse the repository at this point in the history
  • Loading branch information
knikolaou committed May 14, 2024
1 parent 304a542 commit cdadfb5
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions papyrus/recorders/base_recorder.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def load(self):
"""
# By combining storage path and name, we can load the data
data = np.load(self.storage_path + self.name + ".npz")
return data
return dict(data)

def _measure(self, **neural_state):
"""
Expand Down Expand Up @@ -161,11 +161,12 @@ def _store(self, epoch: int):
if epoch % self.chunk_size == 0:
# Load the data from the database
try:
data = dict(self.load())
data = self.load()
# Append the new data
for key in self._results.keys():
data[key] = np.append(data[key], self._results[key], axis=0)
except:
# If the file does not exist, create a new one
except FileNotFoundError:
data = self._results

# Write the data back to the database
Expand Down

0 comments on commit cdadfb5

Please sign in to comment.