memory errors

[RLWOHIO]

How can we help you?
Running PheKnowLator on a Windows workstation with 48 core/512 GB RAM failed due to memory running out. Is it because Pickle used too much memory? Any suggestions? See error log below:

MemoryError Traceback (most recent call last)
Cell In[9], line 24
16 else:
17 kg = FullBuild(construction=construction_approach,
18 node_data=add_node_data_to_kg,
19 inverse_relations=add_inverse_relations_to_kg,
20 decode_owl=decode_owl_semantics,
21 cpus=cpus,
22 write_location=kg_directory_location)
---> 24 kg.construct_knowledge_graph()
25 ray.shutdown()

File w:\MyWork\PheKnowLator\pkt_kg\knowledge_graph.py:636, in FullBuild.construct_knowledge_graph(self)
634 graphs = [updates_pkt_namespace_identifiers(self.graph, self.construct_approach)] + g2
635 owlnets = OwlNets(graphs, self.write_location, kg_owl_main, self.construct_approach, self.owl_tools)
--> 636 results = [results[0]] + list(owlnets.runs_owlnets(self.cpus))
638 # STEP 7: WRITE OUT KNOWLEDGE GRAPH METADATA AND CREATE EDGE LISTS
639 log_str = '*** Writing Knowledge Graph Edge Lists ***'; print('\n' + log_str); logger.info(log_str)

File w:\MyWork\PheKnowLator\pkt_kg\owlnets.py:809, in OwlNets.runs_owlnets(self, cpus)
807 try: ray.init()
808 except RuntimeError: pass
--> 809 acts = [ray.remote(OwlNets).remote(self.graph, loc, f, cons, ot) for _ in range(cpus)] # type: ignore
810 for i in range(0, cpus): acts[i % cpus].cleans_owl_encoded_entities.remote(entities[i]) # type: ignore
...
-> 1245 return super().dump(obj)
1246 except RuntimeError as e:
1247 if len(e.args) > 0 and "recursion" in e.args[0]:

[RLWOHIO]

Here is more complete error log. This seems to suggest ray.cloudpickle is at issue of running out of memory.

---

MemoryError Traceback (most recent call last)
Cell In[9], line 24
16 else:
17 kg = FullBuild(construction=construction_approach,
18 node_data=add_node_data_to_kg,
19 inverse_relations=add_inverse_relations_to_kg,
20 decode_owl=decode_owl_semantics,
21 cpus=cpus,
22 write_location=kg_directory_location)
---> 24 kg.construct_knowledge_graph()
25 ray.shutdown()

File w:\MyWork\PheKnowLator\pkt_kg\knowledge_graph.py:636, in FullBuild.construct_knowledge_graph(self)
634 graphs = [updates_pkt_namespace_identifiers(self.graph, self.construct_approach)] + g2
635 owlnets = OwlNets(graphs, self.write_location, kg_owl_main, self.construct_approach, self.owl_tools)
--> 636 results = [results[0]] + list(owlnets.runs_owlnets(self.cpus))
638 # STEP 7: WRITE OUT KNOWLEDGE GRAPH METADATA AND CREATE EDGE LISTS
639 log_str = '*** Writing Knowledge Graph Edge Lists ***'; print('\n' + log_str); logger.info(log_str)

File w:\MyWork\PheKnowLator\pkt_kg\owlnets.py:809, in OwlNets.runs_owlnets(self, cpus)
807 try: ray.init()
808 except RuntimeError: pass
--> 809 acts = [ray.remote(OwlNets).remote(self.graph, loc, f, cons, ot) for _ in range(cpus)] # type: ignore
810 for i in range(0, cpus): acts[i % cpus].cleans_owl_encoded_entities.remote(entities[i]) # type: ignore
811 _ = ray.wait([x.gets_owlnets_graph.remote() for x in acts], num_returns=len(acts)) # type: ignore

File c:\Users\rwang\AppData\Local\miniforge3\envs\PK3\Lib\site-packages\ray\actor.py:764, in ActorClass.remote(self, *args, **kwargs)
752 def remote(self, *args, **kwargs):
753 """Create an actor.
754
755 Args:
(...) 762 A handle to the newly created actor.
763 """
--> 764 return self._remote(args=args, kwargs=kwargs, **self._default_options)

File c:\Users\rwang\AppData\Local\miniforge3\envs\PK3\Lib\site-packages\ray_private\auto_init_hook.py:21, in wrap_auto_init..auto_init_wrapper(*args, **kwargs)
18 @wraps(fn)
19 def auto_init_wrapper(*args, **kwargs):
20 auto_init_ray()
---> 21 return fn(*args, **kwargs)

File c:\Users\rwang\AppData\Local\miniforge3\envs\PK3\Lib\site-packages\ray\util\tracing\tracing_helper.py:384, in _tracing_actor_creation.._invocation_actor_class_remote_span(self, args, kwargs, *_args, **_kwargs)
382 if not _is_tracing_enabled():
383 assert "_ray_trace_ctx" not in kwargs
--> 384 return method(self, args, kwargs, *_args, **_kwargs)
386 class_name = self.ray_metadata.class_name
387 method_name = "init"

File c:\Users\rwang\AppData\Local\miniforge3\envs\PK3\Lib\site-packages\ray\actor.py:1217, in ActorClass._remote(self, args, kwargs, **actor_options)
1207 func_name = meta.actor_creation_function_descriptor.function_name
1208 meta.actor_creation_function_descriptor = (
1209 cross_language._get_function_descriptor_for_actor_method(
1210 meta.language,
(...) 1214 )
1215 )
-> 1217 actor_id = worker.core_worker.create_actor(
1218 meta.language,
1219 meta.actor_creation_function_descriptor,
1220 creation_args,
1221 max_restarts,
1222 max_task_retries,
1223 resources,
1224 actor_placement_resources,
1225 max_concurrency,
1226 detached,
1227 name if name is not None else "",
1228 namespace if namespace is not None else "",
1229 is_asyncio,
1230 # Store actor_method_cpu in actor handle's extension data.
1231 extension_data=str(actor_method_cpu),
1232 serialized_runtime_env_info=serialized_runtime_env_info or "{}",
1233 concurrency_groups_dict=concurrency_groups_dict or dict(),
1234 max_pending_calls=max_pending_calls,
1235 scheduling_strategy=scheduling_strategy,
1236 enable_task_events=enable_task_events,
1237 labels=actor_options.get("_labels"),
1238 )
1240 if _actor_launch_hook:
1241 _actor_launch_hook(
1242 meta.actor_creation_function_descriptor, resources, scheduling_strategy
1243 )

File python\ray\_raylet.pyx:3837, in ray._raylet.CoreWorker.create_actor()

File python\ray\_raylet.pyx:3843, in ray._raylet.CoreWorker.create_actor()

File python\ray\_raylet.pyx:870, in ray._raylet.prepare_args_and_increment_put_refs()

File python\ray\_raylet.pyx:861, in ray._raylet.prepare_args_and_increment_put_refs()

File python\ray\_raylet.pyx:911, in ray._raylet.prepare_args_internal()

File c:\Users\rwang\AppData\Local\miniforge3\envs\PK3\Lib\site-packages\ray_private\serialization.py:556, in SerializationContext.serialize(self, value)
554 return RawSerializedObject(value)
555 else:
--> 556 return self._serialize_to_msgpack(value)

File c:\Users\rwang\AppData\Local\miniforge3\envs\PK3\Lib\site-packages\ray_private\serialization.py:534, in SerializationContext._serialize_to_msgpack(self, value)
532 if python_objects:
533 metadata = ray_constants.OBJECT_METADATA_TYPE_PYTHON
--> 534 pickle5_serialized_object = self._serialize_to_pickle5(
535 metadata, python_objects
536 )
537 else:
538 pickle5_serialized_object = None

File c:\Users\rwang\AppData\Local\miniforge3\envs\PK3\Lib\site-packages\ray_private\serialization.py:481, in SerializationContext._serialize_to_pickle5(self, metadata, value)
479 except Exception as e:
480 self.get_and_clear_contained_object_refs()
--> 481 raise e
482 finally:
483 self.set_out_of_band_serialization()

File c:\Users\rwang\AppData\Local\miniforge3\envs\PK3\Lib\site-packages\ray_private\serialization.py:476, in SerializationContext._serialize_to_pickle5(self, metadata, value)
474 try:
475 self.set_in_band_serialization()
--> 476 inband = pickle.dumps(
477 value, protocol=5, buffer_callback=writer.buffer_callback
478 )
479 except Exception as e:
480 self.get_and_clear_contained_object_refs()

File c:\Users\rwang\AppData\Local\miniforge3\envs\PK3\Lib\site-packages\ray\cloudpickle\cloudpickle.py:1479, in dumps(obj, protocol, buffer_callback)
1477 with io.BytesIO() as file:
1478 cp = Pickler(file, protocol=protocol, buffer_callback=buffer_callback)
-> 1479 cp.dump(obj)
1480 return file.getvalue()

File c:\Users\rwang\AppData\Local\miniforge3\envs\PK3\Lib\site-packages\ray\cloudpickle\cloudpickle.py:1245, in Pickler.dump(self, obj)
1243 def dump(self, obj):
1244 try:
-> 1245 return super().dump(obj)
1246 except RuntimeError as e:
1247 if len(e.args) > 0 and "recursion" in e.args[0]:

MemoryError: