From d20090ee701ec158973447e982ac9033b4a36930 Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Sat, 23 Aug 2025 00:25:39 -0400 Subject: [PATCH 1/2] function to better estimate number of useable CPUs, as cpu_count() doesnt work in k8s e.g. --- fiftyone/core/utils.py | 47 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/fiftyone/core/utils.py b/fiftyone/core/utils.py index 4ab3f66bd07..a03a0fdb996 100644 --- a/fiftyone/core/utils.py +++ b/fiftyone/core/utils.py @@ -2893,6 +2893,53 @@ def recommend_process_pool_workers(num_workers=None): return num_workers +def get_cpu_count(): + """Returns the number of CPUs available to the current process. + + Returns: + the number of CPUs available to this process + """ + + # On Linux we can use cgroup to get a more accurate count of the + # available CPUs in some environments such as kubernetes where it will + # specify the CPU limit for the container + # 50000 100000 -> 500m CPU. + if sys.platform == "linux": + try: + with open("/sys/fs/cgroup/cpu.max") as f: + cpu_max = f.read().strip() + # max means no limit so we fall back to other methods + if cpu_max != "max": + quota, period = cpu_max.split() + if period != "0": + return max(1, int(int(quota) / int(period))) + except Exception: + pass + + # Python >= 3.13 adds this function which is more accurate to what the + # process can actually use than cpu_count() + # https://docs.python.org/3/library/os.html#os.process_cpu_count + try: + return multiprocessing.process_cpu_count() + except AttributeError: + pass + + # On Linux if python < 3.13, we can manually check the affinity which + # tells us the number of CPUs available to the process rather than the + # total number of CPUs in the system + try: + return len(os.sched_getaffinity(0)) + except AttributeError: + pass + + # Getting CPU count is not implemented on this platform so fall back to + # just 1 + try: + return multiprocessing.cpu_count() + except NotImplementedError: + return 1 + + sync_task_executor = None From d0807674faa28e836757a8e66db86a09d78ae264 Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Sat, 23 Aug 2025 20:21:32 -0400 Subject: [PATCH 2/2] rabbit fix --- fiftyone/core/utils.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fiftyone/core/utils.py b/fiftyone/core/utils.py index a03a0fdb996..ac85eabf71b 100644 --- a/fiftyone/core/utils.py +++ b/fiftyone/core/utils.py @@ -2907,12 +2907,10 @@ def get_cpu_count(): if sys.platform == "linux": try: with open("/sys/fs/cgroup/cpu.max") as f: - cpu_max = f.read().strip() + quota, period = f.read().strip().split() # max means no limit so we fall back to other methods - if cpu_max != "max": - quota, period = cpu_max.split() - if period != "0": - return max(1, int(int(quota) / int(period))) + if quota != "max" and period != "0": + return max(1, int(int(quota) / int(period))) except Exception: pass @@ -2920,7 +2918,7 @@ def get_cpu_count(): # process can actually use than cpu_count() # https://docs.python.org/3/library/os.html#os.process_cpu_count try: - return multiprocessing.process_cpu_count() + return os.process_cpu_count() except AttributeError: pass