Skip to content

Commit e21202c

Browse files
WredeWrede
committed
add scaleout-util
1 parent b3e741a commit e21202c

36 files changed

Lines changed: 5201 additions & 0 deletions

‎scaleout-util/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
scaleoututil/grpc/tmp.proto

‎scaleout-util/pyproject.toml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
[project]
2+
name = "scaleoututil"
3+
version = "1.0.0"
4+
description = "A library for shared resources in the Scaleout Edge"
5+
authors = [{ name = "Scaleout Systems AB" }]
6+
requires-python = ">=3.9, <=3.13"
7+
readme = "scaleoututil/README.rst"
8+
license = "Apache-2.0"
9+
dependencies = [
10+
"grpcio>=1.68.1,<2",
11+
"grpcio-tools>=1.68.1,<=1.70",
12+
"protobuf>=5.0.0,<6.31.0",
13+
"PyYAML~=6.0",
14+
"numpy>=1.21.6",
15+
"requests",
16+
"virtualenv",
17+
]
18+
19+
[project.urls]
20+
Homepage = "https://scaleoutsystems.com"
21+
Repository = "https://github.com/scaleoutsystems/scaleout-client"
22+
23+
[dependency-groups]
24+
dev = [
25+
"pytest~=8.4",
26+
"black~=25.9",
27+
"mypy~=1.0",
28+
"mypy-protobuf"
29+
]
30+
31+
[build-system]
32+
requires = ["hatchling"]
33+
build-backend = "hatchling.build"

‎scaleout-util/scaleout.proto‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
syntax = "proto3";
2+
3+
package scaleout;
4+
5+
import "google/protobuf/timestamp.proto";
6+
import "google/protobuf/wrappers.proto";
7+
8+
message Response {
9+
string response = 1;
10+
string parameters = 2;
11+
}
12+
13+
// Status
14+
15+
enum LogLevel {
16+
NONE = 0;
17+
INFO = 1;
18+
DEBUG = 2;
19+
WARNING = 3;
20+
ERROR = 4;
21+
AUDIT = 5;
22+
}
23+
24+
message Status {
25+
string client_id = 1;
26+
string status = 2;
27+
LogLevel log_level = 3;
28+
google.protobuf.Timestamp timestamp = 6;
29+
string type = 7;
30+
}
31+
32+
// Combiner from Client
33+
34+
message ModelUpdate {
35+
string client_id = 1;
36+
string model_id = 3;
37+
string model_update_id = 4;
38+
string correlation_id = 5;
39+
google.protobuf.Timestamp timestamp = 6;
40+
string meta = 7;
41+
string config = 8;
42+
string round_id = 9;
43+
string session_id = 10;
44+
}
45+
46+
message ModelValidation {
47+
string client_id = 1;
48+
string model_id = 3;
49+
string data = 4;
50+
string correlation_id = 5;
51+
google.protobuf.Timestamp timestamp = 6;
52+
string session_id = 8;
53+
}
54+
55+
message ModelPrediction {
56+
string client_id = 1;
57+
string model_id = 3;
58+
string data = 4;
59+
string correlation_id = 5;
60+
google.protobuf.Timestamp timestamp = 6;
61+
string session_id = 8;
62+
}
63+
64+
message BackwardCompletion {
65+
string client_id = 1;
66+
string gradient_id = 3;
67+
string correlation_id = 4;
68+
string session_id = 5;
69+
google.protobuf.Timestamp timestamp = 6;
70+
}
71+
72+
message ModelMetric {
73+
string client_id = 1;
74+
repeated MetricElem metrics = 2;
75+
google.protobuf.Timestamp timestamp = 3;
76+
google.protobuf.UInt32Value step = 4;
77+
string model_id = 5;
78+
string round_id = 6;
79+
string session_id = 7;
80+
}
81+
82+
message MetricElem {
83+
string key = 1;
84+
float value = 2;
85+
}
86+
87+
message AttributeMessage {
88+
string client_id = 1;
89+
repeated AttributeElem attributes = 2;
90+
google.protobuf.Timestamp timestamp = 3;
91+
}
92+
93+
message AttributeElem {
94+
string key = 1;
95+
string value = 2;
96+
}
97+
98+
message TelemetryMessage {
99+
string client_id = 1;
100+
repeated TelemetryElem telemetries = 2;
101+
google.protobuf.Timestamp timestamp = 3;
102+
}
103+
104+
message TelemetryElem {
105+
string key = 1;
106+
float value = 2;
107+
}
108+
109+
// Task related
110+
111+
112+
message TaskRequest {
113+
string model_id = 3;
114+
string data = 4;
115+
string correlation_id = 5;
116+
string timestamp = 6;
117+
string session_id = 8;
118+
string round_id = 10;
119+
string type = 11;
120+
string status = 12;
121+
string client_id = 1;
122+
}
123+
124+
message ActivityReport {
125+
string node_id = 1; // If sent from a client
126+
string correlation_id = 2;
127+
string status = 3;
128+
bool done = 4;
129+
string response = 5; // json
130+
}
131+
132+
message Heartbeat {
133+
string client_id = 1;
134+
float memory_utilisation = 2;
135+
float cpu_utilisation = 3;
136+
}
137+
138+
message ClientAvailableMessage {
139+
string client_id = 1;
140+
string data = 2;
141+
string timestamp = 3;
142+
}
143+
144+
message ClientAnnounceRequest {
145+
string client_id = 1;
146+
string type = 2;
147+
string parameters = 3;
148+
}
149+
150+
151+
service CombinerClient {
152+
rpc SendModelUpdate (ModelUpdate) returns (Response);
153+
rpc SendModelValidation (ModelValidation) returns (Response);
154+
rpc SendModelPrediction (ModelPrediction) returns (Response);
155+
rpc SendBackwardCompletion (BackwardCompletion) returns (Response);
156+
rpc SendModelMetric(ModelMetric) returns (Response);
157+
rpc SendAttributeMessage(AttributeMessage) returns (Response);
158+
rpc SendTelemetryMessage(TelemetryMessage) returns (Response);
159+
160+
rpc Announce(ClientAnnounceRequest) returns (Response);
161+
162+
rpc PollAndReport(ActivityReport) returns (TaskRequest);
163+
164+
// Status endpoint
165+
rpc SendStatus (Status) returns (Response);
166+
// Client messaging to stay engaged.
167+
rpc SendHeartbeat (Heartbeat) returns (Response);
168+
// Stream endpoints for training/validation pub/sub
169+
rpc TaskStream (ClientAvailableMessage) returns (stream TaskRequest);
170+
}
171+
172+
173+
// ModelService
174+
175+
enum ModelStatus {
176+
UNKNOWN = 0;
177+
IN_PROGRESS = 1;
178+
OK = 2;
179+
FAILED = 3;
180+
}
181+
182+
message ModelRequest {
183+
string model_id = 3;
184+
}
185+
186+
message FileChunk {
187+
bytes data = 1;
188+
}
189+
190+
message ModelResponse {
191+
ModelStatus status = 3;
192+
string message = 4;
193+
}
194+
195+
service ModelService {
196+
rpc Upload(stream FileChunk) returns (ModelResponse);
197+
rpc Download(ModelRequest) returns (stream FileChunk);
198+
}

‎scaleout-util/scaleoututil/README.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
This is the Utility package for FEDn
2+
=====================

‎scaleout-util/scaleoututil/__init__.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
from scaleoututil import config
2+
import scaleoututil.grpc.scaleout_pb2_grpc as grpc_service
3+
import scaleoututil.grpc.scaleout_pb2 as grpc_message
4+
from scaleoututil.logging import FednLogger
5+
6+
__all__ = ["config", "grpc_service", "grpc_message", "FednLogger"]

0 commit comments

Comments
 (0)