Skip to content

Commit eb6d107

Browse files
faridseififaridseifi
committed
refactoring
1 parent b18bbf4 commit eb6d107

File tree

6 files changed

+153
-156
lines changed

6 files changed

+153
-156
lines changed

‎src/python_activator/api.py‎

Lines changed: 6 additions & 143 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,10 @@
11
import os
2-
import importlib
3-
import json
4-
import subprocess
5-
import sys
62
from pathlib import Path
7-
83
import uvicorn
9-
import yaml
104
from fastapi import FastAPI, Header, HTTPException, Request
115
from fastapi.openapi.docs import get_swagger_ui_html
12-
136
from python_activator.loader import load_packages
14-
15-
16-
class knowledge_object:
17-
def __init__(self, name, status, function, id, url, entry):
18-
self.id = id
19-
self.name = name
20-
self.status = status
21-
self.function = function
22-
self.url = url
23-
self.entry = entry
24-
25-
async def execute(self, request):
26-
data = (
27-
await request.json()
28-
) # if content_type == 'application/json': data = await request.json() else: data="test"
29-
try:
30-
return self.function(data)
31-
except TypeError as e:
32-
raise HTTPException(
33-
status_code=422,
34-
detail={"status": self.status, "cause": e.__cause__}
35-
)
7+
from python_activator.installer import install_packages
368

379

3810
app = FastAPI()
@@ -94,128 +66,19 @@ async def execute_endpoint(
9466
)
9567

9668

97-
# Install requirements using the requirements.txt file for each package
98-
def install_requirements(modulepath):
99-
dependency_requirements = Path(modulepath).joinpath("requirements.txt")
100-
101-
# To Do: using pip install bellow explore installing requirements in a folder specific to the ko
102-
# you may need to add that folder to the sys.path
103-
104-
# uses 'pip install -r requirements.txt' to install requirements
105-
if os.path.exists(dependency_requirements):
106-
subprocess.check_call(
107-
[sys.executable, "-m", "pip", "install", "-r", dependency_requirements]
108-
)
109-
110-
111-
# look into the main directory that has all the packages and have the python ones installed
112-
def install_packages(directory, manifest: dict):
113-
for ko in manifest:
114-
install_module(directory, manifest[ko])
115-
list_installed_packages()
116-
117-
118-
def install_module(directory, ko):
119-
Knowledge_Objects[ko.name] = knowledge_object(ko.name, ko.status, None, "", "", "")
120-
121-
try:
122-
modulepath = os.path.join(Path(directory).joinpath(ko.name), "")
123-
#########delete me: temporarily ignoring execute package
124-
if ko.name == "python-executive-v1.0":
125-
return
126-
127-
if ko.status != "Ready for install":
128-
return
129-
130-
Knowledge_Objects[ko.name].status = "Activating"
131-
132-
# get metadata and deployment files
133-
with open(Path(modulepath).joinpath("deployment.yaml"), "r") as file:
134-
deployment_data = yaml.safe_load(file)
135-
with open(Path(modulepath).joinpath("metadata.json"), "r") as file:
136-
metadata = json.load(file)
137-
first_key = next(iter(deployment_data))
138-
second_key = next(iter(deployment_data[first_key]))
139-
140-
# do not install non python packages
141-
if deployment_data[first_key][second_key]["engine"] != "python":
142-
del Knowledge_Objects[ko.name]
143-
Knowledge_Objects[metadata["@id"]] = knowledge_object(
144-
ko.name,
145-
"Knowledge object is not activated. It is not a python object.",
146-
None,
147-
metadata["@id"],
148-
"",
149-
"",
150-
)
151-
return
152-
153-
# install requirements
154-
install_requirements(modulepath)
155-
156-
# import module, get the function and add it to the dictionary
157-
spec = importlib.util.spec_from_file_location(
158-
ko.name, Path(modulepath).joinpath("src").joinpath("__init__.py")
159-
)
160-
pac_a = importlib.util.module_from_spec(spec)
161-
sys.modules[pac_a.__name__] = pac_a
162-
module = importlib.import_module(
163-
str.replace(
164-
deployment_data[first_key][second_key]["entry"], "src/", "."
165-
).replace(".py", ""),
166-
pac_a.__name__,
167-
)
168-
mymethod = getattr(module, deployment_data[first_key][second_key]["function"])
169-
del Knowledge_Objects[ko.name]
170-
Knowledge_Objects[metadata["@id"]] = knowledge_object(
171-
ko.name,
172-
"Activated",
173-
mymethod,
174-
metadata["@id"],
175-
"",
176-
Path(modulepath).joinpath(deployment_data[first_key][second_key]["entry"]),
177-
)
178-
except Exception as e:
179-
Knowledge_Objects[ko.name].status = "Faield activating with error: " + repr(e)
180-
181-
182-
def list_installed_packages():
183-
print("-------------------\nPackages installed:")
184-
print("{:<4}. {:<30} {:<30} {:<30}".format("", "ID", "NAME", "STATUS"))
185-
for i, item in enumerate(Knowledge_Objects):
186-
print(
187-
"{:<4}. {:<30} {:<30} {:<30}".format(
188-
str(i),
189-
Knowledge_Objects[item].id,
190-
Knowledge_Objects[item].name[:30],
191-
Knowledge_Objects[item].status[:50],
192-
)
193-
)
194-
print("-------------------")
195-
196-
19769
# run install if the app is starated using poetry run uvicorn python_activator.api:app --reload
19870
@app.on_event("startup")
19971
async def startup_event():
20072
print(">>>>>> running startup event")
201-
if os.environ.get("COLLECTION_PATH"):
202-
object_directory = os.path.join(Path(os.environ["COLLECTION_PATH"]), "")
203-
else:
204-
object_directory = os.path.join(Path(os.getcwd()).joinpath("pyshelf"), "")
205-
206-
manifest = load_packages(object_directory)
207-
install_packages(object_directory, manifest)
208-
try:
209-
del os.environ["COLLECTION_PATH"]
210-
del os.environ["MANIFEST_PATH"]
211-
except Exception as e:
212-
print("error deleting env variables")
213-
73+
global Knowledge_Objects
74+
Knowledge_Objects, object_directory = load_packages()
75+
Knowledge_Objects = install_packages(object_directory, Knowledge_Objects)
76+
21477

21578
# run virtual server when running this .py file directly for debugging. It will look for objects at {code folder}/pyshelf
21679
if __name__ == "__main__":
21780
print(">>>>>running with debug<<<<<")
218-
# os.environ["MANIFEST_PATH"]="/home/faridsei/dev/test/package/manifest.json"
81+
os.environ["MANIFEST_PATH"]="/home/faridsei/dev/test/manifest/manifest.json"
21982
# os.environ["MANIFEST_PATH"] = "https://github.com/kgrid-objects/example-collection/releases/download/4.2.1/manifest.json"
22083
os.environ["COLLECTION_PATH"] = "/home/faridsei/dev/test/pyshelf/"
22184
uvicorn.run(app, host="127.0.0.1", port=8000)

‎src/python_activator/cli.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from python_activator.api import *
22
import typer
3+
import importlib
34

45
cli = typer.Typer()
56

‎src/python_activator/installer.py‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import os
2+
import importlib
3+
import json
4+
import subprocess
5+
import sys
6+
import yaml
7+
from python_activator.loader import knowledge_object
8+
from pathlib import Path
9+
10+
Knowledge_Objects = {}
11+
12+
13+
# Install requirements using the requirements.txt file for each package
14+
def install_requirements(modulepath):
15+
dependency_requirements = Path(modulepath).joinpath("requirements.txt")
16+
17+
# uses 'pip install -r requirements.txt' to install requirements
18+
if os.path.exists(dependency_requirements):
19+
subprocess.check_call(
20+
[sys.executable, "-m", "pip", "install", "-r", dependency_requirements]
21+
)
22+
23+
24+
# look into the main directory that has all the packages and have the python ones installed
25+
def install_packages(directory, Knowledge_Objects_List: dict):
26+
for ko in Knowledge_Objects_List:
27+
install_module(directory, Knowledge_Objects_List[ko])
28+
list_installed_packages()
29+
30+
try:
31+
del os.environ["COLLECTION_PATH"]
32+
del os.environ["MANIFEST_PATH"]
33+
except Exception as e:
34+
print("error deleting env variables")
35+
36+
return Knowledge_Objects
37+
38+
39+
def install_module(directory, ko):
40+
Knowledge_Objects[ko.name] = knowledge_object(ko.name, ko.status, None, "", "", "")
41+
42+
try:
43+
modulepath = os.path.join(Path(directory).joinpath(ko.name), "")
44+
#########delete me: temporarily ignoring execute package
45+
if ko.name == "python-executive-v1.0":
46+
return
47+
48+
if ko.status != "Ready for install":
49+
return
50+
51+
Knowledge_Objects[ko.name].status = "Activating"
52+
53+
# get metadata and deployment files
54+
with open(Path(modulepath).joinpath("deployment.yaml"), "r") as file:
55+
deployment_data = yaml.safe_load(file)
56+
with open(Path(modulepath).joinpath("metadata.json"), "r") as file:
57+
metadata = json.load(file)
58+
first_key = next(iter(deployment_data))
59+
second_key = next(iter(deployment_data[first_key]))
60+
61+
# do not install non python packages
62+
if deployment_data[first_key][second_key]["engine"] != "python":
63+
del Knowledge_Objects[ko.name]
64+
Knowledge_Objects[metadata["@id"]] = knowledge_object(
65+
ko.name, "Knowledge object is not activated. It is not a python object.",
66+
None, metadata["@id"], "", "",
67+
)
68+
return
69+
70+
# install requirements
71+
install_requirements(modulepath)
72+
73+
# import module, get the function and add it to the dictionary
74+
spec = importlib.util.spec_from_file_location(
75+
ko.name, Path(modulepath).joinpath("src").joinpath("__init__.py")
76+
)
77+
pac_a = importlib.util.module_from_spec(spec)
78+
sys.modules[pac_a.__name__] = pac_a
79+
module = importlib.import_module(
80+
str.replace(
81+
deployment_data[first_key][second_key]["entry"], "src/", "."
82+
).replace(".py", ""),
83+
pac_a.__name__,
84+
)
85+
mymethod = getattr(module, deployment_data[first_key][second_key]["function"])
86+
del Knowledge_Objects[ko.name]
87+
Knowledge_Objects[metadata["@id"]] = knowledge_object(
88+
ko.name, "Activated", mymethod, metadata["@id"], "",
89+
Path(modulepath).joinpath(deployment_data[first_key][second_key]["entry"]),
90+
)
91+
except Exception as e:
92+
Knowledge_Objects[ko.name].status = "Faield activating with error: " + repr(e)
93+
94+
95+
def list_installed_packages():
96+
print("-------------------\nPackages installed:")
97+
print("{:<4}. {:<30} {:<30} {:<30}".format("", "ID", "NAME", "STATUS"))
98+
for i, item in enumerate(Knowledge_Objects):
99+
print(
100+
"{:<4}. {:<30} {:<30} {:<30}".format(
101+
str(i),
102+
Knowledge_Objects[item].id,
103+
Knowledge_Objects[item].name[:30],
104+
Knowledge_Objects[item].status[:50],
105+
)
106+
)
107+
print("-------------------")

‎src/python_activator/loader.py‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,30 +6,52 @@
66
import os
77
from io import BytesIO
88
from pathlib import Path
9+
from fastapi import HTTPException
910

10-
class ko_object:
11-
def __init__(self, name, status):
11+
class knowledge_object:
12+
def __init__(self, name, status, function=None, id="", url="", entry=""):
13+
self.id = id
1214
self.name = name
1315
self.status = status
16+
self.function = function
17+
self.url = url
18+
self.entry = entry
19+
20+
async def execute(self, request):
21+
data = (
22+
await request.json()
23+
) # if content_type == 'application/json': data = await request.json() else: data="test"
24+
try:
25+
return self.function(data)
26+
except TypeError as e:
27+
raise HTTPException(
28+
status_code=422,
29+
detail={"status": self.status, "cause": e.__cause__}
30+
)
31+
1432

33+
def load_packages() -> dict:
34+
if os.environ.get("COLLECTION_PATH"):
35+
object_directory = os.path.join(Path(os.environ["COLLECTION_PATH"]), "")
36+
else:
37+
object_directory = os.path.join(Path(os.getcwd()).joinpath("pyshelf"), "")
1538

16-
def load_packages(object_directory: str) -> dict:
1739
manifest_path = os.environ.get("MANIFEST_PATH")
1840
scanned_directories = [f.name for f in os.scandir(object_directory) if f.is_dir()]
1941
output_manifest = {}
2042

2143
# 0. if no manifest provided consider list of existing knowledge object folders as manifest
2244
if not manifest_path:
2345
for item in scanned_directories:
24-
output_manifest[str.replace(item, ".zip", "")] = ko_object(
46+
output_manifest[str.replace(item, ".zip", "")] = knowledge_object(
2547
str.replace(item, ".zip", ""), "Ready for install"
2648
)
2749

2850
#create a manifest
2951
with open(Path(object_directory).joinpath('manifest_generated.json'), 'w') as json_file:
3052
json.dump({"manifest": [obj.name for obj in output_manifest.values()]}, json_file)
3153

32-
return output_manifest
54+
return output_manifest, object_directory
3355

3456
with urllib.request.urlopen(get_uri(manifest_path)) as response:
3557
input_manifest = json.loads(response.read())["manifest"] #load manifest
@@ -42,7 +64,7 @@ def load_packages(object_directory: str) -> dict:
4264
ko_path = os.path.dirname(manifest_path) + "/" + manifest_item
4365
ko_name = os.path.splitext(os.path.basename(manifest_item))[0]
4466

45-
output_manifest[ko_name] = ko_object(ko_name, "")
67+
output_manifest[ko_name] = knowledge_object(ko_name, "")
4668

4769
# 1. if knowledge object already in the collection directory continue to the next ko in list
4870
if ko_name in scanned_directories:
@@ -60,12 +82,12 @@ def load_packages(object_directory: str) -> dict:
6082

6183
output_manifest[ko_name].status = "Ready for install"
6284

63-
return output_manifest
85+
return output_manifest, object_directory
6486

6587

6688
def get_uri(path: str):
6789
uri = path
68-
if os.path.exists(path): # if a local path create the URI
90+
if os.path.isabs(path): # if a local path create the URI
6991
uri = pathlib.Path(path).as_uri() # adds file:// if local path
7092
return uri
7193

‎tests/test_dot_dirs.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
from python_activator.api import install_module
2-
from python_activator.loader import ko_object
1+
from python_activator.installer import install_module
2+
from python_activator.loader import knowledge_object
33
import os
44
import sys
55
import importlib
@@ -12,7 +12,7 @@ def test_basics():
1212
directory="/home/faridsei/dev/code/python-activator/etc/pyshelf/"
1313
#sys.path.append( directory )
1414

15-
install_module(directory, ko_object("python-multiartifact-v1-0","Ready to install"))
15+
install_module(directory, knowledge_object("python-multiartifact-v1-0","Ready to install"))
1616
#del sys.modules["ko_folder"]
1717
#sys.path.remove(
1818
# directory

0 commit comments

Comments
 (0)