Skip to content

Commit 3bc649b

Browse files
WHALEEYEWHALEEYE
committed
draft
1 parent 3e4b5e2 commit 3bc649b

File tree

8 files changed

+349
-0
lines changed

8 files changed

+349
-0
lines changed

‎gui/__init__.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
2+
# Licensed under the Apache License, Version 2.0 (the “License”);
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an “AS IS” BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========

‎gui/envs/__init__.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
2+
# Licensed under the Apache License, Version 2.0 (the “License”);
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an “AS IS” BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
14+
from .android import ANDROID_ENV
15+
from .mac import MAC_ENV
16+
from .ubuntu import UBUNTU_ENV
17+
from .windows import WINDOWS_ENV
18+
19+
__all__ = ["MAC_ENV", "UBUNTU_ENV", "WINDOWS_ENV", "ANDROID_ENV"]

‎gui/envs/android.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
2+
# Licensed under the Apache License, Version 2.0 (the “License”);
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an “AS IS” BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
14+
from crab import EnvironmentConfig
15+
from crab.actions.android_actions import (
16+
key_press,
17+
open_app_drawer,
18+
screenshot,
19+
setup,
20+
swipe,
21+
tap,
22+
write_text,
23+
)
24+
25+
ANDROID_ENV = EnvironmentConfig(
26+
name="android",
27+
action_space=[tap, key_press, write_text, swipe, open_app_drawer],
28+
observation_space=[screenshot],
29+
description=(
30+
"An Android operating system running on Google Pixel smartphone. The interface "
31+
"displays a current screenshot at each step and primarily supports interaction "
32+
"through tapping and typing. This device offers a suite of standard "
33+
"applications including Phone, Photos, Camera, Chrome, and Calendar, among "
34+
"others. Access the app drawer to view all installed applications on the "
35+
"device. The Google account is logged in, synchronized with the same account "
36+
"used in other environments."
37+
),
38+
extra_attributes={"device": None},
39+
reset=setup,
40+
)

‎gui/envs/mac.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
2+
# Licensed under the Apache License, Version 2.0 (the “License”);
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an “AS IS” BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
14+
15+
from crab.actions.desktop_actions import (
16+
click,
17+
key_press,
18+
press_hotkey,
19+
right_click,
20+
screenshot,
21+
search_application,
22+
write_text,
23+
)
24+
from crab.core import EnvironmentConfig
25+
26+
MAC_ENV = EnvironmentConfig(
27+
name="mac",
28+
action_space=[
29+
click,
30+
key_press,
31+
write_text,
32+
press_hotkey,
33+
search_application,
34+
right_click,
35+
],
36+
observation_space=[screenshot],
37+
description=(
38+
"A MacOS desktop operating system. The interface displays a current screenshot "
39+
"at each step and primarily supports interaction via mouse and keyboard. You "
40+
"are encouraged to use keyboard shortcuts and searching functionality to open"
41+
" applications in the system. This device includes system-related applications "
42+
"like Terminal, Finder, TextEdit, Settings, etc. For communication, Slack is "
43+
"available. It also features Safari as the web browser. The Google account is "
44+
"already logged in on Safari, synchronized with the same account used in other "
45+
"environments."
46+
),
47+
remote_url="http://127.0.0.1:8080",
48+
)

‎gui/envs/ubuntu.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
2+
# Licensed under the Apache License, Version 2.0 (the “License”);
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an “AS IS” BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
14+
15+
from crab.actions.desktop_actions import (
16+
click,
17+
key_press,
18+
press_hotkey,
19+
right_click,
20+
screenshot,
21+
search_application,
22+
write_text,
23+
)
24+
from crab.core import EnvironmentConfig
25+
26+
UBUNTU_ENV = EnvironmentConfig(
27+
name="ubuntu",
28+
action_space=[
29+
click,
30+
key_press,
31+
write_text,
32+
press_hotkey,
33+
search_application,
34+
right_click,
35+
],
36+
observation_space=[screenshot],
37+
description=(
38+
"An Ubuntu desktop OS. The interface displays a current screenshot at each step"
39+
" and primarily supports interaction via mouse and keyboard. You are encouraged"
40+
"to use searching functionalities to open applications in the system. This "
41+
"device includes system-related applications like Terminal, Files, Text Editor,"
42+
" Vim, Settings, etc. It also features Firefox as the web browser, and the "
43+
"LibreOffice suite—Writer, Calc, and Impress. For communication, Slack is "
44+
"available. The Google account is logged in on Firefox, synchronized with the "
45+
"same account used in other environments."
46+
),
47+
remote_url="http://127.0.0.1:8080",
48+
)

‎gui/envs/windows.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
2+
# Licensed under the Apache License, Version 2.0 (the “License”);
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an “AS IS” BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
14+
15+
from crab.actions.desktop_actions import (
16+
click,
17+
key_press,
18+
press_hotkey,
19+
right_click,
20+
screenshot,
21+
search_application,
22+
write_text,
23+
)
24+
from crab.core import EnvironmentConfig
25+
26+
WINDOWS_ENV = EnvironmentConfig(
27+
name="windows",
28+
action_space=[
29+
click,
30+
key_press,
31+
write_text,
32+
press_hotkey,
33+
search_application,
34+
right_click,
35+
],
36+
observation_space=[screenshot],
37+
description=(
38+
"A Windows desktop OS. The interface displays a current screenshot at each step"
39+
" and primarily supports interaction via mouse and keyboard. You are encouraged"
40+
"to use searching functionalities to open applications in the system. This "
41+
"device includes system-related applications like Powershell, Explorer, "
42+
"Notepad, etc. It also features Edge as the web browser. For communication, "
43+
"Slack is available. The Google account is logged in on Edge, synchronized with"
44+
" the same account used in other environments."
45+
),
46+
remote_url="http://127.0.0.1:8080",
47+
)

‎gui/host_os.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
2+
# Licensed under the Apache License, Version 2.0 (the “License”);
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an “AS IS” BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
14+
from enum import Enum
15+
16+
17+
class HostOS(str, Enum):
18+
WINDOWS = "ubuntu"
19+
LINUX = "ubuntu"
20+
MAC = "macos"
21+
UNKNOWN = "unknown"

‎gui/main.py‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
2+
# Licensed under the Apache License, Version 2.0 (the “License”);
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an “AS IS” BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
# =========== Copyright 2024 @ CAMEL-AI.org. All Rights Reserved. ===========
14+
import warnings
15+
16+
from crab import (
17+
BenchmarkConfig,
18+
create_benchmark,
19+
)
20+
from crab.actions.crab_actions import complete
21+
from crab.actions.visual_prompt_actions import (
22+
get_elements_prompt,
23+
groundingdino_easyocr,
24+
)
25+
from crab.environments.macos import mac_env
26+
from gui.envs import ANDROID_ENV, UBUNTU_ENV, WINDOWS_ENV
27+
from gui.host_os import HostOS
28+
29+
warnings.filterwarnings("ignore")
30+
31+
32+
def check_host_os() -> HostOS:
33+
return HostOS.WINDOWS
34+
35+
36+
def get_benchmark(env: str, ubuntu_url: str):
37+
ubuntu_tool = {
38+
"screenshot": groundingdino_easyocr(font_size=16) >> get_elements_prompt
39+
}
40+
android_tool = {
41+
"screenshot": groundingdino_easyocr(font_size=40) >> get_elements_prompt
42+
}
43+
mac_tool = {
44+
"screenshot": groundingdino_easyocr(font_size=24) >> get_elements_prompt
45+
}
46+
47+
if env == "ubuntu":
48+
prompting_tools = {"ubuntu": ubuntu_tool}
49+
benchmark_config = BenchmarkConfig(
50+
name="ubuntu_benchmark",
51+
tasks=[],
52+
environments=[UBUNTU_ENV],
53+
prompting_tools=prompting_tools,
54+
root_action_space=[complete],
55+
multienv=True,
56+
)
57+
elif env == "android":
58+
prompting_tools = {"android": android_tool}
59+
benchmark_config = BenchmarkConfig(
60+
name="android_benchmark",
61+
tasks=[],
62+
environments=[ANDROID_ENV],
63+
prompting_tools=prompting_tools,
64+
root_action_space=[complete],
65+
multienv=True,
66+
)
67+
elif env == "cross":
68+
prompting_tools = {
69+
"android": android_tool,
70+
"ubuntu": ubuntu_tool,
71+
}
72+
benchmark_config = BenchmarkConfig(
73+
name="ubuntu_android_benchmark",
74+
tasks=[],
75+
environments=[UBUNTU_ENV, ANDROID_ENV],
76+
prompting_tools=prompting_tools,
77+
root_action_space=[complete],
78+
multienv=True,
79+
)
80+
elif env == "mac":
81+
prompting_tools = {"macos": mac_tool}
82+
benchmark_config = BenchmarkConfig(
83+
name="mac_benchmark",
84+
tasks=[],
85+
environments=[mac_env, ANDROID_ENV],
86+
prompting_tools=prompting_tools,
87+
root_action_space=[complete],
88+
multienv=True,
89+
)
90+
elif env == "windows":
91+
prompting_tools = {"windows": ubuntu_tool}
92+
benchmark_config = BenchmarkConfig(
93+
name="windows_benchmark",
94+
tasks=[],
95+
environments=[WINDOWS_ENV],
96+
prompting_tools=prompting_tools,
97+
root_action_space=[complete],
98+
multienv=True,
99+
)
100+
else:
101+
raise ValueError("Env not support")
102+
103+
benchmark_config.step_limit = 15
104+
return create_benchmark(benchmark_config)
105+
106+
107+
def main():
108+
host_os = check_host_os()
109+
print(f"Host OS: {host_os}")
110+
111+
112+
if __name__ == "__main__":
113+
main()

0 commit comments

Comments
 (0)