fix benchmark

microsoft · Apr 17, 2024 · 680606e · 680606e
1 parent 0c88d58
commit 680606e
Show file tree

Hide file tree

Showing 74 changed files with 19 additions and 45 deletions.
diff --git a/.gitignore b/.gitignore
@@ -51,15 +51,15 @@ sample_configs/engine/openai*
 !sample_configs/engine/openai-example-config.json
 
 # Benchmark workloads
-benchmark/workloads/*/*.json
-benchmark/workloads/*/*.jsonl
+artifact_osdi24/workloads/*/*.json
+artifact_osdi24/workloads/*/*.jsonl
 *.parquet
 # It's safe!
-!benchmark/workloads/gpts/*.json 
+!artifact_osdi24/workloads/gpts/*.json 
 
 # Safe to upload
-!benchmark/workloads/metagpt/*.json
-!benchmark/workloads/metagpt/*.jsonl
+!artifact_osdi24/workloads/metagpt/*.json
+!artifact_osdi24/workloads/metagpt/*.jsonl
 
 # Evaluation data
 !benchmark/*/results/*/*.log

diff --git a/benchmark/benchmark_details.md → artifact_osdi24/eval_details.md b/benchmark/benchmark_details.md → artifact_osdi24/eval_details.md
diff --git a/...ads/MeetingBank-transcript/.gitattributes → ...ads/MeetingBank-transcript/.gitattributes b/...ads/MeetingBank-transcript/.gitattributes → ...ads/MeetingBank-transcript/.gitattributes
diff --git a/...rkloads/MeetingBank-transcript/LICENSE.md → ...rkloads/MeetingBank-transcript/LICENSE.md b/...rkloads/MeetingBank-transcript/LICENSE.md → ...rkloads/MeetingBank-transcript/LICENSE.md
diff --git a/...orkloads/MeetingBank-transcript/README.md → ...orkloads/MeetingBank-transcript/README.md b/...orkloads/MeetingBank-transcript/README.md → ...orkloads/MeetingBank-transcript/README.md
diff --git a/benchmark/workloads/README.md → artifact_osdi24/workloads/README.md b/benchmark/workloads/README.md → artifact_osdi24/workloads/README.md
diff --git a/...workloads/arxiv-march-2023/.gitattributes → ...workloads/arxiv-march-2023/.gitattributes b/...workloads/arxiv-march-2023/.gitattributes → ...workloads/arxiv-march-2023/.gitattributes
diff --git a/...mark/workloads/arxiv-march-2023/README.md → ...di24/workloads/arxiv-march-2023/README.md b/...mark/workloads/arxiv-march-2023/README.md → ...di24/workloads/arxiv-march-2023/README.md
diff --git a/...-march-2023/arxiv-sampled-1/article_0.txt → ...-march-2023/arxiv-sampled-1/article_0.txt b/...-march-2023/arxiv-sampled-1/article_0.txt → ...-march-2023/arxiv-sampled-1/article_0.txt
diff --git a/...-march-2023/arxiv-sampled-1/article_1.txt → ...-march-2023/arxiv-sampled-1/article_1.txt b/...-march-2023/arxiv-sampled-1/article_1.txt → ...-march-2023/arxiv-sampled-1/article_1.txt
diff --git a/...march-2023/arxiv-sampled-1/article_10.txt → ...march-2023/arxiv-sampled-1/article_10.txt b/...march-2023/arxiv-sampled-1/article_10.txt → ...march-2023/arxiv-sampled-1/article_10.txt
diff --git a/...march-2023/arxiv-sampled-1/article_11.txt → ...march-2023/arxiv-sampled-1/article_11.txt b/...march-2023/arxiv-sampled-1/article_11.txt → ...march-2023/arxiv-sampled-1/article_11.txt
diff --git a/...march-2023/arxiv-sampled-1/article_12.txt → ...march-2023/arxiv-sampled-1/article_12.txt b/...march-2023/arxiv-sampled-1/article_12.txt → ...march-2023/arxiv-sampled-1/article_12.txt
diff --git a/...march-2023/arxiv-sampled-1/article_13.txt → ...march-2023/arxiv-sampled-1/article_13.txt b/...march-2023/arxiv-sampled-1/article_13.txt → ...march-2023/arxiv-sampled-1/article_13.txt
diff --git a/...march-2023/arxiv-sampled-1/article_14.txt → ...march-2023/arxiv-sampled-1/article_14.txt b/...march-2023/arxiv-sampled-1/article_14.txt → ...march-2023/arxiv-sampled-1/article_14.txt
diff --git a/...march-2023/arxiv-sampled-1/article_15.txt → ...march-2023/arxiv-sampled-1/article_15.txt b/...march-2023/arxiv-sampled-1/article_15.txt → ...march-2023/arxiv-sampled-1/article_15.txt
diff --git a/...march-2023/arxiv-sampled-1/article_16.txt → ...march-2023/arxiv-sampled-1/article_16.txt b/...march-2023/arxiv-sampled-1/article_16.txt → ...march-2023/arxiv-sampled-1/article_16.txt
diff --git a/...march-2023/arxiv-sampled-1/article_17.txt → ...march-2023/arxiv-sampled-1/article_17.txt b/...march-2023/arxiv-sampled-1/article_17.txt → ...march-2023/arxiv-sampled-1/article_17.txt
diff --git a/...march-2023/arxiv-sampled-1/article_18.txt → ...march-2023/arxiv-sampled-1/article_18.txt b/...march-2023/arxiv-sampled-1/article_18.txt → ...march-2023/arxiv-sampled-1/article_18.txt
diff --git a/...march-2023/arxiv-sampled-1/article_19.txt → ...march-2023/arxiv-sampled-1/article_19.txt b/...march-2023/arxiv-sampled-1/article_19.txt → ...march-2023/arxiv-sampled-1/article_19.txt
diff --git a/...-march-2023/arxiv-sampled-1/article_2.txt → ...-march-2023/arxiv-sampled-1/article_2.txt b/...-march-2023/arxiv-sampled-1/article_2.txt → ...-march-2023/arxiv-sampled-1/article_2.txt
diff --git a/...march-2023/arxiv-sampled-1/article_20.txt → ...march-2023/arxiv-sampled-1/article_20.txt b/...march-2023/arxiv-sampled-1/article_20.txt → ...march-2023/arxiv-sampled-1/article_20.txt
diff --git a/...march-2023/arxiv-sampled-1/article_21.txt → ...march-2023/arxiv-sampled-1/article_21.txt b/...march-2023/arxiv-sampled-1/article_21.txt → ...march-2023/arxiv-sampled-1/article_21.txt
diff --git a/...march-2023/arxiv-sampled-1/article_22.txt → ...march-2023/arxiv-sampled-1/article_22.txt b/...march-2023/arxiv-sampled-1/article_22.txt → ...march-2023/arxiv-sampled-1/article_22.txt
diff --git a/...march-2023/arxiv-sampled-1/article_23.txt → ...march-2023/arxiv-sampled-1/article_23.txt b/...march-2023/arxiv-sampled-1/article_23.txt → ...march-2023/arxiv-sampled-1/article_23.txt
diff --git a/...march-2023/arxiv-sampled-1/article_24.txt → ...march-2023/arxiv-sampled-1/article_24.txt b/...march-2023/arxiv-sampled-1/article_24.txt → ...march-2023/arxiv-sampled-1/article_24.txt
diff --git a/...march-2023/arxiv-sampled-1/article_25.txt → ...march-2023/arxiv-sampled-1/article_25.txt b/...march-2023/arxiv-sampled-1/article_25.txt → ...march-2023/arxiv-sampled-1/article_25.txt
diff --git a/...march-2023/arxiv-sampled-1/article_26.txt → ...march-2023/arxiv-sampled-1/article_26.txt b/...march-2023/arxiv-sampled-1/article_26.txt → ...march-2023/arxiv-sampled-1/article_26.txt
diff --git a/...march-2023/arxiv-sampled-1/article_27.txt → ...march-2023/arxiv-sampled-1/article_27.txt b/...march-2023/arxiv-sampled-1/article_27.txt → ...march-2023/arxiv-sampled-1/article_27.txt
diff --git a/...march-2023/arxiv-sampled-1/article_28.txt → ...march-2023/arxiv-sampled-1/article_28.txt b/...march-2023/arxiv-sampled-1/article_28.txt → ...march-2023/arxiv-sampled-1/article_28.txt
diff --git a/...march-2023/arxiv-sampled-1/article_29.txt → ...march-2023/arxiv-sampled-1/article_29.txt b/...march-2023/arxiv-sampled-1/article_29.txt → ...march-2023/arxiv-sampled-1/article_29.txt
diff --git a/...-march-2023/arxiv-sampled-1/article_3.txt → ...-march-2023/arxiv-sampled-1/article_3.txt b/...-march-2023/arxiv-sampled-1/article_3.txt → ...-march-2023/arxiv-sampled-1/article_3.txt
diff --git a/...-march-2023/arxiv-sampled-1/article_4.txt → ...-march-2023/arxiv-sampled-1/article_4.txt b/...-march-2023/arxiv-sampled-1/article_4.txt → ...-march-2023/arxiv-sampled-1/article_4.txt
diff --git a/...-march-2023/arxiv-sampled-1/article_5.txt → ...-march-2023/arxiv-sampled-1/article_5.txt b/...-march-2023/arxiv-sampled-1/article_5.txt → ...-march-2023/arxiv-sampled-1/article_5.txt
diff --git a/...-march-2023/arxiv-sampled-1/article_6.txt → ...-march-2023/arxiv-sampled-1/article_6.txt b/...-march-2023/arxiv-sampled-1/article_6.txt → ...-march-2023/arxiv-sampled-1/article_6.txt
diff --git a/...-march-2023/arxiv-sampled-1/article_7.txt → ...-march-2023/arxiv-sampled-1/article_7.txt b/...-march-2023/arxiv-sampled-1/article_7.txt → ...-march-2023/arxiv-sampled-1/article_7.txt
diff --git a/...-march-2023/arxiv-sampled-1/article_8.txt → ...-march-2023/arxiv-sampled-1/article_8.txt b/...-march-2023/arxiv-sampled-1/article_8.txt → ...-march-2023/arxiv-sampled-1/article_8.txt
diff --git a/...-march-2023/arxiv-sampled-1/article_9.txt → ...-march-2023/arxiv-sampled-1/article_9.txt b/...-march-2023/arxiv-sampled-1/article_9.txt → ...-march-2023/arxiv-sampled-1/article_9.txt
diff --git a/...iv-march-2023/arxiv-sampled/article_0.txt → ...iv-march-2023/arxiv-sampled/article_0.txt b/...iv-march-2023/arxiv-sampled/article_0.txt → ...iv-march-2023/arxiv-sampled/article_0.txt
diff --git a/...iv-march-2023/arxiv-sampled/article_1.txt → ...iv-march-2023/arxiv-sampled/article_1.txt b/...iv-march-2023/arxiv-sampled/article_1.txt → ...iv-march-2023/arxiv-sampled/article_1.txt
diff --git a/...iv-march-2023/arxiv-sampled/article_2.txt → ...iv-march-2023/arxiv-sampled/article_2.txt b/...iv-march-2023/arxiv-sampled/article_2.txt → ...iv-march-2023/arxiv-sampled/article_2.txt
diff --git a/...iv-march-2023/arxiv-sampled/article_3.txt → ...iv-march-2023/arxiv-sampled/article_3.txt b/...iv-march-2023/arxiv-sampled/article_3.txt → ...iv-march-2023/arxiv-sampled/article_3.txt
diff --git a/...iv-march-2023/arxiv-sampled/article_4.txt → ...iv-march-2023/arxiv-sampled/article_4.txt b/...iv-march-2023/arxiv-sampled/article_4.txt → ...iv-march-2023/arxiv-sampled/article_4.txt
diff --git a/...iv-march-2023/arxiv-sampled/article_5.txt → ...iv-march-2023/arxiv-sampled/article_5.txt b/...iv-march-2023/arxiv-sampled/article_5.txt → ...iv-march-2023/arxiv-sampled/article_5.txt
diff --git a/...iv-march-2023/arxiv-sampled/article_6.txt → ...iv-march-2023/arxiv-sampled/article_6.txt b/...iv-march-2023/arxiv-sampled/article_6.txt → ...iv-march-2023/arxiv-sampled/article_6.txt
diff --git a/...iv-march-2023/arxiv-sampled/article_7.txt → ...iv-march-2023/arxiv-sampled/article_7.txt b/...iv-march-2023/arxiv-sampled/article_7.txt → ...iv-march-2023/arxiv-sampled/article_7.txt
diff --git a/...iv-march-2023/arxiv-sampled/article_8.txt → ...iv-march-2023/arxiv-sampled/article_8.txt b/...iv-march-2023/arxiv-sampled/article_8.txt → ...iv-march-2023/arxiv-sampled/article_8.txt
diff --git a/...iv-march-2023/arxiv-sampled/article_9.txt → ...iv-march-2023/arxiv-sampled/article_9.txt b/...iv-march-2023/arxiv-sampled/article_9.txt → ...iv-march-2023/arxiv-sampled/article_9.txt
diff --git a/...oads/arxiv-march-2023/arxiv_gpt4_chain.py → ...oads/arxiv-march-2023/arxiv_gpt4_chain.py b/...oads/arxiv-march-2023/arxiv_gpt4_chain.py → ...oads/arxiv-march-2023/arxiv_gpt4_chain.py
diff --git a/...orkloads/arxiv-march-2023/arxiv_loader.py → ...orkloads/arxiv-march-2023/arxiv_loader.py b/...orkloads/arxiv-march-2023/arxiv_loader.py → ...orkloads/arxiv-march-2023/arxiv_loader.py
diff --git a/...kloads/bbc_news_march_2023/.gitattributes → ...kloads/bbc_news_march_2023/.gitattributes b/...kloads/bbc_news_march_2023/.gitattributes → ...kloads/bbc_news_march_2023/.gitattributes
diff --git a/...k/workloads/bbc_news_march_2023/README.md → ...4/workloads/bbc_news_march_2023/README.md b/...k/workloads/bbc_news_march_2023/README.md → ...4/workloads/bbc_news_march_2023/README.md
diff --git a/...loads/bingchat/sydney-prompt-preamble.txt → ...loads/bingchat/sydney-prompt-preamble.txt b/...loads/bingchat/sydney-prompt-preamble.txt → ...loads/bingchat/sydney-prompt-preamble.txt
diff --git a/benchmark/workloads/gpts/top4.json → artifact_osdi24/workloads/gpts/top4.json b/benchmark/workloads/gpts/top4.json → artifact_osdi24/workloads/gpts/top4.json
diff --git a/benchmark/workloads/metagpt/README.md → artifact_osdi24/workloads/metagpt/README.md b/benchmark/workloads/metagpt/README.md → artifact_osdi24/workloads/metagpt/README.md
diff --git a/benchmark/workloads/metagpt/architect.txt → ...ct_osdi24/workloads/metagpt/architect.txt b/benchmark/workloads/metagpt/architect.txt → ...ct_osdi24/workloads/metagpt/architect.txt
diff --git a/.../workloads/metagpt/build_pytorch.py.ipynb → .../workloads/metagpt/build_pytorch.py.ipynb b/.../workloads/metagpt/build_pytorch.py.ipynb → .../workloads/metagpt/build_pytorch.py.ipynb
diff --git a/...orkloads/metagpt/build_pytorch_1.py.ipynb → ...orkloads/metagpt/build_pytorch_1.py.ipynb b/...orkloads/metagpt/build_pytorch_1.py.ipynb → ...orkloads/metagpt/build_pytorch_1.py.ipynb
diff --git a/benchmark/workloads/metagpt/image.png → artifact_osdi24/workloads/metagpt/image.png b/benchmark/workloads/metagpt/image.png → artifact_osdi24/workloads/metagpt/image.png
diff --git a/...hmark/workloads/metagpt/log_3_round.jsonl → ...sdi24/workloads/metagpt/log_3_round.jsonl b/...hmark/workloads/metagpt/log_3_round.jsonl → ...sdi24/workloads/metagpt/log_3_round.jsonl
diff --git a/...rk/workloads/metagpt/log_new_rounds.jsonl → ...24/workloads/metagpt/log_new_rounds.jsonl b/...rk/workloads/metagpt/log_new_rounds.jsonl → ...24/workloads/metagpt/log_new_rounds.jsonl
diff --git a/...workloads/metagpt/multiagent-pytorch.json → ...workloads/metagpt/multiagent-pytorch.json b/...workloads/metagpt/multiagent-pytorch.json → ...workloads/metagpt/multiagent-pytorch.json
diff --git a/...ark/workloads/metagpt/parrot_functions.py → ...i24/workloads/metagpt/parrot_functions.py b/...ark/workloads/metagpt/parrot_functions.py → ...i24/workloads/metagpt/parrot_functions.py
diff --git a/benchmark/workloads/metagpt/programmer.txt → ...t_osdi24/workloads/metagpt/programmer.txt b/benchmark/workloads/metagpt/programmer.txt → ...t_osdi24/workloads/metagpt/programmer.txt
diff --git a/benchmark/workloads/metagpt/reviewer.txt → ...act_osdi24/workloads/metagpt/reviewer.txt b/benchmark/workloads/metagpt/reviewer.txt → ...act_osdi24/workloads/metagpt/reviewer.txt
diff --git a/benchmark/workloads/metagpt/reviser.txt → ...fact_osdi24/workloads/metagpt/reviser.txt b/benchmark/workloads/metagpt/reviser.txt → ...fact_osdi24/workloads/metagpt/reviser.txt
diff --git a/benchmark/workloads/sharegpt/README.md → artifact_osdi24/workloads/sharegpt/README.md b/benchmark/workloads/sharegpt/README.md → artifact_osdi24/workloads/sharegpt/README.md
diff --git a/...oads/test_examples/chain_summarization.py → ...oads/test_examples/chain_summarization.py b/...oads/test_examples/chain_summarization.py → ...oads/test_examples/chain_summarization.py
diff --git a/...test_examples/map_reduce_summarization.py → ...test_examples/map_reduce_summarization.py b/...test_examples/map_reduce_summarization.py → ...test_examples/map_reduce_summarization.py
diff --git a/...rkloads/test_examples/normal_functions.py → ...rkloads/test_examples/normal_functions.py b/...rkloads/test_examples/normal_functions.py → ...rkloads/test_examples/normal_functions.py
diff --git a/benchmark/microbench/bench_parrot_builtin_iter.py b/benchmark/microbench/bench_parrot_builtin_iter.py
@@ -20,11 +20,13 @@ def _init(model):
 
     prompt_len = 3000
 
-    fill1 = Fill(pid=0, tid=0, context_id=0, parent_context_id=-1, token_ids=[100])
+    fill1 = Fill(
+        session_id=0, task_id=0, context_id=0, parent_context_id=-1, token_ids=[100]
+    )
 
     fill2 = Fill(
-        pid=0,
-        tid=0,
+        session_id=0,
+        task_id=0,
         context_id=0,
         parent_context_id=-1,
         token_ids=[100] * prompt_len,
@@ -36,8 +38,8 @@ def _init(model):
     )
 
     gen = Generate(
-        pid=0,
-        tid=0,
+        session_id=0,
+        task_id=0,
         context_id=0,
         parent_context_id=-1,
         sampling_config=sampling_config,

diff --git a/benchmark/microbench/bench_parrot_multi_gpus.py b/benchmark/microbench/bench_parrot_multi_gpus.py
@@ -2,7 +2,7 @@
 # Please manually do this by:
 #   bash sample_configs/launch/launch_4_vicuna_7b.sh
 
-import parrot as P
+from parrot import P
 import asyncio
 import time
 

diff --git a/benchmark/microbench/bench_shared_prompt_e2e.py b/benchmark/microbench/bench_shared_prompt_e2e.py
@@ -131,8 +131,8 @@ def __init__(
     def push(self, prompt_token_ids: list[int], parent_context_id: int, gen_limit: int):
         context_idx = self._jobs.index(None)
         self._jobs[context_idx] = Fill(
-            pid=0,
-            tid=0,
+            session_id=0,
+            task_id=0,
             context_id=context_idx,
             parent_context_id=parent_context_id,
             token_ids=prompt_token_ids,
@@ -150,8 +150,8 @@ def run(self):
             else:
                 if isinstance(job, Fill):
                     self._jobs[context_idx] = Generate(
-                        pid=0,
-                        tid=0,
+                        session_id=0,
+                        task_id=0,
                         context_id=job.context_id,
                         parent_context_id=job.parent_context_id,
                         sampling_config=self._sampling_config,
@@ -213,8 +213,8 @@ def profile_bing_chat(
 
     if shared:
         shared_fill = Fill(
-            pid=0,
-            tid=0,
+            session_id=0,
+            task_id=0,
             context_id=batch_size,
             parent_context_id=-1,
             token_ids=prompt_token_ids[0][:shared_ids],

diff --git a/benchmark/microbench/bench_single_function.py b/benchmark/microbench/bench_single_function.py