Merge branch 'main' into slogdet

Author: mengfei25

.github/scripts/apply_torch_pr.py

@@ -1,9 +1,9 @@
 
 import requests
 import argparse
-import urllib
 import subprocess
 import sys
+import os
 
 
 parser = argparse.ArgumentParser()
@@ -54,12 +54,17 @@ def check_merged(pr_info):
         merged = False
     return merged
 
-def appyly_pr(pr_info, re_apply_msg):
-    # get pr diff
-    pr_file = pr_info["diff_url"].split("/")[-1]
-    urllib.request.urlretrieve(pr_info["diff_url"], pr_file)
+def appyly_pr(pull_number, re_apply_msg):
+    # get the diff
+    os.system(f"\
+        git fetch origin pull/{pull_number}/head:{pull_number} && \
+        git checkout -f {pull_number} && \
+        git merge ci-tmp-$(hostname) --no-edit --no-ff > /dev/null && \
+        git diff ci-tmp-$(hostname) {pull_number} > {pull_number}.diff \
+    ")
     # apply diff
-    apply_cmd = "git apply --3way " + pr_file
+    os.system("git checkout ci-test-$(hostname)")
+    apply_cmd = f"git reset --hard && git apply --3way {pull_number}.diff"
     apply_info = subprocess.Popen(apply_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
     apply_message = apply_info.communicate()[0].decode("utf-8")
     apply_status = apply_info.returncode
@@ -72,15 +77,22 @@ def appyly_pr(pr_info, re_apply_msg):
         sys.exit(1)
 
 
-# headers = {'Authorization': 'Bearer ' + args.token} if args.token != None else args.token
+github_token = os.environ.get("GH_TOKEN")
+headers = {
+    "Authorization": f"Bearer {github_token}",
+    "Accept": "application/vnd.github+json",
+    "X-GitHub-Api-Version": "2022-11-28",
+}
 pr_list = args.pr_list + args.extra_pr_list
 pr_list = set(pr_list)
 pr_list = sorted(pr_list)
+# checkout a base branch
+os.system("git checkout -b ci-tmp-$(hostname) && git checkout -b ci-test-$(hostname) && rm -f *.diff")
+os.system("git config --global user.email intel.com && git config --global user.name intel")
 for pr_link in pr_list:
     repo_info = pr_link.split("/")
     pr_info = requests.get('https://api.' + repo_info[-5] + '/repos/' + repo_info[-4] + '/' + \
-                        repo_info[-3] + '/pulls/' + repo_info[-1], timeout=60).json()
-
+                        repo_info[-3] + '/pulls/' + repo_info[-1], headers=headers, timeout=60).json()
     if pr_info["state"].lower() == "open":
         # for reverted PR
         reverted_id = next((item["id"] for item in pr_info["labels"] if item["name"] == "Reverted"), -1)
@@ -93,7 +105,7 @@ def appyly_pr(pr_info, re_apply_msg):
                 continue
             else:
                 re_apply_msg = "is re-opened and reverted,"
-        appyly_pr(pr_info, re_apply_msg)
+        appyly_pr(repo_info[-1], re_apply_msg)
     elif pr_info["state"].lower() == "closed":
         merged_id = next((item["id"] for item in pr_info["labels"] if item["name"] == "Merged"), -1)
         re_apply_msg = "is closed but not merged"
@@ -102,7 +114,8 @@ def appyly_pr(pr_info, re_apply_msg):
             if merged:
                 print("{} is closed and merged, no need to apply".format(pr_info["diff_url"]))
                 continue
-        appyly_pr(pr_info, re_apply_msg)
+        appyly_pr(repo_info[-1], re_apply_msg)
     else:
         print("{} is {}, no need to apply".format(pr_info["diff_url"], pr_info["state"]))
         sys.exit(1)
+os.system("git checkout ci-test-$(hostname) && git reset --hard && git apply --3way *.diff && git status")

.github/workflows/_linux_op_benchmark.yml

@@ -41,6 +41,7 @@ jobs:
     runs-on: ${{ inputs.runner }} 
     timeout-minutes: 900
     env:
+      GH_TOKEN: ${{ github.token }}
       NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
       DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
     steps:

.github/workflows/_linux_ut.yml

@@ -47,6 +47,7 @@ jobs:
     if: ${{ inputs.ut != 'xpu_distributed' }}
     timeout-minutes: 900
     env:
+      GH_TOKEN: ${{ github.token }}
       NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
       DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
     steps:
@@ -279,6 +280,7 @@ jobs:
     if: contains(inputs.ut, 'xpu_distributed')
     timeout-minutes: 900
     env:
+      GH_TOKEN: ${{ github.token }}
       NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
       DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
     steps:

.github/workflows/nightly_ondemand.yml

@@ -101,6 +101,7 @@ jobs:
     needs: Linux-Nightly-Ondemand-Build
     timeout-minutes: 3600
     env:
+      GH_TOKEN: ${{ github.token }}
       pytorch: ${{ needs.Linux-Nightly-Ondemand-Build.outputs.torch_commit_id }}
       keep_torch_xpu_ops: ${{ github.event_name == 'schedule' && 'false' || inputs.keep_torch_xpu_ops }}
       python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }}

.github/workflows/nightly_ondemand_rolling.yml

@@ -116,6 +116,7 @@ jobs:
     needs: Linux-Nightly-Ondemand-Build-Rolling
     timeout-minutes: 3600
     env:
+      GH_TOKEN: ${{ github.token }}
       pytorch: ${{ needs.Linux-Nightly-Ondemand-Build-Rolling.outputs.torch_commit_id }}
       keep_torch_xpu_ops: ${{ github.event_name == 'schedule' && 'false' || inputs.keep_torch_xpu_ops }}
       python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }}

.github/workflows/nightly_ondemand_whl.yml

@@ -71,6 +71,7 @@ jobs:
     if: ${{ github.repository_owner == 'intel' }}
     timeout-minutes: 3600
     env:
+      GH_TOKEN: ${{ github.token }}
       pytorch: ${{ github.event_name == 'schedule' && 'nightly' || inputs.pytorch }}
       python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }}
       run_type: ${{ (github.event_name == 'schedule' && (github.event.schedule == '0 17 * * 5' && 'weekly' || 'nightly')) || 'on-demand' }}

.github/workflows/pull.yml

@@ -71,6 +71,8 @@ jobs:
     name: preci-linux / e2e_test
     needs: preci-linux-build
     runs-on: pvc_e2e
+    env:
+      GH_TOKEN: ${{ github.token }}
     # Don't run on forked repos and draft PRs
     if: ${{ (github.repository_owner == 'intel') && (github.event.pull_request.draft == false) }}
     timeout-minutes: 900
@@ -93,8 +95,8 @@ jobs:
           cd pytorch && git checkout ${{ needs.preci-linux-build.outputs.torch_commit_id }}
           # apply PRs for stock pytorch
           pip install requests
-          # https://github.com/mengfei25/pytorch/pull/18 internal use only for subset model list
-          python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py -e https://github.com/mengfei25/pytorch/pull/18
+          # https://github.com/pytorch/pytorch/pull/152940 internal use only for subset model list
+          python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py -e https://github.com/pytorch/pytorch/pull/152940
           git status && git show -s
           rm -rf third_party/torch-xpu-ops && cp -r ../torch-xpu-ops third_party/
           # Workaround for torch-xpu-ops ci test