diff --git a/README.md b/README.md index a4494b29..21a61e10 100644 --- a/README.md +++ b/README.md @@ -253,14 +253,21 @@ Once you finish the installation, follow these steps: By default, **@coqbot** considers that both GitHub and GitLab repositories share the same URL except for the "lab" replacing the "hub" part. If that is not the case, assuming you created a GitLab repository whose - URL is , add a file `coqbot.toml` at + URL is , add a file `coqbot.toml` at the root of your GitHub repository and in its default branch (most often named `master`), containing: ``` [mapping] gitlab = "owner/repo" + gitlab_domain = "mygitlab.example.com" ``` - If you use other instance of **@coqbot**, this repository-specific + If omitted, the `gitlab_domain` value defaults to `"gitlab.com"`. + Note that the value of `gitlab_domain` must be a supported GitLab + instance, i.e., it needs to be defined in the bot's own configuration + file (check [coqbot-config.toml](coqbot-config.toml) for the coqbot + instance configuration). + + If you use another instance of **@coqbot**, this repository-specific configuration file becomes `BOT_NAME.toml` where `BOT_NAME` is the name of the bot. @@ -345,11 +352,14 @@ to [Heroku](https://www.heroku.com/). Simply follow the official The bot will need to read a few environment variables so make sure these are configured in your Heroku app: -- `GITLAB_ACCESS_TOKEN` -- `GITHUB_ACCESS_TOKEN` -- `GITHUB_WEBHOOK_SECRET` +- `GITHUB_ACCESS_TOKEN` (can also be defined in the configuration file as `github.api_token`) +- `GITLAB_ACCESS_TOKEN` (can also be defined for each GitLab instance through the configuration file as `api_token` or through an environment variable whose name is defined in the configuration file as `api_token_env_var`) +- `GITHUB_WEBHOOK_SECRET` (can also be defined in the configuration file as `github.webhook_secret`) +- `GITLAB_WEBHOOK_SECRET` (can also be defined in the configuration file as `gitlab.webhook_secret`, will default to `GITHUB_WEBHOOK_SECRET` if not defined) +- `DAILY_SCHEDULE_SECRET` (can also be defined in the configuration file as `github.daily_schedule_secret`, will default to `GITHUB_WEBHOOK_SECRET` if not defined) +- `GITHUB_APP_ID` (can also be defined in the configuration file as `github.app_id`) - `GITHUB_PRIVATE_KEY` (a private key of your GitHub app) -- `GITHUB_APP_ID` (your GitHub App ID) +- `PORT` (can also be defined in the configuration file as `server.port`) Then, you must configure the bot with a configuration file. Here is an example to adapt to your needs [`example-config.toml`](example-config.toml)). diff --git a/bot-components/Bot_info.ml b/bot-components/Bot_info.ml index 81faa893..6eab876f 100644 --- a/bot-components/Bot_info.ml +++ b/bot-components/Bot_info.ml @@ -1,8 +1,10 @@ +open Base + type t = - { gitlab_token: string + { gitlab_instances: (string, string * string) Hashtbl.t ; github_pat: string ; github_install_token: string option - ; name: string + ; github_name: string ; email: string ; domain: string ; app_id: int } @@ -13,3 +15,15 @@ let github_token bot_info = t | None -> bot_info.github_pat + +let gitlab_name_and_token bot_info gitlab_domain = + match Hashtbl.find bot_info.gitlab_instances gitlab_domain with + | Some t -> + Ok t + | None -> + Error + ( "I don't know about GitLab domain " ^ gitlab_domain + ^ " (not in my configuration file)" ) + +let gitlab_token bot_info gitlab_domain = + gitlab_name_and_token bot_info gitlab_domain |> Result.map ~f:snd diff --git a/bot-components/Bot_info.mli b/bot-components/Bot_info.mli index a146e0f9..984ee1e5 100644 --- a/bot-components/Bot_info.mli +++ b/bot-components/Bot_info.mli @@ -1,10 +1,14 @@ type t = - { gitlab_token: string + { gitlab_instances: (string, string * string) Base.Hashtbl.t ; github_pat: string ; github_install_token: string option - ; name: string + ; github_name: string ; email: string ; domain: string ; app_id: int } val github_token : t -> string + +val gitlab_token : t -> string -> (string, string) Result.t + +val gitlab_name_and_token : t -> string -> (string * string, string) Result.t diff --git a/bot-components/GitHub_app.ml b/bot-components/GitHub_app.ml index 9a679fc5..ed5e4031 100644 --- a/bot-components/GitHub_app.ml +++ b/bot-components/GitHub_app.ml @@ -39,13 +39,13 @@ let make_jwt ~key ~app_id = let get ~bot_info ~token ~url = Stdio.print_endline ("Making get request to " ^ url) ; - let headers = headers ~bot_info (github_headers token) in + let headers = headers (github_headers token) bot_info.Bot_info.github_name in Client.get ~headers (Uri.of_string url) >>= fun (_response, body) -> Cohttp_lwt.Body.to_string body let post ~bot_info ~body ~token ~url = Stdio.print_endline ("Making post request to " ^ url) ; - let headers = headers ~bot_info (github_headers token) in + let headers = headers (github_headers token) bot_info.Bot_info.github_name in let body = (match body with None -> "{}" | Some json -> Yojson.to_string json) |> Cohttp_lwt.Body.of_string diff --git a/bot-components/GitHub_mutations.ml b/bot-components/GitHub_mutations.ml index 74ab2c12..4648c306 100644 --- a/bot-components/GitHub_mutations.ml +++ b/bot-components/GitHub_mutations.ml @@ -4,7 +4,7 @@ open Cohttp_lwt_unix open Lwt open Utils -let send_graphql_query = GraphQL_query.send_graphql_query ~api:`GitHub +let send_graphql_query = GraphQL_query.send_graphql_query ~api:GitHub let mv_card_to_column ~bot_info ({card_id; column_id} : mv_card_to_column_input) = @@ -213,7 +213,7 @@ let remove_labels ~bot_info ~labels ~issue = (* TODO: use GraphQL API *) let update_milestone ~bot_info new_milestone (issue : issue) = - let headers = headers (github_header bot_info) ~bot_info in + let headers = headers (github_header bot_info) bot_info.github_name in let uri = f "https://api.github.com/repos/%s/%s/issues/%d" issue.owner issue.repo issue.number @@ -245,7 +245,7 @@ let send_status_check ~bot_info ~repo_full_name ~commit ~state ~url ~context "https://api.github.com/repos/" ^ repo_full_name ^ "/statuses/" ^ commit |> Uri.of_string in - send_request ~body ~uri (github_header bot_info) ~bot_info + send_request ~body ~uri (github_header bot_info) bot_info.github_name let add_pr_to_column ~bot_info ~pr_id ~column_id = let body = @@ -265,4 +265,4 @@ let add_pr_to_column ~bot_info ~pr_id ~column_id = in send_request ~body ~uri (project_api_preview_header @ github_header bot_info) - ~bot_info + bot_info.github_name diff --git a/bot-components/GitHub_queries.ml b/bot-components/GitHub_queries.ml index 30aec79e..1e5df327 100644 --- a/bot-components/GitHub_queries.ml +++ b/bot-components/GitHub_queries.ml @@ -4,7 +4,7 @@ open GitHub_types open Lwt open Utils -let send_graphql_query = GraphQL_query.send_graphql_query ~api:`GitHub +let send_graphql_query = GraphQL_query.send_graphql_query ~api:GitHub let extract_backport_info ~(bot_info : Bot_info.t) description : full_backport_info option = @@ -12,7 +12,8 @@ let extract_backport_info ~(bot_info : Bot_info.t) description : "https://github.com/[^/]*/[^/]*/projects/[0-9]+#column-\\([0-9]+\\)" in let regexp = - bot_info.name ^ ": backport to \\([^ ]*\\) (request inclusion column: " + bot_info.github_name + ^ ": backport to \\([^ ]*\\) (request inclusion column: " ^ project_column_regexp ^ "; backported column: " ^ project_column_regexp ^ "; move rejected PRs to: " ^ "https://github.com/[^/]*/[^/]*/milestone/\\([0-9]+\\)" ^ ")" @@ -29,7 +30,7 @@ let extract_backport_info ~(bot_info : Bot_info.t) description : [{backport_to; request_inclusion_column; backported_column}] ; rejected_milestone } else - let begin_regexp = bot_info.name ^ ": \\(.*\\)$" in + let begin_regexp = bot_info.github_name ^ ": \\(.*\\)$" in let backport_info_unit = "backport to \\([^ ]*\\) (request inclusion column: " ^ project_column_regexp ^ "; backported column: " ^ project_column_regexp diff --git a/bot-components/GitLab_mutations.ml b/bot-components/GitLab_mutations.ml index b25bc680..5263ec97 100644 --- a/bot-components/GitLab_mutations.ml +++ b/bot-components/GitLab_mutations.ml @@ -2,38 +2,48 @@ open Base open Bot_info open Utils -let generic_retry ~bot_info ~url_part = +let generic_retry ~bot_info ~gitlab_domain ~url_part = let uri = - "https://gitlab.com/api/v4/" ^ url_part ^ "/retry" |> Uri.of_string + f "https://%s/api/v4/%s/retry" gitlab_domain url_part |> Uri.of_string in - let gitlab_header = [("Private-Token", bot_info.gitlab_token)] in - Utils.send_request ~body:Cohttp_lwt.Body.empty ~uri gitlab_header ~bot_info + match gitlab_name_and_token bot_info gitlab_domain with + | Error err -> + Lwt_io.printlf "Error when retrying job %s: %s." url_part err + | Ok (name, token) -> + let gitlab_header = [("Private-Token", token)] in + Utils.send_request ~body:Cohttp_lwt.Body.empty ~uri gitlab_header name -let retry_job ~bot_info ~project_id ~build_id = - generic_retry ~bot_info +let retry_job ~bot_info ~gitlab_domain ~project_id ~build_id = + generic_retry ~bot_info ~gitlab_domain ~url_part: ( "projects/" ^ Int.to_string project_id ^ "/jobs/" ^ Int.to_string build_id ) -let play_job ~bot_info ~project_id ~build_id ?(key_value_pairs = []) () = +let play_job ~bot_info ~gitlab_domain ~project_id ~build_id + ?(key_value_pairs = []) () = let uri = Uri.of_string - @@ Printf.sprintf "https://gitlab.com/api/v4/projects/%d/jobs/%d/play" + @@ Printf.sprintf "https://%s/api/v4/projects/%d/jobs/%d/play" gitlab_domain project_id build_id in - let gitlab_header = - [ ("Private-Token", bot_info.gitlab_token) - ; ("Content-Type", "application/json") ] - in - let body = - match key_value_pairs with - | [] -> - Cohttp_lwt.Body.empty - | _ -> - key_value_pairs - |> List.map ~f:(fun (k, v) -> f {|{ "key": "%s", "value": "%s" }|} k v) - |> String.concat ~sep:"," - |> f {|{ "job_variables_attributes": [%s] }|} - |> Cohttp_lwt.Body.of_string - in - Utils.send_request ~body ~uri gitlab_header ~bot_info + match gitlab_name_and_token bot_info gitlab_domain with + | Error err -> + Lwt_io.printlf "Error when playing job %d of project %d: %s." build_id + project_id err + | Ok (name, token) -> + let gitlab_header = + [("Private-Token", token); ("Content-Type", "application/json")] + in + let body = + match key_value_pairs with + | [] -> + Cohttp_lwt.Body.empty + | _ -> + key_value_pairs + |> List.map ~f:(fun (k, v) -> + f {|{ "key": "%s", "value": "%s" }|} k v ) + |> String.concat ~sep:"," + |> f {|{ "job_variables_attributes": [%s] }|} + |> Cohttp_lwt.Body.of_string + in + Utils.send_request ~body ~uri gitlab_header name diff --git a/bot-components/GitLab_mutations.mli b/bot-components/GitLab_mutations.mli index a90d6225..3e7812c5 100644 --- a/bot-components/GitLab_mutations.mli +++ b/bot-components/GitLab_mutations.mli @@ -1,10 +1,16 @@ val retry_job : - bot_info:Bot_info.t -> project_id:int -> build_id:int -> unit Lwt.t + bot_info:Bot_info.t + -> gitlab_domain:string + -> project_id:int + -> build_id:int + -> unit Lwt.t -val generic_retry : bot_info:Bot_info.t -> url_part:string -> unit Lwt.t +val generic_retry : + bot_info:Bot_info.t -> gitlab_domain:string -> url_part:string -> unit Lwt.t val play_job : bot_info:Bot_info.t + -> gitlab_domain:string -> project_id:int -> build_id:int -> ?key_value_pairs:(string * string) list diff --git a/bot-components/GitLab_queries.ml b/bot-components/GitLab_queries.ml index 42ea8f95..e89e75a7 100644 --- a/bot-components/GitLab_queries.ml +++ b/bot-components/GitLab_queries.ml @@ -1,30 +1,37 @@ open Base open Cohttp_lwt_unix -open Lwt open Bot_info open Utils -let send_graphql_query = GraphQL_query.send_graphql_query ~api:`GitLab +let send_graphql_query ~gitlab_domain = + GraphQL_query.send_graphql_query ~api:(GitLab gitlab_domain) -let get_build_trace ~bot_info ~project_id ~build_id = +let get_build_trace ~bot_info ~gitlab_domain ~project_id ~build_id = let uri = - "https://gitlab.com/api/v4/projects/" ^ Int.to_string project_id ^ "/jobs/" - ^ Int.to_string build_id ^ "/trace" + f "https://%s/api/v4/projects/%d/jobs/%d/trace" gitlab_domain project_id + build_id |> Uri.of_string in - let gitlab_header = [("Private-Token", bot_info.gitlab_token)] in - let headers = Utils.headers gitlab_header ~bot_info in + let open Lwt_result.Infix in + gitlab_name_and_token bot_info gitlab_domain + |> Lwt.return + >>= fun (name, token) -> + let gitlab_header = [("Private-Token", token)] in + let headers = Utils.headers gitlab_header name in + let open Lwt.Infix in Client.get ~headers uri - >>= fun (_response, body) -> Cohttp_lwt.Body.to_string body + >>= fun (_response, body) -> + Cohttp_lwt.Body.to_string body |> Lwt.map Result.return -let get_retry_nb ~bot_info ~full_name ~build_id ~build_name = +let get_retry_nb ~bot_info ~gitlab_domain ~full_name ~build_id ~build_name = let open GitLab_GraphQL.GetRetriedJobs in + let open Lwt.Infix in makeVariables ~fullPath:full_name ~jobId: (build_id |> f {|"gid://gitlab/Ci::Build/%d"|} |> Yojson.Basic.from_string) () |> serializeVariables |> variablesToJson - |> send_graphql_query ~bot_info ~query + |> send_graphql_query ~bot_info ~gitlab_domain ~query ~parse:(Fn.compose parse unsafe_fromJson) >|= function | Ok {project= Some {job= Some {pipeline= Some {jobs= Some {count= 0}}}}} -> diff --git a/bot-components/GitLab_queries.mli b/bot-components/GitLab_queries.mli index 48c68fd3..9873e21e 100644 --- a/bot-components/GitLab_queries.mli +++ b/bot-components/GitLab_queries.mli @@ -1,8 +1,13 @@ val get_build_trace : - bot_info:Bot_info.t -> project_id:int -> build_id:int -> string Lwt.t + bot_info:Bot_info.t + -> gitlab_domain:string + -> project_id:int + -> build_id:int + -> (string, string) Lwt_result.t val get_retry_nb : bot_info:Bot_info.t + -> gitlab_domain:string -> full_name:string -> build_id:int -> build_name:string diff --git a/bot-components/GitLab_subscriptions.ml b/bot-components/GitLab_subscriptions.ml index 6431faab..68631656 100644 --- a/bot-components/GitLab_subscriptions.ml +++ b/bot-components/GitLab_subscriptions.ml @@ -32,7 +32,9 @@ let job_info_of_json json = let project_id = json |> member "project_id" |> to_int in let base_commit, head_commit = json |> extract_commit in let branch = json |> member "ref" |> to_string in - let repo_url = json |> member "repository" |> member "url" |> to_string in + let http_repo_url = + json |> member "repository" |> member "homepage" |> to_string + in let stage = json |> member "build_stage" |> to_string in let failure_reason = json |> member "build_failure_reason" |> to_string |> Option.some @@ -44,7 +46,8 @@ let job_info_of_json json = ; stage ; failure_reason ; allow_fail - ; common_info= {base_commit; head_commit; branch; repo_url; project_id} } + ; common_info= {base_commit; head_commit; branch; http_repo_url; project_id} + } (* For use to decode builds inside a pipeline webhook *) let build_info_of_json json = @@ -70,8 +73,7 @@ let pipeline_info_of_json json = let base_commit, head_commit = json |> extract_commit in let branch = pipeline_json |> member "ref" |> to_string in let project = json |> member "project" in - let repo_url = project |> member "web_url" |> to_string in - let project_path = project |> member "path_with_namespace" |> to_string in + let http_repo_url = project |> member "web_url" |> to_string in let project_id = project |> member "id" |> to_int in let variables = pipeline_json |> member "variables" |> to_list @@ -88,8 +90,7 @@ let pipeline_info_of_json json = in { state ; pipeline_id - ; project_path - ; common_info= {head_commit; base_commit; branch; repo_url; project_id} + ; common_info= {head_commit; base_commit; branch; http_repo_url; project_id} ; variables ; stages ; builds } diff --git a/bot-components/GitLab_types.mli b/bot-components/GitLab_types.mli index a8099ebb..9c1e55dd 100644 --- a/bot-components/GitLab_types.mli +++ b/bot-components/GitLab_types.mli @@ -2,7 +2,7 @@ type ci_common_info = { head_commit: string ; base_commit: string option ; branch: string - ; repo_url: string + ; http_repo_url: string ; project_id: int } type 'a job_info = @@ -17,7 +17,6 @@ type 'a job_info = type pipeline_info = { state: string ; pipeline_id: int - ; project_path: string ; common_info: ci_common_info ; variables: (string * string) list ; stages: string list diff --git a/bot-components/GraphQL_query.ml b/bot-components/GraphQL_query.ml index 9111a5fd..c069bcc1 100644 --- a/bot-components/GraphQL_query.ml +++ b/bot-components/GraphQL_query.ml @@ -1,29 +1,31 @@ open Base open Bot_info -open Lwt.Infix open Utils +type api = GitHub | GitLab of string + let send_graphql_query ~bot_info ?(extra_headers = []) ~api ~query ~parse variables = let uri = ( match api with - | `GitLab -> - "https://gitlab.com/api/graphql" - | `GitHub -> + | GitLab gitlab_domain -> + f "https://%s/api/graphql" gitlab_domain + | GitHub -> "https://api.github.com/graphql" ) |> Uri.of_string in + let open Lwt_result.Infix in + ( match api with + | GitLab gitlab_domain -> + gitlab_name_and_token bot_info gitlab_domain + | GitHub -> + Ok (bot_info.github_name, github_token bot_info) ) + |> Lwt.return + >>= fun (name, token) -> let headers = Cohttp.Header.of_list - ( [ ( "Authorization" - , "Bearer " - ^ - match api with - | `GitLab -> - bot_info.gitlab_token - | `GitHub -> - github_token bot_info ) - ; ("User-Agent", bot_info.name) + ( [ ("Authorization", "Bearer " ^ token) + ; ("User-Agent", name) ; ("Content-Type", "application/json") ] @ extra_headers ) in @@ -31,6 +33,7 @@ let send_graphql_query ~bot_info ?(extra_headers = []) ~api ~query ~parse `Assoc [("query", `String query); ("variables", variables)] in let request = Yojson.Basic.to_string request_json in + let open Lwt.Infix in Cohttp_lwt_unix.Client.post ~headers ~body:(`String request) uri >>= fun (rsp, body) -> Cohttp_lwt.Body.to_string body @@ -49,7 +52,8 @@ let send_graphql_query ~bot_info ?(extra_headers = []) ~api ~query ~parse | errors -> let errors = to_list errors - |> List.map ~f:(fun error -> error |> member "message" |> to_string) + |> List.map ~f:(fun error -> + error |> member "message" |> to_string ) in Error ( "Server responded to GraphQL request with errors: " diff --git a/bot-components/GraphQL_query.mli b/bot-components/GraphQL_query.mli index 4b098c16..87029670 100644 --- a/bot-components/GraphQL_query.mli +++ b/bot-components/GraphQL_query.mli @@ -1,7 +1,9 @@ +type api = GitHub | GitLab of string + val send_graphql_query : bot_info:Bot_info.t -> ?extra_headers:(string * string) list - -> api:[`GitHub | `GitLab] + -> api:api -> query:string -> parse:(Yojson.Basic.t -> 'a) -> Yojson.Basic.t diff --git a/bot-components/Utils.ml b/bot-components/Utils.ml index 702da226..8587df60 100644 --- a/bot-components/Utils.ml +++ b/bot-components/Utils.ml @@ -12,10 +12,10 @@ let string_match ~regexp string = true with Stdlib.Not_found -> false -let headers ~bot_info header_list = +let headers header_list user_agent = Header.init () |> (fun headers -> Header.add_list headers header_list) - |> fun headers -> Header.add headers "User-Agent" bot_info.name + |> fun headers -> Header.add headers "User-Agent" user_agent let print_response (resp, body) = let code = resp |> Response.status |> Code.code_of_status in @@ -28,8 +28,8 @@ let print_response (resp, body) = body |> Cohttp_lwt.Body.to_string >>= Lwt_io.printf "Body:\n%s\n" else Lwt.return_unit -let send_request ~bot_info ~body ~uri header_list = - let headers = headers header_list ~bot_info in +let send_request ~body ~uri header_list user_agent = + let headers = headers header_list user_agent in Client.post ~body ~headers uri >>= print_response let handle_json action body = @@ -56,7 +56,9 @@ let github_header bot_info = let generic_get ~bot_info relative_uri ?(header_list = []) json_handler = let uri = "https://api.github.com/" ^ relative_uri |> Uri.of_string in - let headers = headers (header_list @ github_header bot_info) ~bot_info in + let headers = + headers (header_list @ github_header bot_info) bot_info.github_name + in Client.get ~headers uri >>= (fun (_response, body) -> Cohttp_lwt.Body.to_string body) >|= handle_json json_handler diff --git a/bot-components/Utils.mli b/bot-components/Utils.mli index 6a119625..142a99e2 100644 --- a/bot-components/Utils.mli +++ b/bot-components/Utils.mli @@ -2,15 +2,15 @@ val f : ('a, unit, string) format -> 'a val string_match : regexp:string -> string -> bool -val headers : bot_info:Bot_info.t -> (string * string) list -> Cohttp.Header.t +val headers : (string * string) list -> string -> Cohttp.Header.t val print_response : Cohttp.Response.t * Cohttp_lwt.Body.t -> unit Lwt.t val send_request : - bot_info:Bot_info.t - -> body:Cohttp_lwt.Body.t + body:Cohttp_lwt.Body.t -> uri:Uri.t -> (string * string) list + -> string -> unit Lwt.t val project_api_preview_header : (string * string) list diff --git a/coqbot-config.toml b/coqbot-config.toml index 6c02effe..6198b5b6 100644 --- a/coqbot-config.toml +++ b/coqbot-config.toml @@ -8,6 +8,16 @@ domain="coqbot.herokuapp.com" [github] app_id="31373" +[gitlab] + [gitlab.com] + domain="gitlab.com" + api_token_env_var="GITLAB_ACCESS_TOKEN" + + [gitlab.inria] + domain="gitlab.inria.fr" + api_token_env_var="INRIA_GITLAB_ACCESS_TOKEN" + bot_name="x-CBot" + [mappings] [mappings.coq] github="coq/coq" @@ -16,3 +26,13 @@ app_id="31373" [mappings.opam-coq-archive] github="coq/opam-coq-archive" gitlab="coq/opam-coq-archive" + + [mappings.math-comp] + github="math-comp/math-comp" + gitlab="math-comp/math-comp" + gitlab_domain="gitlab.inria.fr" + + [mappings.docker-mathcomp] + github="math-comp/docker-mathcomp" + gitlab="math-comp/docker-mathcomp" + gitlab_domain="gitlab.inria.fr" diff --git a/src/actions.ml b/src/actions.ml index 51fd77bf..73fb7adf 100644 --- a/src/actions.ml +++ b/src/actions.ml @@ -18,10 +18,11 @@ type coq_job_info = ; opam_switch: string } let send_status_check ~bot_info job_info ~pr_num (gh_owner, gh_repo) - ~github_repo_full_name ~gitlab_repo_full_name ~context ~failure_reason - ~external_id ~trace = + ~github_repo_full_name ~gitlab_domain ~gitlab_repo_full_name ~context + ~failure_reason ~external_id ~trace = let job_url = - f "https://gitlab.com/%s/-/jobs/%d" gitlab_repo_full_name job_info.build_id + f "https://%s/%s/-/jobs/%d" gitlab_domain gitlab_repo_full_name + job_info.build_id in let trace_lines = trace @@ -553,8 +554,8 @@ let trace_action ~repo_full_name trace = else Warn trace ) let job_failure ~bot_info job_info ~pr_num (gh_owner, gh_repo) - ~github_repo_full_name ~gitlab_repo_full_name ~context ~failure_reason - ~external_id = + ~github_repo_full_name ~gitlab_domain ~gitlab_repo_full_name ~context + ~failure_reason ~external_id = let build_id = job_info.build_id in let project_id = job_info.common_info.project_id in Lwt_io.printf "Failed job %d of project %d.\nFailure reason: %s\n" build_id @@ -567,19 +568,25 @@ let job_failure ~bot_info job_info ~pr_num (gh_owner, gh_repo) "Failure reason reported by GitLab CI: %s.\nRetrieving the trace..." failure_reason >>= fun () -> - GitLab_queries.get_build_trace ~bot_info ~project_id ~build_id - >>= trace_action ~repo_full_name:gitlab_repo_full_name ) + GitLab_queries.get_build_trace ~bot_info ~gitlab_domain ~project_id + ~build_id + >>= function + | Ok trace -> + trace_action ~repo_full_name:gitlab_repo_full_name trace + | Error err -> + Lwt.return (Ignore (f "Error while retrieving the trace: %s." err)) ) >>= function | Warn trace -> Lwt_io.printf "Actual failure.\n" <&> send_status_check ~bot_info job_info ~pr_num (gh_owner, gh_repo) - ~github_repo_full_name ~gitlab_repo_full_name ~context - ~failure_reason ~external_id ~trace + ~github_repo_full_name ~gitlab_domain ~gitlab_repo_full_name + ~context ~failure_reason ~external_id ~trace | Retry reason -> ( Lwt_io.printlf "%s... Checking whether to retry the job." reason >>= fun () -> - GitLab_queries.get_retry_nb ~bot_info ~full_name:gitlab_repo_full_name - ~build_id ~build_name:job_info.build_name + GitLab_queries.get_retry_nb ~bot_info ~gitlab_domain + ~full_name:gitlab_repo_full_name ~build_id + ~build_name:job_info.build_name >>= function | Ok retry_nb when retry_nb < 3 -> Lwt_io.printlf @@ -587,7 +594,8 @@ let job_failure ~bot_info job_info ~pr_num (gh_owner, gh_repo) retries = %d). Retrying..." retry_nb >>= fun () -> - GitLab_mutations.retry_job ~bot_info ~project_id ~build_id + GitLab_mutations.retry_job ~bot_info ~gitlab_domain ~project_id + ~build_id | Ok retry_nb -> Lwt_io.printlf "The job has been retried %d times before. Not retrying." retry_nb @@ -597,8 +605,8 @@ let job_failure ~bot_info job_info ~pr_num (gh_owner, gh_repo) Lwt_io.printl reason let job_success_or_pending ~bot_info (gh_owner, gh_repo) - ({build_id} as job_info) ~github_repo_full_name ~gitlab_repo_full_name - ~context ~state ~external_id = + ({build_id} as job_info) ~github_repo_full_name ~gitlab_domain + ~gitlab_repo_full_name ~context ~state ~external_id = GitHub_queries.get_status_check ~bot_info ~owner:gh_owner ~repo:gh_repo ~commit:job_info.common_info.head_commit ~context >>= function @@ -608,7 +616,7 @@ let job_success_or_pending ~bot_info (gh_owner, gh_repo) it.\n" <&> let job_url = - f "https://gitlab.com/%s/-/jobs/%d" gitlab_repo_full_name build_id + f "https://%s/%s/-/jobs/%d" gitlab_domain gitlab_repo_full_name build_id in let state, status, conclusion, description = match state with @@ -658,22 +666,21 @@ let job_success_or_pending ~bot_info (gh_owner, gh_repo) | Error e -> Lwt_io.printf "%s\n" e -let job_action ~bot_info ({build_name} as job_info) ~gitlab_mapping = +let job_action ~bot_info + ({build_name; common_info= {http_repo_url}} as job_info) ~gitlab_mapping = let pr_num, branch_or_pr = pr_from_branch job_info.common_info.branch in let context = f "GitLab CI job %s (%s)" build_name branch_or_pr in - let owner, repo = - let repo_url = job_info.common_info.repo_url in - if not (string_match ~regexp:".*:\\(.*\\)/\\(.*\\).git" repo_url) then - failwith "Could not match project name on repository url.\n" ; - (Str.matched_group 1 repo_url, Str.matched_group 2 repo_url) + let gitlab_domain, gitlab_repo_full_name = + parse_gitlab_repo_url ~http_repo_url in - let gitlab_repo_full_name = owner ^ "/" ^ repo in let gh_owner, gh_repo = - github_repo_of_gitlab_project_path ~gitlab_mapping gitlab_repo_full_name + github_repo_of_gitlab_project_path ~gitlab_mapping ~gitlab_domain + ~gitlab_repo_full_name in let github_repo_full_name = gh_owner ^ "/" ^ gh_repo in let external_id = - f "projects/%d/jobs/%d" job_info.common_info.project_id job_info.build_id + f "%s,projects/%d/jobs/%d" http_repo_url job_info.common_info.project_id + job_info.build_id in match (github_repo_full_name, job_info.build_name) with | "coq/coq", "bench" -> @@ -684,17 +691,17 @@ let job_action ~bot_info ({build_name} as job_info) ~gitlab_mapping = | "failed" -> let failure_reason = Option.value_exn job_info.failure_reason in job_failure ~bot_info job_info ~pr_num (gh_owner, gh_repo) - ~github_repo_full_name ~gitlab_repo_full_name ~context ~failure_reason - ~external_id + ~github_repo_full_name ~gitlab_domain ~gitlab_repo_full_name ~context + ~failure_reason ~external_id | "success" as state -> job_success_or_pending ~bot_info (gh_owner, gh_repo) job_info - ~github_repo_full_name ~gitlab_repo_full_name ~context ~state - ~external_id + ~github_repo_full_name ~gitlab_domain ~gitlab_repo_full_name ~context + ~state ~external_id <&> send_doc_url ~bot_info job_info ~github_repo_full_name | ("created" | "running") as state -> job_success_or_pending ~bot_info (gh_owner, gh_repo) job_info - ~github_repo_full_name ~gitlab_repo_full_name ~context ~state - ~external_id + ~github_repo_full_name ~gitlab_domain ~gitlab_repo_full_name ~context + ~state ~external_id | "cancelled" | "canceled" | "pending" -> (* Ideally we should check if a status was already reported for this job. But it is important to avoid making dozens of @@ -724,7 +731,7 @@ let create_pipeline_summary ?summary_top pipeline_info pipeline_url = if String.equal build.stage stage then Some (f " - [%s](%s/-/jobs/%d)" build.build_name - pipeline_info.common_info.repo_url build.build_id ) + pipeline_info.common_info.http_repo_url build.build_id ) else None ) |> List.cons ("- " ^ stage) ) |> String.concat ~sep:"\n" @@ -1636,7 +1643,7 @@ let minimize_failed_tests ~bot_info ~owner ~repo ~pr_number (f ":runner: @%s ci minimize will minimize the \ following %s: %s" - bot_info.name + bot_info.github_name (pluralize "target" suggested_jobs_to_minimize) ( suggested_jobs_to_minimize |> List.map ~f:(fun {target} -> target) @@ -1648,7 +1655,7 @@ let minimize_failed_tests ~bot_info ~owner ~repo ~pr_number "- If you tag me saying `@%s ci minimize all`, I will \ additionally minimize the following %s (which I do not \ suggest minimizing):" - bot_info.name + bot_info.github_name (pluralize "target" possible_jobs_to_minimize) in match possible_jobs_to_minimize with @@ -1778,17 +1785,10 @@ let ci_minimize ~bot_info ~comment_info ~requests ~comment_on_error ~options RequestExplicit requests ) ~comment_on_error ~options ~bug_file_contents () -let pipeline_action ~bot_info pipeline_info ~gitlab_mapping : unit Lwt.t = - let gitlab_full_name = pipeline_info.project_path in - let repo_full_name = - match Hashtbl.find gitlab_mapping gitlab_full_name with - | Some value -> - value - | None -> - Stdio.printf - "Warning: No correspondence found for GitLab repository %s.\n" - gitlab_full_name ; - gitlab_full_name +let pipeline_action ~bot_info ({common_info= {http_repo_url}} as pipeline_info) + ~gitlab_mapping : unit Lwt.t = + let gh_owner, gh_repo = + github_repo_of_gitlab_url ~gitlab_mapping ~http_repo_url in let pr_number, _ = pr_from_branch pipeline_info.common_info.branch in match pipeline_info.state with @@ -1796,18 +1796,17 @@ let pipeline_action ~bot_info pipeline_info ~gitlab_mapping : unit Lwt.t = Lwt.return_unit | _ -> ( let pipeline_url = - f "%s/pipelines/%d" pipeline_info.common_info.repo_url - pipeline_info.pipeline_id + f "%s/pipelines/%d" http_repo_url pipeline_info.pipeline_id in let external_id = - f "projects/%d/pipelines/%d" pipeline_info.common_info.project_id - pipeline_info.pipeline_id + f "%s,projects/%d/pipelines/%d" http_repo_url + pipeline_info.common_info.project_id pipeline_info.pipeline_id in let state, status, conclusion, title, summary_top = (* For the Coq repo only, we report whether this was a full or a light CI *) let full_ci = - match repo_full_name with - | "coq/coq" -> ( + match (gh_owner, gh_repo) with + | "coq", "coq" -> ( try List.find_map ~f:(fun (key, value) -> @@ -1875,7 +1874,8 @@ let pipeline_action ~bot_info pipeline_info ~gitlab_mapping : unit Lwt.t = in match bot_info.github_install_token with | None -> - GitHub_mutations.send_status_check ~repo_full_name + GitHub_mutations.send_status_check + ~repo_full_name:(gh_owner ^ "/" ^ gh_repo) ~commit:pipeline_info.common_info.head_commit ~state ~url:pipeline_url ~context: @@ -1883,10 +1883,8 @@ let pipeline_action ~bot_info pipeline_info ~gitlab_mapping : unit Lwt.t = (pr_from_branch pipeline_info.common_info.branch |> snd) ) ~description:title ~bot_info | Some _ -> ( - let owner, repo = - github_repo_of_gitlab_project_path ~gitlab_mapping repo_full_name - in - GitHub_queries.get_repository_id ~bot_info ~owner ~repo + GitHub_queries.get_repository_id ~bot_info ~owner:gh_owner + ~repo:gh_repo >>= function | Error e -> Lwt_io.printf "No repo id: %s\n" e @@ -1904,11 +1902,12 @@ let pipeline_action ~bot_info pipeline_info ~gitlab_mapping : unit Lwt.t = >>= fun _ -> Lwt_unix.sleep 5. >>= fun () -> - match (owner, repo, pipeline_info.state, pr_number) with + match (gh_owner, gh_repo, pipeline_info.state, pr_number) with | "coq", "coq", "failed", Some pr_number -> - minimize_failed_tests ~bot_info ~owner ~repo ~pr_number - ~head_pipeline_summary:(Some summary) ~request:Auto - ~comment_on_error:false ~options:"" ~bug_file_contents:None + minimize_failed_tests ~bot_info ~owner:gh_owner ~repo:gh_repo + ~pr_number ~head_pipeline_summary:(Some summary) + ~request:Auto ~comment_on_error:false ~options:"" + ~bug_file_contents:None ?base_sha:pipeline_info.common_info.base_commit ~head_sha:pipeline_info.common_info.head_commit () | _ -> @@ -1995,7 +1994,8 @@ let coq_bug_minimizer_results_action ~bot_info ~ci ~key ~app_id body = the run-coq-bug-minimizer repo, not coqbot the GitHub App *) (f "git push https://%s:%s@github.com/%s.git --delete '%s'" - bot_info.name bot_info.github_pat repo_name branch_name ) + bot_info.github_name bot_info.github_pat repo_name + branch_name ) >>= function | Ok () -> Lwt.return_unit @@ -2290,18 +2290,18 @@ let remove_labels_if_present ~bot_info (issue : issue_info) labels = |> Lwt.async (* TODO: ensure there's no race condition for 2 push with very close timestamps *) -let mirror_action ~bot_info ?(force = true) ~owner ~repo ~base_ref ~head_sha () - = +let mirror_action ~bot_info ?(force = true) ~gitlab_domain ~owner ~repo + ~base_ref ~head_sha () = (let open Lwt_result.Infix in let local_ref = head_sha in let gh_ref = {repo_url= f "https://github.com/%s/%s" owner repo; name= base_ref} in - (* TODO: generalize to case where mapping is not one-to-one *) - let gl_ref = - { repo_url= gitlab_repo ~bot_info ~gitlab_full_name:(owner ^ "/" ^ repo) - ; name= base_ref } - in + (* TODO: generalize to use repository mappings, with enhanced security *) + gitlab_repo ~bot_info ~gitlab_domain ~gitlab_full_name:(owner ^ "/" ^ repo) + |> Lwt.return + >>= fun gl_repo -> + let gl_ref = {repo_url= gl_repo; name= base_ref} in git_fetch gh_ref local_ref |> execute_cmd >>= fun () -> git_push ~force ~remote_ref:gl_ref ~local_ref () |> execute_cmd ) @@ -2435,12 +2435,12 @@ let update_pr ?full_ci ?(skip_author_check = false) ~bot_info (* Force push *) get_options >>= fun options -> + let open Lwt_result.Infix in gitlab_ref ~issue:pr_info.issue.issue ~gitlab_mapping ~github_mapping ~bot_info - >|= (fun remote_ref -> - git_push ~force:true ~options ~remote_ref - ~local_ref:local_head_branch () ) - >>= execute_cmd ) + >>= fun remote_ref -> + git_push ~force:true ~options ~remote_ref ~local_ref:local_head_branch () + |> execute_cmd ) else ( (* Add rebase label if it exists *) add_labels_if_absent ~bot_info pr_info.issue [rebase_label] ; @@ -2522,8 +2522,11 @@ let pull_request_closed_action ~bot_info let open Lwt.Infix in gitlab_ref ~issue:pr_info.issue.issue ~gitlab_mapping ~github_mapping ~bot_info - >|= (fun remote_ref -> git_delete ~remote_ref) - >>= execute_cmd >|= ignore + >>= (function + | Ok remote_ref -> + git_delete ~remote_ref |> execute_cmd >|= ignore + | Error err -> + Lwt_io.printlf "Error: %s" err ) <&> if not pr_info.merged then Lwt_io.printf @@ -2847,8 +2850,8 @@ let run_bench ~bot_info ?key_value_pairs comment_info = match (allowed_to_bench, process_summary) with | Ok true, Ok (build_id, project_id) -> (* Permission to bench has been granted *) - GitLab_mutations.play_job ~bot_info ~project_id ~build_id ?key_value_pairs - () + GitLab_mutations.play_job ~bot_info ~gitlab_domain:"gitlab.com" + ~project_id ~build_id ?key_value_pairs () | Error err, _ | _, Error err -> GitHub_mutations.post_comment ~bot_info ~message:err ~id:pr.id >>= GitHub_mutations.report_on_posting_comment diff --git a/src/actions.mli b/src/actions.mli index 2199250c..b2a897f0 100644 --- a/src/actions.mli +++ b/src/actions.mli @@ -48,7 +48,7 @@ val run_ci_action : -> comment_info:GitHub_types.comment_info -> ?full_ci:bool -> gitlab_mapping:(string, string) Base.Hashtbl.t - -> github_mapping:(string, string) Base.Hashtbl.t + -> github_mapping:(string, string * string) Base.Hashtbl.t -> unit -> (Cohttp.Response.t * Cohttp_lwt__Body.t) Lwt.t @@ -56,7 +56,7 @@ val pull_request_closed_action : bot_info:Bot_info.t -> GitHub_types.issue_info GitHub_types.pull_request_info -> gitlab_mapping:(string, string) Base.Hashtbl.t - -> github_mapping:(string, string) Base.Hashtbl.t + -> github_mapping:(string, string * string) Base.Hashtbl.t -> unit Lwt.t val pull_request_updated_action : @@ -64,7 +64,7 @@ val pull_request_updated_action : -> action:GitHub_types.pull_request_action -> pr_info:GitHub_types.issue_info GitHub_types.pull_request_info -> gitlab_mapping:(string, string) Base.Hashtbl.t - -> github_mapping:(string, string) Base.Hashtbl.t + -> github_mapping:(string, string * string) Base.Hashtbl.t -> (Cohttp.Response.t * Cohttp_lwt__.Body.t) Lwt.t val adjust_milestone : @@ -85,6 +85,7 @@ val coq_push_action : val mirror_action : bot_info:Bot_info.t -> ?force:bool + -> gitlab_domain:string -> owner:string -> repo:string -> base_ref:string diff --git a/src/bot.ml b/src/bot.ml index 2500391d..bec2b048 100644 --- a/src/bot.ml +++ b/src/bot.ml @@ -13,17 +13,16 @@ let toml_data = Config.toml_of_file (Sys.get_argv ()).(1) let port = Config.port toml_data -let gitlab_access_token = Config.gitlab_access_token toml_data - let github_access_token = Config.github_access_token toml_data let github_webhook_secret = Config.github_webhook_secret toml_data +(* TODO: make webhook secret project-specific *) let gitlab_webhook_secret = Config.gitlab_webhook_secret toml_data let daily_schedule_secret = Config.daily_schedule_secret toml_data -let bot_name = Config.bot_name toml_data +let github_bot_name = Config.github_bot_name toml_data let key = Config.github_private_key () @@ -32,8 +31,8 @@ let app_id = Config.github_app_id toml_data let bot_info : Bot_components.Bot_info.t = { github_pat= github_access_token ; github_install_token= None - ; gitlab_token= gitlab_access_token - ; name= bot_name + ; gitlab_instances= Config.gitlab_instances toml_data + ; github_name= github_bot_name ; email= Config.bot_email toml_data ; domain= Config.bot_domain toml_data ; app_id } @@ -73,7 +72,7 @@ let callback _conn req body = ]*\\)\n\ \\(\\(.\\|\n\ \\)+\\)" - @@ Str.quote bot_name ) + @@ Str.quote github_bot_name ) body then (* avoid internal server errors from unclear execution order *) @@ -92,7 +91,7 @@ let callback _conn req body = string_match ~regexp: ( f "@%s? [Mm]inimize\\([^`]*\\)\\[\\([^]]*\\)\\] *(\\([^)]*\\))" - @@ Str.quote bot_name ) + @@ Str.quote github_bot_name ) body then (* avoid internal server errors from unclear execution order *) @@ -109,7 +108,7 @@ let callback _conn req body = string_match ~regexp: ( f "@%s:?\\( [^\n]*\\)\\b[Cc][Ii][- ][Mm]inimize:?\\([^\n]*\\)" - @@ Str.quote bot_name ) + @@ Str.quote github_bot_name ) body then let options, requests = @@ -130,7 +129,7 @@ let callback _conn req body = ]*\n\ \\(\\(.\\|\n\ \\)+\\)" - @@ Str.quote bot_name ) + @@ Str.quote github_bot_name ) body then let options, requests, body = @@ -154,8 +153,8 @@ let callback _conn req body = the tagging to "@`coqbot minimize foo`" so that the matching below doesn't pick up the name *) Str.global_replace - (Str.regexp (f "\\(`\\|\\)@%s " @@ Str.quote bot_name)) - (f "@\\1%s " @@ Str.quote bot_name) + (Str.regexp (f "\\(`\\|\\)@%s " @@ Str.quote github_bot_name)) + (f "@\\1%s " @@ Str.quote github_bot_name) body in let body = Cohttp_lwt.Body.to_string body in @@ -168,22 +167,21 @@ let callback _conn req body = GitLab_subscriptions.receive_gitlab ~secret:gitlab_webhook_secret (Request.headers req) body with - | Ok (_, JobEvent job_info) -> + | Ok (_, JobEvent ({common_info= {http_repo_url}} as job_info)) -> (fun () -> let gh_owner, gh_repo = - github_repo_of_gitlab_url ~gitlab_mapping - job_info.common_info.repo_url + github_repo_of_gitlab_url ~gitlab_mapping ~http_repo_url in action_as_github_app ~bot_info ~key ~app_id ~owner:gh_owner ~repo:gh_repo (job_action ~gitlab_mapping job_info) ) |> Lwt.async ; Server.respond_string ~status:`OK ~body:"Job event." () - | Ok (_, PipelineEvent pipeline_info) -> + | Ok (_, PipelineEvent ({common_info= {http_repo_url}} as pipeline_info)) + -> (fun () -> let owner, repo = - github_repo_of_gitlab_project_path ~gitlab_mapping - pipeline_info.project_path + github_repo_of_gitlab_url ~gitlab_mapping ~http_repo_url in action_as_github_app ~bot_info ~key ~app_id ~owner ~repo (pipeline_action ~gitlab_mapping pipeline_info) ) @@ -222,13 +220,28 @@ let callback _conn req body = () | Ok (true, PushEvent {owner; repo; base_ref; head_sha; _}) -> ( match (owner, repo) with - | "coq-community", ("docker-base" | "docker-coq") - (*| "math-comp", ("docker-mathcomp" | "math-comp")*) -> + | "coq-community", ("docker-base" | "docker-coq") -> + (fun () -> + init_git_bare_repository ~bot_info + >>= fun () -> + action_as_github_app ~bot_info ~key ~app_id ~owner ~repo + (mirror_action ~gitlab_domain:"gitlab.com" ~owner ~repo + ~base_ref ~head_sha () ) ) + |> Lwt.async ; + Server.respond_string ~status:`OK + ~body: + (f + "Processing push event on %s/%s repository: mirroring \ + branch on GitLab." + owner repo ) + () + | "math-comp", ("docker-mathcomp" | "math-comp") -> (fun () -> init_git_bare_repository ~bot_info >>= fun () -> action_as_github_app ~bot_info ~key ~app_id ~owner ~repo - (mirror_action ~owner ~repo ~base_ref ~head_sha ()) ) + (mirror_action ~gitlab_domain:"gitlab.inria.fr" ~owner ~repo + ~base_ref ~head_sha () ) ) |> Lwt.async ; Server.respond_string ~status:`OK ~body: @@ -369,7 +382,7 @@ let callback _conn req body = string_match ~regexp: ( f "@%s:? [Rr]un \\(full\\|light\\|\\) ?[Cc][Ii]" - @@ Str.quote bot_name ) + @@ Str.quote github_bot_name ) body && comment_info.issue.pull_request && String.equal comment_info.issue.issue.owner "coq" @@ -396,7 +409,8 @@ let callback _conn req body = ~github_mapping () ) else if string_match - ~regexp:(f "@%s:? [Mm]erge now" @@ Str.quote bot_name) + ~regexp: + (f "@%s:? [Mm]erge now" @@ Str.quote github_bot_name) body && comment_info.issue.pull_request && String.equal comment_info.issue.issue.owner "coq" @@ -414,7 +428,8 @@ let callback _conn req body = () ) else if string_match - ~regexp:(f "@%s:? [Bb]ench native" @@ Str.quote bot_name) + ~regexp: + (f "@%s:? [Bb]ench native" @@ Str.quote github_bot_name) body && comment_info.issue.pull_request && String.equal comment_info.issue.issue.owner "coq" @@ -434,7 +449,7 @@ let callback _conn req body = () ) else if string_match - ~regexp:(f "@%s:? [Bb]ench" @@ Str.quote bot_name) + ~regexp:(f "@%s:? [Bb]ench" @@ Str.quote github_bot_name) body && comment_info.issue.pull_request && String.equal comment_info.issue.issue.owner "coq" @@ -454,24 +469,45 @@ let callback _conn req body = Server.respond_string ~status:`OK ~body:(f "Unhandled comment: %s" body) () ) ) ) - | Ok (signed, CheckRunReRequested {external_id}) -> + | Ok (signed, CheckRunReRequested {external_id}) -> ( if not signed then Server.respond_string ~status:(Code.status_of_code 401) ~body:"Request to rerun check run must be signed." () else if String.is_empty external_id then Server.respond_string ~status:(Code.status_of_code 400) ~body:"Request to rerun check run but empty external ID." () - else ( - (fun () -> - GitLab_mutations.generic_retry ~bot_info ~url_part:external_id ) - |> Lwt.async ; - Server.respond_string ~status:`OK - ~body: - (f - "Received a request to re-run a job / pipeline (GitLab ID : \ - %s)." - external_id ) - () ) + else + let external_id_parsed = + match String.split ~on:',' external_id with + | [gitlab_domain; url_part] -> + Some (gitlab_domain, url_part) + | [url_part] -> + (* Backward compatibility *) + Some ("gitlab.com", url_part) + | _ -> + None + in + match external_id_parsed with + | None -> + Server.respond_string ~status:(Code.status_of_code 400) + ~body: + (f + "Request to rerun check run but external ID is not \ + well-formed: %s" + external_id ) + () + | Some (gitlab_domain, url_part) -> + (fun () -> + GitLab_mutations.generic_retry ~bot_info ~gitlab_domain + ~url_part ) + |> Lwt.async ; + Server.respond_string ~status:`OK + ~body: + (f + "Received a request to re-run a job / pipeline \ + (External ID : %s)." + external_id ) + () ) | Ok (_, UnsupportedEvent s) -> Server.respond_string ~status:`OK ~body:(f "No action taken: %s" s) () | Ok _ -> diff --git a/src/config.ml b/src/config.ml index 846bbaaf..4444a573 100644 --- a/src/config.ml +++ b/src/config.ml @@ -25,12 +25,50 @@ let port toml_data = ~default: (Option.value_map (Sys.getenv "PORT") ~f:Int.of_string ~default:8000) -let gitlab_access_token toml_data = - match subkey_value toml_data "gitlab" "api_token" with - | None -> - Sys.getenv_exn "GITLAB_ACCESS_TOKEN" - | Some secret -> - secret +let github_bot_name toml_data = + Option.value_map + (subkey_value toml_data "bot" "name") + ~f:String.of_string ~default:"coqbot" + +let gitlab_instances toml_data = + ( try + match find "gitlab" toml_data with + | Toml.Types.TTable a -> + list_table_keys a + |> List.map ~f:(fun k -> + let bot_name = + subkey_value a k "bot_name" + |> Option.value ~default:(github_bot_name toml_data) + in + match + (subkey_value a k "domain", subkey_value a k "api_token") + with + | None, _ -> + failwith + (f "Invalid gitlab.%s configuration: missing domain key." + k ) + | Some domain, Some api_token -> + (* If api_token is found, we use its value in priority *) + (domain, (bot_name, api_token)) + | Some domain, None -> ( + (* Otherwise, we look for an environment variable, whose + name is given by api_token_env_var *) + match subkey_value a k "api_token_env_var" with + | Some api_token_env_var -> + (domain, (bot_name, Sys.getenv_exn api_token_env_var)) + | _ -> + failwith + (f + "Invalid gitlab.%s configuration: missing \ + api_token and api_token_env_var keys." + k ) ) ) + | _ -> + failwith "Invalid gitlab configuration: not a table." + with Stdlib.Not_found -> + [ ( "gitlab.com" + , (github_bot_name toml_data, Sys.getenv_exn "GITLAB_ACCESS_TOKEN") ) ] + ) + |> Hashtbl.of_alist_exn (module String) let github_access_token toml_data = match subkey_value toml_data "github" "api_token" with @@ -64,22 +102,17 @@ let daily_schedule_secret toml_data = | Some secret -> secret -let bot_name toml_data = - Option.value_map - (subkey_value toml_data "bot" "name") - ~f:String.of_string ~default:"coqbot" - let bot_domain toml_data = Option.value_map (subkey_value toml_data "server" "domain") ~f:String.of_string - ~default:(f "%s.herokuapp.com" (bot_name toml_data)) + ~default:(f "%s.herokuapp.com" (github_bot_name toml_data)) let bot_email toml_data = Option.value_map (subkey_value toml_data "bot" "email") ~f:String.of_string - ~default:(f "%s@users.noreply.github.com" (bot_name toml_data)) + ~default:(f "%s@users.noreply.github.com" (github_bot_name toml_data)) let github_app_id toml_data = match subkey_value toml_data "github" "app_id" with @@ -90,8 +123,6 @@ let github_app_id toml_data = | Some app_id -> app_id |> Int.of_string -(*let string_of_file_path path = Stdio.In_channel.(with_file path ~f:input_all)*) - let github_private_key () = (*string_of_file_path "./github.private-key.pem"*) match @@ -109,21 +140,24 @@ let github_private_key () = failwith (f "Error while decoding RSA key: %s" e) let parse_mappings mappings = - let keys = list_table_keys mappings in let assoc = - List.( - fold_left - ~f:(fun assoc_table k -> - (subkey_value mappings k "github", subkey_value mappings k "gitlab") - :: assoc_table ) - ~init:[] keys - |> filter_map ~f:(function + list_table_keys mappings + |> List.map ~f:(fun k -> + match + (subkey_value mappings k "github", subkey_value mappings k "gitlab") + with | Some gh, Some gl -> - Some (gh, gl) + let gl_domain = + subkey_value mappings k "gitlab_domain" + |> Option.value ~default:"gitlab.com" + in + (gh, (gl_domain, gl)) | _, _ -> - None ) ) + failwith (f "Missing github or gitlab key for mappings.%s" k) ) + in + let assoc_rev = + List.map assoc ~f:(fun (gh, (gl_domain, gl)) -> (gl_domain ^ "/" ^ gl, gh)) in - let assoc_rev = List.map assoc ~f:(fun (gh, gl) -> (gl, gh)) in let get_table t = match t with | `Duplicate_key _ -> diff --git a/src/config.mli b/src/config.mli index 55abeff6..fe997584 100644 --- a/src/config.mli +++ b/src/config.mli @@ -8,26 +8,27 @@ val string_of_mapping : (string, string) Base.Hashtbl.t -> string val port : Toml.Types.table -> int -val gitlab_access_token : Toml.Types.table -> string - -val github_access_token : Toml.Types.table -> string - -val github_webhook_secret : Toml.Types.table -> string +val gitlab_instances : + Toml.Types.table -> (string, string * string) Base.Hashtbl.t val gitlab_webhook_secret : Toml.Types.table -> string val daily_schedule_secret : Toml.Types.table -> string -val bot_name : Toml.Types.table -> string - val bot_domain : Toml.Types.table -> string val bot_email : Toml.Types.table -> string +val github_bot_name : Toml.Types.table -> string + +val github_access_token : Toml.Types.table -> string + val github_app_id : Toml.Types.table -> int +val github_webhook_secret : Toml.Types.table -> string + val github_private_key : unit -> Mirage_crypto_pk.Rsa.priv val make_mappings_table : Toml.Types.value Toml.Types.Table.t - -> (string, string) Base.Hashtbl.t * (string, string) Base.Hashtbl.t + -> (string, string * string) Base.Hashtbl.t * (string, string) Base.Hashtbl.t diff --git a/src/git_utils.ml b/src/git_utils.ml index f1645c57..4b69ab0e 100644 --- a/src/git_utils.ml +++ b/src/git_utils.ml @@ -5,13 +5,16 @@ open Bot_components.GitHub_types open Helpers open Lwt.Infix -let gitlab_repo ~bot_info ~gitlab_full_name = - f "https://oauth2:%s@gitlab.com/%s.git" bot_info.gitlab_token gitlab_full_name +let gitlab_repo ~bot_info ~gitlab_domain ~gitlab_full_name = + gitlab_token bot_info gitlab_domain + |> Result.map ~f:(fun token -> + f "https://oauth2:%s@%s/%s.git" token gitlab_domain gitlab_full_name ) let report_status command report code = Error (f {|Command "%s" %s %d\n|} command report code) let gitlab_ref ~bot_info ~(issue : issue) ~github_mapping ~gitlab_mapping = + let default_gitlab_domain = "gitlab.com" in let gh_repo = issue.owner ^ "/" ^ issue.repo in let open Lwt.Infix in (* First, we check our hashtable for a key named after the GitHub @@ -19,6 +22,7 @@ let gitlab_ref ~bot_info ~(issue : issue) ~github_mapping ~gitlab_mapping = key is not found, we load the config file from the default branch. Last (backward-compatibility) we assume the GitLab and GitHub projects are named the same. *) + let default_value = (default_gitlab_domain, gh_repo) in ( match Hashtbl.find github_mapping gh_repo with | None -> ( Stdio.printf "No correspondence found for GitHub repository %s/%s.\n" @@ -29,9 +33,16 @@ let gitlab_ref ~bot_info ~(issue : issue) ~github_mapping ~gitlab_mapping = | Ok branch -> ( GitHub_queries.get_file_content ~bot_info ~owner:issue.owner ~repo:issue.repo ~branch - ~file_name:(f "%s.toml" bot_info.name) + ~file_name:(f "%s.toml" bot_info.github_name) >>= function | Ok (Some content) -> + let gl_domain = + Option.value + (Config.subkey_value + (Config.toml_of_string content) + "mapping" "gitlab_domain" ) + ~default:default_gitlab_domain + in let gl_repo = Option.value (Config.subkey_value @@ -39,26 +50,34 @@ let gitlab_ref ~bot_info ~(issue : issue) ~github_mapping ~gitlab_mapping = "mapping" "gitlab" ) ~default:gh_repo in - ( match Hashtbl.add gitlab_mapping ~key:gl_repo ~data:gh_repo with + ( match + Hashtbl.add gitlab_mapping + ~key:(gl_domain ^ "/" ^ gl_repo) + ~data:gh_repo + with | `Duplicate -> () | `Ok -> () ) ; - ( match Hashtbl.add github_mapping ~key:gh_repo ~data:gl_repo with + ( match + Hashtbl.add github_mapping ~key:gh_repo + ~data:(gl_domain, gl_repo) + with | `Duplicate -> () | `Ok -> () ) ; - Lwt.return gl_repo + Lwt.return (gl_domain, gl_repo) | _ -> - Lwt.return gh_repo ) + Lwt.return default_value ) | _ -> - Lwt.return gh_repo ) + Lwt.return default_value ) | Some r -> Lwt.return r ) - >|= fun gitlab_full_name -> - { name= f "refs/heads/pr-%d" issue.number - ; repo_url= gitlab_repo ~gitlab_full_name ~bot_info } + >|= fun (gitlab_domain, gitlab_full_name) -> + gitlab_repo ~gitlab_domain ~gitlab_full_name ~bot_info + |> Result.map ~f:(fun gl_repo -> + {name= f "refs/heads/pr-%d" issue.number; repo_url= gl_repo} ) let ( |&& ) command1 command2 = command1 ^ " && " ^ command2 @@ -139,7 +158,7 @@ let git_coq_bug_minimizer ~bot_info ~script ~comment_thread_id ~comment_author ; GitHub_ID.to_string comment_thread_id ; comment_author ; bot_info.github_pat - ; bot_info.name + ; bot_info.github_name ; bot_info.domain ; owner ; repo @@ -156,7 +175,7 @@ let git_run_ci_minimization ~bot_info ~comment_thread_id ~owner ~repo ~pr_number not coqbot the GitHub App *) ( [ GitHub_ID.to_string comment_thread_id ; bot_info.github_pat - ; bot_info.name + ; bot_info.github_name ; bot_info.domain ; owner ; repo @@ -178,7 +197,7 @@ let init_git_bare_repository ~bot_info = Stdio.printf "Initializing repository...\n" ; "git init --bare" |&& f {|git config user.email "%s"|} bot_info.email - |&& f {|git config user.name "%s"|} bot_info.name + |&& f {|git config user.name "%s"|} bot_info.github_name |> execute_cmd >|= function | Ok _ -> diff --git a/src/git_utils.mli b/src/git_utils.mli index e42dbb11..04334562 100644 --- a/src/git_utils.mli +++ b/src/git_utils.mli @@ -1,12 +1,15 @@ val gitlab_repo : - bot_info:Bot_components.Bot_info.t -> gitlab_full_name:string -> string + bot_info:Bot_components.Bot_info.t + -> gitlab_domain:string + -> gitlab_full_name:string + -> (string, string) Result.t val gitlab_ref : bot_info:Bot_components.Bot_info.t -> issue:Bot_components.GitHub_types.issue - -> github_mapping:(string, string) Base.Hashtbl.t + -> github_mapping:(string, string * string) Base.Hashtbl.t -> gitlab_mapping:(string, string) Base.Hashtbl.t - -> Bot_components.GitHub_types.remote_ref_info Lwt.t + -> (Bot_components.GitHub_types.remote_ref_info, string) Lwt_result.t val ( |&& ) : string -> string -> string diff --git a/src/helpers.ml b/src/helpers.ml index 9bc63512..b0c3dd4a 100644 --- a/src/helpers.ml +++ b/src/helpers.ml @@ -52,28 +52,67 @@ let trim_comments comment = in aux comment 0 false -let github_repo_of_gitlab_project_path ~gitlab_mapping gitlab_full_name = +let github_repo_of_gitlab_project_path ~gitlab_mapping ~gitlab_domain + ~gitlab_repo_full_name = + let full_name_with_domain = gitlab_domain ^ "/" ^ gitlab_repo_full_name in let github_full_name = - match Hashtbl.find gitlab_mapping gitlab_full_name with + match Hashtbl.find gitlab_mapping full_name_with_domain with | Some value -> value | None -> Stdio.printf "Warning: No correspondence found for GitLab repository %s.\n" - gitlab_full_name ; - gitlab_full_name + full_name_with_domain ; + gitlab_repo_full_name in match Str.split (Str.regexp "/") github_full_name with | [owner; repo] -> (owner, repo) | _ -> - failwith "Could not split github_full_name into (owner, repo)." + failwith + (f "Could not split repository full name %s into (owner, repo)." + github_full_name ) -let github_repo_of_gitlab_url ~gitlab_mapping gitlab_repo_url = - let owner, repo = - if not (string_match ~regexp:".*:\\(.*\\)/\\(.*\\).git" gitlab_repo_url) - then Stdio.printf "Could not match project name on repository url.\n" ; - (Str.matched_group 1 gitlab_repo_url, Str.matched_group 2 gitlab_repo_url) +let parse_gitlab_repo_url ~http_repo_url = + if not (string_match ~regexp:"https?://\\([^/]*\\)/\\(.*/.*\\)" http_repo_url) + then failwith "Could not match project name on repository url.\n" ; + (Str.matched_group 1 http_repo_url, Str.matched_group 2 http_repo_url) + +let%expect_test "http_repo_url_parsing_coq" = + let gitlab_domain, gitlab_repo_full_name = + parse_gitlab_repo_url ~http_repo_url:"https://gitlab.com/coq/coq" + in + Stdio.print_endline gitlab_domain ; + Stdio.print_endline gitlab_repo_full_name ; + [%expect {| + gitlab.com + coq/coq |}] + +let%expect_test "http_repo_url_parsing_mathcomp" = + let gitlab_domain, gitlab_repo_full_name = + parse_gitlab_repo_url + ~http_repo_url:"https://gitlab.inria.fr/math-comp/math-comp" + in + Stdio.print_endline gitlab_domain ; + Stdio.print_endline gitlab_repo_full_name ; + [%expect {| + gitlab.inria.fr + math-comp/math-comp |}] + +let%expect_test "http_repo_url_parsing_example_from_gitlab_docs" = + let gitlab_domain, gitlab_repo_full_name = + parse_gitlab_repo_url + ~http_repo_url:"http://192.168.64.1:3005/gitlab-org/gitlab-test" + in + Stdio.print_endline gitlab_domain ; + Stdio.print_endline gitlab_repo_full_name ; + [%expect {| + 192.168.64.1:3005 + gitlab-org/gitlab-test |}] + +let github_repo_of_gitlab_url ~gitlab_mapping ~http_repo_url = + let gitlab_domain, gitlab_repo_full_name = + parse_gitlab_repo_url ~http_repo_url in - let repo_full_name = owner ^ "/" ^ repo in - github_repo_of_gitlab_project_path ~gitlab_mapping repo_full_name + github_repo_of_gitlab_project_path ~gitlab_mapping ~gitlab_domain + ~gitlab_repo_full_name diff --git a/src/helpers.mli b/src/helpers.mli index 1b33a262..0f2f6602 100644 --- a/src/helpers.mli +++ b/src/helpers.mli @@ -21,7 +21,14 @@ val remove_between : string -> int -> int -> string val trim_comments : string -> string val github_repo_of_gitlab_project_path : - gitlab_mapping:(string, string) Base.Hashtbl.t -> string -> string * string + gitlab_mapping:(string, string) Base.Hashtbl.t + -> gitlab_domain:string + -> gitlab_repo_full_name:string + -> string * string + +val parse_gitlab_repo_url : http_repo_url:string -> string * string val github_repo_of_gitlab_url : - gitlab_mapping:(string, string) Base.Hashtbl.t -> string -> string * string + gitlab_mapping:(string, string) Base.Hashtbl.t + -> http_repo_url:string + -> string * string