diff --git a/README.md b/README.md index 4bf6495e..b83287e0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Monorobot -A Slackbot for GitHub monorepos. Configure how repo notifications should be routed to specified Slack channels based on file prefixes, issue/PR labels, and CI build statuses. +A Slackbot for GitHub monorepos. Configure how repo notifications should be routed to specified Slack channels based on file prefixes, issue/PR labels, and CI build statuses. Supports custom notification behavior for multiple repositories. ## Setting Up @@ -26,7 +26,7 @@ Run the `_build/default/src/notabot.exe` binary. The following commands are supp ### Documentation -The bot expects two configuration files to be present. +Add a configuration file to each repository you want to support, and a secrets file on the bot server itself. Read on for instructions to set up each file: * [Repository configuration](./documentation/config_docs.md) * [Secrets](./documentation/secret_docs.md) diff --git a/documentation/secret_docs.md b/documentation/secret_docs.md index 516d894d..d052dff6 100644 --- a/documentation/secret_docs.md +++ b/documentation/secret_docs.md @@ -46,11 +46,39 @@ A secrets file stores sensitive information. Unlike the repository configuration | `slack_hooks` | list of channel names (`channel`) and their corresponding webhook endpoint (`url`) | No | - | | `gh_token` | specify to grant the bot access to private repositories; omit for public repositories | Yes | - | | `gh_hook_token` | specify to ensure the bot only receives GitHub notifications from pre-approved repositories | Yes | - | +| `repositories` | an object mapping repository URLs to repository-specific GitHub secrets | Yes | - | +| `allowed_repositories` | a whitelist of repository URLs to process payloads for | Yes | all incoming payloads are processed | ## `gh_token` Some operations, such as fetching a config file from a private repository, or the commit corresponding to a commit comment event, require a personal access token. Refer [here](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token) for detailed instructions on token generation. +*See `repositories` if you need to support multiple repositories that use different tokens.* + ## `gh_hook_token` Refer [here](https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/securing-your-webhooks) for more information on securing webhooks with a token. + +*See `repositories` if you need to support multiple repositories that use different tokens.* + +## `repositories` + +If you're using Monorobot for multiple repositories that need different secrets (e.g., one on github.com and another on GitHub Enterprise), you can provide them as an object. Secrets defined here will take precedence over those defined at the top level of the secrets file. + +Repository URLs should be fully qualified (include the protocol). + +```json +{ + "https://github.com/acme/runner" : { + "gh_token": "XXX" + }, + "https://git.acme.com/acme/coyote" : { + "gh_token": "XXX", + "gh_hook_token": "XXX" + } +} +``` + +## `allowed_repositories` + +Use this option to restrict incoming notifications from GitHub to approved repository URLs. diff --git a/lib/action.ml b/lib/action.ml index d63ac13a..6415ebaa 100644 --- a/lib/action.ml +++ b/lib/action.ml @@ -87,13 +87,14 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct if List.is_empty matched_channel_names then default else matched_channel_names let partition_status (ctx : Context.t) (n : status_notification) = - let cfg = Context.get_config_exn ctx in + let repo = n.repository in + let cfg = State.find_repo_config_exn ctx.state repo.url in let pipeline = n.context in let current_status = n.state in let rules = cfg.status_rules.rules in let action_on_match (branches : branch list) = let default = Option.to_list cfg.prefix_rules.default_channel in - let () = Context.refresh_pipeline_status ~pipeline ~branches ~status:current_status ctx in + State.set_repo_pipeline_status ctx.state repo.url ~pipeline ~branches ~status:current_status; match List.is_empty branches with | true -> Lwt.return [] | false -> @@ -105,18 +106,18 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct | false -> Lwt.return default | true -> let sha = n.commit.sha in - let repo = n.repository in ( match%lwt Github_api.get_api_commit ~ctx ~repo ~sha with | Error e -> action_error e | Ok commit -> Lwt.return @@ partition_commit cfg commit.files ) in - if Context.is_pipeline_allowed ctx ~pipeline then begin + if State.is_pipeline_allowed ctx.state repo.url ~pipeline then begin + let repo_state = State.find_repo_exn ctx.state repo.url in match Rule.Status.match_rules ~rules n with | Some Ignore | None -> Lwt.return [] | Some Allow -> action_on_match n.branches | Some Allow_once -> - match Map.find ctx.state.pipeline_statuses pipeline with + match Map.find repo_state.pipeline_statuses pipeline with | Some branch_statuses -> let has_same_status_state_as_prev (branch : branch) = match Map.find branch_statuses branch.name with @@ -130,7 +131,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct else Lwt.return [] let partition_commit_comment (ctx : Context.t) n = - let cfg = Context.get_config_exn ctx in + let cfg = State.find_repo_config_exn ctx.state n.repository.url in match n.comment.commit_id with | None -> action_error "unable to find commit id for this commit comment event" | Some sha -> @@ -149,7 +150,8 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct ) let generate_notifications (ctx : Context.t) req = - let cfg = Context.get_config_exn ctx in + let repo = Github.repo_of_notification req in + let cfg = State.find_repo_config_exn ctx.state repo.url in match req with | Github.Push n -> partition_push cfg n |> List.map ~f:(fun (webhook, n) -> webhook, generate_push_notification n) |> Lwt.return @@ -190,20 +192,21 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct in Lwt_list.iter_s notify notifications - (** `refresh_config_of_context ctx n` updates the current context if the configuration - hasn't been loaded yet, or if the incoming request `n` is a push + (** `refresh_repo_config ctx n` fetches the latest repo config if it's + uninitialized in state, or if the incoming request `n` is a push notification containing commits that touched the config file. *) - let refresh_config_of_context (ctx : Context.t) notification = + let refresh_repo_config (ctx : Context.t) notification = + let repo = Github.repo_of_notification notification in let fetch_config () = - let repo = Github.repo_of_notification notification in match%lwt Github_api.get_config ~ctx ~repo with | Ok config -> - Context.print_config ctx; - ctx.config <- Some config; + State.set_repo_config ctx.state ~repo_url:repo.url ~config; + if ctx.verbose then Context.print_config ctx repo.url; Lwt.return @@ Ok () | Error e -> action_error e in - match ctx.config with + let repo_state = State.find_or_add_repo ctx.state repo.url in + match repo_state.config with | None -> fetch_config () | Some _ -> match notification with @@ -215,12 +218,28 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct | _ -> Lwt.return @@ Ok () let process_github_notification (ctx : Context.t) headers body = + let validate_signature secrets payload = + let repo = Github.repo_of_notification payload in + let signing_key = Context.gh_hook_token_of_secrets secrets repo.url in + Github.validate_signature ?signing_key ~headers body + in + let repo_is_allowed secrets payload = + let repo = Github.repo_of_notification payload in + let allowed_repositories = secrets.allowed_repositories in + List.is_empty allowed_repositories || List.exists allowed_repositories ~f:(String.equal repo.url) + in try%lwt let secrets = Context.get_secrets_exn ctx in - match Github.parse_exn ~secret:secrets.gh_hook_token headers body with + match Github.parse_exn headers body with | exception exn -> Exn_lwt.fail ~exn "failed to parse payload" | payload -> - ( match%lwt refresh_config_of_context ctx payload with + match validate_signature secrets payload with + | Error e -> action_error e + | Ok () -> + match repo_is_allowed secrets payload with + | false -> action_error "unsupported repository" + | true -> + ( match%lwt refresh_repo_config ctx payload with | Error e -> action_error e | Ok () -> let%lwt notifications = generate_notifications ctx payload in @@ -244,4 +263,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct | Context.Context_error msg -> log#error "%s" msg; Lwt.return_unit + | State.State_error msg -> + log#error "%s" msg; + Lwt.return_unit end diff --git a/lib/api_local.ml b/lib/api_local.ml index 4d78c883..9d6a522e 100644 --- a/lib/api_local.ml +++ b/lib/api_local.ml @@ -7,8 +7,10 @@ let cwd = Caml.Sys.getcwd () let cache_dir = Caml.Filename.concat cwd "github-api-cache" module Github : Api.Github = struct + let mock_config_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_config" + let get_config ~(ctx : Context.t) ~repo:_ = - let url = Caml.Filename.concat cwd ctx.config_filename in + let url = Caml.Filename.concat mock_config_dir ctx.config_filename in match get_local_file url with | Error e -> Lwt.return @@ fmt_error "error while getting local file: %s\nfailed to get config %s" e url | Ok file -> Lwt.return @@ Ok (Config_j.config_of_string file) diff --git a/lib/api_remote.ml b/lib/api_remote.ml index 8ec07bf7..416958ea 100644 --- a/lib/api_remote.ml +++ b/lib/api_remote.ml @@ -5,7 +5,7 @@ open Common module Github : Api.Github = struct let commits_url ~(repo : Github_t.repository) ~sha = - String.substr_replace_first ~pattern:"{/sha}" ~with_:sha repo.commits_url + String.substr_replace_first ~pattern:"{/sha}" ~with_:("/" ^ sha) repo.commits_url let contents_url ~(repo : Github_t.repository) ~path = String.substr_replace_first ~pattern:"{+path}" ~with_:path repo.contents_url @@ -17,7 +17,8 @@ module Github : Api.Github = struct let get_config ~(ctx : Context.t) ~repo = let secrets = Context.get_secrets_exn ctx in let url = contents_url ~repo ~path:ctx.config_filename in - let headers = build_headers ?token:secrets.gh_token () in + let token = Context.gh_token_of_secrets secrets repo.url in + let headers = build_headers ?token () in match%lwt http_request ~headers `GET url with | Error e -> Lwt.return @@ fmt_error "error while querying remote: %s\nfailed to get config from file %s" e url | Ok res -> @@ -41,7 +42,8 @@ module Github : Api.Github = struct let get_api_commit ~(ctx : Context.t) ~repo ~sha = let secrets = Context.get_secrets_exn ctx in let url = commits_url ~repo ~sha in - let headers = build_headers ?token:secrets.gh_token () in + let token = Context.gh_token_of_secrets secrets repo.url in + let headers = build_headers ?token () in match%lwt http_request ~headers `GET url with | Ok res -> Lwt.return @@ Ok (Github_j.api_commit_of_string res) | Error e -> Lwt.return @@ fmt_error "error while querying remote: %s\nfailed to get api commit from file %s" e url diff --git a/lib/common.atd b/lib/common.atd new file mode 100644 index 00000000..280a4c5d --- /dev/null +++ b/lib/common.atd @@ -0,0 +1,7 @@ +type 'v map_as_object = + (string * 'v) list + wrap + +type 'v table_as_object = + (string * 'v) list + wrap diff --git a/lib/common.ml b/lib/common.ml index aa4fa3ee..e0b84278 100644 --- a/lib/common.ml +++ b/lib/common.ml @@ -15,6 +15,18 @@ module StringMap = struct let unwrap = to_list end +module Table = struct + type 'a t = (string, 'a) Hashtbl.t + + let to_list (l : 'a t) : (string * 'a) list = Hashtbl.to_alist l + + let of_list (m : (string * 'a) list) : 'a t = Hashtbl.of_alist_exn (module String) m + + let wrap = of_list + + let unwrap = to_list +end + let fmt_error fmt = Printf.ksprintf (fun s -> Error s) fmt let first_line s = diff --git a/lib/config.atd b/lib/config.atd index 08529549..1fcd092a 100644 --- a/lib/config.atd +++ b/lib/config.atd @@ -1,6 +1,7 @@ type status_rule = abstract type prefix_rule = abstract type label_rule = abstract +type 'v map_as_object = abstract (* This type of rule is used for CI build notifications. *) type status_rules = { @@ -36,10 +37,17 @@ type webhook = { channel : string; (* name of the Slack channel to post the message *) } +type gh_repo_secrets = { + ?gh_token : string option; (* GitHub personal access token, if repo access requires it *) + ?gh_hook_token : string option; (* GitHub webhook token to secure the webhook *) +} + (* This is the structure of the secrets file which stores sensitive information, and shouldn't be checked into version control. *) type secrets = { slack_hooks : webhook list; ?gh_token : string option; (* GitHub personal access token, if repo access requires it *) ?gh_hook_token : string option; (* GitHub webhook token to secure the webhook *) + ~repositories : gh_repo_secrets map_as_object; + ~allowed_repositories : string list; (* whitelist of repository URLs to handle notifications for *) } diff --git a/lib/context.ml b/lib/context.ml index bbbd70a8..3058c7dd 100644 --- a/lib/context.ml +++ b/lib/context.ml @@ -11,8 +11,8 @@ type t = { secrets_filepath : string; state_filepath : string option; mutable secrets : Config_t.secrets option; - mutable config : Config_t.config option; state : State_t.state; + verbose : bool; } let default : t = @@ -21,24 +21,30 @@ let default : t = secrets_filepath = "secrets.json"; state_filepath = None; secrets = None; - config = None; - state = State.empty; + state = State.empty (); + verbose = true; } -let make ?config_filename ?secrets_filepath ?state_filepath () = +let make ?config_filename ?secrets_filepath ?state_filepath ?verbose () = let config_filename = Option.value config_filename ~default:default.config_filename in let secrets_filepath = Option.value secrets_filepath ~default:default.secrets_filepath in - { default with config_filename; secrets_filepath; state_filepath } + let verbose = Option.value verbose ~default:default.verbose in + { default with config_filename; secrets_filepath; state_filepath; verbose } let get_secrets_exn ctx = match ctx.secrets with | None -> context_error "secrets is uninitialized" | Some secrets -> secrets -let get_config_exn ctx = - match ctx.config with - | None -> context_error "config is uninitialized" - | Some config -> config +let gh_token_of_secrets (secrets : Config_t.secrets) repo_url = + match Map.find secrets.repositories repo_url with + | None -> secrets.gh_token + | Some repo_secrets -> repo_secrets.gh_token + +let gh_hook_token_of_secrets (secrets : Config_t.secrets) repo_url = + match Map.find secrets.repositories repo_url with + | None -> secrets.gh_hook_token + | Some repo_secrets -> repo_secrets.gh_hook_token let hook_of_channel ctx channel_name = let secrets = get_secrets_exn ctx in @@ -46,20 +52,6 @@ let hook_of_channel ctx channel_name = | Some hook -> Some hook.url | None -> None -(** `is_pipeline_allowed ctx p` returns `true` if ctx.config.status_rules - doesn't define a whitelist of allowed pipelines, or if the list - contains pipeline `p`; returns `false` otherwise. *) -let is_pipeline_allowed ctx ~pipeline = - match ctx.config with - | None -> false - | Some config -> - match config.status_rules.allowed_pipelines with - | Some allowed_pipelines when not @@ List.exists allowed_pipelines ~f:(String.equal pipeline) -> false - | _ -> true - -let refresh_pipeline_status ctx ~pipeline ~(branches : Github_t.branch list) ~status = - if is_pipeline_allowed ctx ~pipeline then State.refresh_pipeline_status ctx.state ~pipeline ~branches ~status else () - let log = Log.from "context" let refresh_secrets ctx = @@ -74,19 +66,24 @@ let refresh_state ctx = match ctx.state_filepath with | None -> Ok ctx | Some path -> - log#info "loading saved state from file %s" path; - ( match get_local_file path with - | Error e -> fmt_error "error while getting local file: %s\nfailed to get state from file %s" e path - | Ok file -> - let state = State_j.state_of_string file in - Ok { ctx with state } - ) + if Caml.Sys.file_exists path then begin + log#info "loading saved state from file %s" path; + match get_local_file path with + | Error e -> fmt_error "error while getting local file: %s\nfailed to get state from file %s" e path + | Ok file -> + let state = State_j.state_of_string file in + Ok { ctx with state } + end + else Ok ctx -let print_config ctx = - let cfg = get_config_exn ctx in +let print_config ctx repo_url = + let cfg = State.find_repo_config_exn ctx.state repo_url in let secrets = get_secrets_exn ctx in - log#info "using prefix routing:"; + let hook_token = gh_hook_token_of_secrets secrets repo_url in + let token = gh_token_of_secrets secrets repo_url in + Stdio.print_endline "using prefix routing:"; Rule.Prefix.print_prefix_routing cfg.prefix_rules.rules; - log#info "using label routing:"; + Stdio.print_endline "using label routing:"; Rule.Label.print_label_routing cfg.label_rules.rules; - log#info "signature checking %s" (if Option.is_some secrets.gh_hook_token then "enabled" else "disabled") + Stdio.printf "signature checking %s\n" (if Option.is_some hook_token then "enabled" else "disabled"); + Stdio.printf "personal access token %s\n" (if Option.is_some token then "enabled" else "disabled") diff --git a/lib/dune b/lib/dune index 733a68e1..c1103bd9 100644 --- a/lib/dune +++ b/lib/dune @@ -6,6 +6,18 @@ (preprocess (pps lwt_ppx))) +(rule + (targets common_t.ml common_t.mli) + (deps common.atd) + (action + (run atdgen -t %{deps}))) + +(rule + (targets common_j.ml common_j.mli) + (deps common.atd) + (action + (run atdgen -j -j-std %{deps}))) + (rule (targets github_t.ml github_t.mli) (deps github.atd) diff --git a/lib/github.ml b/lib/github.ml index e969f6e5..0592be34 100644 --- a/lib/github.ml +++ b/lib/github.ml @@ -64,16 +64,16 @@ let is_valid_signature ~secret headers_sig body = let (`Hex request_hash) = Hex.of_string request_hash in String.equal headers_sig (sprintf "sha1=%s" request_hash) +let validate_signature ?signing_key ~headers body = + match signing_key with + | None -> Ok () + | Some secret -> + match List.Assoc.find headers "x-hub-signature" ~equal:String.equal with + | None -> Error "unable to find header x-hub-signature" + | Some signature -> if is_valid_signature ~secret signature body then Ok () else Error "signatures don't match" + (* Parse a payload. The type of the payload is detected from the headers. *) -let parse_exn ~secret headers body = - begin - match secret with - | None -> () - | Some secret -> - match List.Assoc.find headers "x-hub-signature" ~equal:String.equal with - | None -> Exn.fail "unable to find header x-hub-signature" - | Some req_sig -> if not @@ is_valid_signature ~secret req_sig body then failwith "request signature invalid" - end; +let parse_exn headers body = match List.Assoc.find_exn headers "x-github-event" ~equal:String.equal with | exception exn -> Exn.fail ~exn "unable to read x-github-event" | "push" -> Push (commit_pushed_notification_of_string body) diff --git a/lib/state.atd b/lib/state.atd index 66338a56..cf3d82a6 100644 --- a/lib/state.atd +++ b/lib/state.atd @@ -1,8 +1,7 @@ +type config = abstract type status_state = abstract - -type 'v map_as_object = - (string * 'v) list - wrap +type 'v map_as_object = abstract +type 'v table_as_object = abstract (* A map from branch names to build statuses *) type branch_statuses = status_state map_as_object @@ -12,7 +11,11 @@ type branch_statuses = status_state map_as_object branch *) type pipeline_statuses = branch_statuses map_as_object -(* The serializable runtime state of the bot *) -type state = { +(* The runtime state of a given GitHub repository *) +type repo_state = { + ?config : config option; pipeline_statuses : pipeline_statuses -} \ No newline at end of file +} + +(* The serializable runtime state of the bot *) +type state = repo_state table_as_object diff --git a/lib/state.ml b/lib/state.ml index dcc7afe6..36a6093e 100644 --- a/lib/state.ml +++ b/lib/state.ml @@ -2,15 +2,53 @@ open Base open Common open Devkit -let empty : State_t.state = { pipeline_statuses = StringMap.empty } +exception State_error of string -let refresh_pipeline_status (state : State_t.state) ~pipeline ~(branches : Github_t.branch list) ~status = - let update_pipeline_status branch_statuses = +let state_error fmt = Printf.ksprintf (fun msg -> raise (State_error msg)) fmt + +let empty_repo_state () : State_t.repo_state = { pipeline_statuses = StringMap.empty; config = None } + +let empty () : State_t.state = Hashtbl.create (module String) + +let find_or_add_repo (state : State_t.state) repo_url = Hashtbl.find_or_add state repo_url ~default:empty_repo_state + +let find_repo_exn (state : State_t.state) repo_url = + match Hashtbl.find state repo_url with + | None -> state_error "state uninitialized for repo %s" repo_url + | Some repo_state -> repo_state + +let find_repo_config_exn state repo_url = + match (find_repo_exn state repo_url).config with + | None -> state_error "config uninitialized for repo %s" repo_url + | Some config -> config + +let set_repo_config (state : State_t.state) ~repo_url ~config = + match Hashtbl.find state repo_url with + | None -> state_error "state uninitialized for repo %s" repo_url + | Some repo_state -> repo_state.config <- Some config + +let set_repo_pipeline_status (state : State_t.state) repo_url ~pipeline ~(branches : Github_t.branch list) ~status = + let set_branch_status branch_statuses = let new_statuses = List.map branches ~f:(fun b -> b.name, status) in let init = Option.value branch_statuses ~default:(Map.empty (module String)) in List.fold_left new_statuses ~init ~f:(fun m (key, data) -> Map.set m ~key ~data) in - state.pipeline_statuses <- Map.update state.pipeline_statuses pipeline ~f:update_pipeline_status + match Hashtbl.find state repo_url with + | None -> state_error "state uninitialized for repo %s" repo_url + | Some repo_state -> + repo_state.pipeline_statuses <- Map.update repo_state.pipeline_statuses pipeline ~f:set_branch_status + +(** `is_pipeline_allowed s r p` returns `true` if + `status_rules` doesn't define a whitelist of allowed + pipelines in the config of repo `r`, or if the list + contains pipeline `p`; returns `false` otherwise. *) +let is_pipeline_allowed (state : State_t.state) repo_url ~pipeline = + match (find_repo_exn state repo_url).config with + | None -> false + | Some config -> + match config.status_rules.allowed_pipelines with + | Some allowed_pipelines when not @@ List.exists allowed_pipelines ~f:(String.equal pipeline) -> false + | _ -> true let log = Log.from "state" diff --git a/test/notabot.json b/mock_config/notabot.json similarity index 100% rename from test/notabot.json rename to mock_config/notabot.json diff --git a/mock_config/push.multi_repo_custom_repo_secrets.json b/mock_config/push.multi_repo_custom_repo_secrets.json new file mode 100644 index 00000000..338e4f17 --- /dev/null +++ b/mock_config/push.multi_repo_custom_repo_secrets.json @@ -0,0 +1,14 @@ +{ + "main_branch_name": "develop", + "status_rules": { + "rules": [] + }, + "prefix_rules": { + "default_channel": "default", + "rules": [{ "channel": "all-push-events" }] + }, + "label_rules": { + "default_channel": "default", + "rules": [] + } +} diff --git a/mock_payloads/push.multi_repo_custom_repo_secrets.json b/mock_payloads/push.multi_repo_custom_repo_secrets.json new file mode 100644 index 00000000..6332204d --- /dev/null +++ b/mock_payloads/push.multi_repo_custom_repo_secrets.json @@ -0,0 +1,113 @@ +{ + "ref": "refs/heads/master", + "before": "fb245e2a6d52d10025c8bd4f36f6e3134d85ae18", + "after": "e2173f38ae43865433a182c1fc1b5442d9763b54", + "created": false, + "deleted": false, + "forced": false, + "base_ref": null, + "compare": "https://github.com/acme/runner/compare/fb245e2a6d52...e2173f38ae43", + "commits": [ + { + "id": "80452f696a8988e7234063d47720a62e902f2afc", + "tree_id": "bf7530bb0b13d0b9f471ff5d1951c59488b5704c", + "distinct": true, + "message": "Update readme", + "timestamp": "2019-07-06T11:47:56+01:00", + "url": "https://github.com/acme/runner/commit/80452f696a8988e7234063d47720a62e902f2afc", + "author": { + "name": "wile e coyote", + "email": "wile.e@coyote.com", + "username": "wileecoyote" + }, + "committer": { + "name": "wile e coyote", + "email": "wile.e@coyote.com", + "username": "wileecoyote" + }, + "added": [], + "removed": [], + "modified": [ + "README.md" + ] + }, + { + "id": "e2173f38ae43865433a182c1fc1b5442d9763b54", + "tree_id": "6dc065fdfacfc72cd7b1f5e7a9391c238e4fb74e", + "distinct": true, + "message": "Add TESTING.md", + "timestamp": "2019-07-06T11:48:38+01:00", + "url": "https://github.com/acme/runner/commit/e2173f38ae43865433a182c1fc1b5442d9763b54", + "author": { + "name": "wile e coyote", + "email": "wile.e@coyote.com", + "username": "wileecoyote" + }, + "committer": { + "name": "wile e coyote", + "email": "wile.e@coyote.com", + "username": "wileecoyote" + }, + "added": [ + "TESTING.md", + "backend/api/longest/test.ml" + ], + "removed": [], + "modified": [] + } + ], + "head_commit": { + "id": "e2173f38ae43865433a182c1fc1b5442d9763b54", + "tree_id": "6dc065fdfacfc72cd7b1f5e7a9391c238e4fb74e", + "distinct": true, + "message": "Add TESTING.md", + "timestamp": "2019-07-06T11:48:38+01:00", + "url": "https://github.com/acme/runner/commit/e2173f38ae43865433a182c1fc1b5442d9763b54", + "author": { + "name": "wile e coyote", + "email": "wile.e@coyote.com", + "username": "wileecoyote" + }, + "committer": { + "name": "wile e coyote", + "email": "wile.e@coyote.com", + "username": "wileecoyote" + }, + "added": [ + "TESTING.md" + ], + "removed": [], + "modified": [] + }, + "pusher": { + "name": "wileecoyote", + "email": "wile.e@coyote.com" + }, + "sender": { + "login": "wileecoyote", + "id": 8755205, + "node_id": "MDQ6VXNlcjg3NTUyMDU=", + "avatar_url": "https://avatars1.githubusercontent.com/u/8755205?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/wileecoyote", + "html_url": "https://github.com/wileecoyote", + "followers_url": "https://api.github.com/users/wileecoyote/followers", + "following_url": "https://api.github.com/users/wileecoyote/following{/other_user}", + "gists_url": "https://api.github.com/users/wileecoyote/gists{/gist_id}", + "starred_url": "https://api.github.com/users/wileecoyote/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/wileecoyote/subscriptions", + "organizations_url": "https://api.github.com/users/wileecoyote/orgs", + "repos_url": "https://api.github.com/users/wileecoyote/repos", + "events_url": "https://api.github.com/users/wileecoyote/events{/privacy}", + "received_events_url": "https://api.github.com/users/wileecoyote/received_events", + "type": "User", + "site_admin": false + }, + "repository": { + "name": "runner", + "full_name": "acme/runner", + "html_url": "https://github.com/acme/runner", + "contents_url": "https://api.github.com/repos/acme/runner/contents/{+path}", + "commits_url": "https://api.github.com/repos/acme/runner/commits{/sha}" + } +} diff --git a/mock_payloads/status.multi_repo_disallowed_repo.json b/mock_payloads/status.multi_repo_disallowed_repo.json new file mode 100644 index 00000000..7e2ecc14 --- /dev/null +++ b/mock_payloads/status.multi_repo_disallowed_repo.json @@ -0,0 +1,84 @@ +{ + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "context": "buildkite/notabot-test/build", + "state": "pending", + "commit": { + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "node_id": "MDY6Q29tbWl0MTgyOjBkOTUzMDJhZGRkNjZjMTgxNmJjZTFiMWQ0OTVlZDFjOTNjY2Q0Nzg=", + "commit": { + "author": { + "name": "Louis Roché", + "email": "louis.roche@ahrefs.com", + "date": "2020-06-02T03:14:51Z" + }, + "committer": { + "name": "GitHub Enterprise", + "email": "git@ahrefs.com", + "date": "2020-06-02T03:14:51Z" + }, + "message": "Update README.md", + "tree": { + "sha": "ee5c539cad37c77348ce7a55756acc542b41cfc7", + "url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/git/trees/ee5c539cad37c77348ce7a55756acc542b41cfc7" + }, + "url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/git/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "comment_count": 0, + "verification": { + "verified": false, + "reason": "unsigned", + "signature": null, + "payload": null + } + }, + "url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "html_url": "https://git.ahrefs.com/ahrefs/notabot_test/commit/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "comments_url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478/comments", + "author": { + "login": "louis", + "id": 34, + "node_id": "MDQ6VXNlcjM0", + "avatar_url": "https://git.ahrefs.com/avatars/u/34?", + "gravatar_id": "", + "url": "https://git.ahrefs.com/api/v3/users/louis", + "html_url": "https://git.ahrefs.com/louis", + "followers_url": "https://git.ahrefs.com/api/v3/users/louis/followers", + "following_url": "https://git.ahrefs.com/api/v3/users/louis/following{/other_user}", + "gists_url": "https://git.ahrefs.com/api/v3/users/louis/gists{/gist_id}", + "starred_url": "https://git.ahrefs.com/api/v3/users/louis/starred{/owner}{/repo}", + "subscriptions_url": "https://git.ahrefs.com/api/v3/users/louis/subscriptions", + "organizations_url": "https://git.ahrefs.com/api/v3/users/louis/orgs", + "repos_url": "https://git.ahrefs.com/api/v3/users/louis/repos", + "events_url": "https://git.ahrefs.com/api/v3/users/louis/events{/privacy}", + "received_events_url": "https://git.ahrefs.com/api/v3/users/louis/received_events", + "type": "User", + "site_admin": false + }, + "committer": null, + "parents": [ + { + "sha": "04cb72d6dc8d92131282a7eff57f6caf632f0a39", + "url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/commits/04cb72d6dc8d92131282a7eff57f6caf632f0a39", + "html_url": "https://git.ahrefs.com/ahrefs/notabot_test/commit/04cb72d6dc8d92131282a7eff57f6caf632f0a39" + } + ] + }, + "branches": [ + { + "name": "master", + "commit": { + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478" + }, + "protected": false + } + ], + "created_at": "2020-06-02T03:20:35+00:00", + "updated_at": "2020-06-02T03:20:35+00:00", + "repository": { + "name": "runner", + "full_name": "acme/runner", + "html_url": "https://github.com/acme/runner", + "contents_url": "https://api.github.com/repos/acme/runner/contents/{+path}", + "commits_url": "https://api.github.com/repos/acme/runner/commits{/sha}" + } +} diff --git a/mock_payloads/status.multi_repo_independent_status_state.json b/mock_payloads/status.multi_repo_independent_status_state.json new file mode 100644 index 00000000..4f12adfc --- /dev/null +++ b/mock_payloads/status.multi_repo_independent_status_state.json @@ -0,0 +1,97 @@ +{ + "id": 1437427, + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "name": "ahrefs/notabot_test", + "target_url": "https://buildkite.com/ahrefs/notabot-test/builds/2", + "avatar_url": "https://git.ahrefs.com/avatars/oa/6?", + "context": "buildkite/notabot-test", + "description": "Build #2 passed (5 minutes, 19 seconds)", + "state": "success", + "commit": { + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "node_id": "MDY6Q29tbWl0MTgyOjBkOTUzMDJhZGRkNjZjMTgxNmJjZTFiMWQ0OTVlZDFjOTNjY2Q0Nzg=", + "commit": { + "author": { + "name": "Louis Roché", + "email": "louis.roche@ahrefs.com", + "date": "2020-06-02T03:14:51Z" + }, + "committer": { + "name": "GitHub Enterprise", + "email": "git@ahrefs.com", + "date": "2020-06-02T03:14:51Z" + }, + "message": "Update README.md", + "tree": { + "sha": "ee5c539cad37c77348ce7a55756acc542b41cfc7", + "url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/git/trees/ee5c539cad37c77348ce7a55756acc542b41cfc7" + }, + "url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/git/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "comment_count": 0, + "verification": { + "verified": false, + "reason": "unsigned", + "signature": null, + "payload": null + } + }, + "url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "html_url": "https://git.ahrefs.com/ahrefs/notabot_test/commit/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "comments_url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478/comments", + "author": { + "login": "louis", + "id": 34, + "node_id": "MDQ6VXNlcjM0", + "avatar_url": "https://git.ahrefs.com/avatars/u/34?", + "gravatar_id": "", + "url": "https://git.ahrefs.com/api/v3/users/louis", + "html_url": "https://git.ahrefs.com/louis", + "followers_url": "https://git.ahrefs.com/api/v3/users/louis/followers", + "following_url": "https://git.ahrefs.com/api/v3/users/louis/following{/other_user}", + "gists_url": "https://git.ahrefs.com/api/v3/users/louis/gists{/gist_id}", + "starred_url": "https://git.ahrefs.com/api/v3/users/louis/starred{/owner}{/repo}", + "subscriptions_url": "https://git.ahrefs.com/api/v3/users/louis/subscriptions", + "organizations_url": "https://git.ahrefs.com/api/v3/users/louis/orgs", + "repos_url": "https://git.ahrefs.com/api/v3/users/louis/repos", + "events_url": "https://git.ahrefs.com/api/v3/users/louis/events{/privacy}", + "received_events_url": "https://git.ahrefs.com/api/v3/users/louis/received_events", + "type": "User", + "site_admin": false + }, + "committer": null, + "parents": [ + { + "sha": "04cb72d6dc8d92131282a7eff57f6caf632f0a39", + "url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/commits/04cb72d6dc8d92131282a7eff57f6caf632f0a39", + "html_url": "https://git.ahrefs.com/ahrefs/notabot_test/commit/04cb72d6dc8d92131282a7eff57f6caf632f0a39" + } + ] + }, + "branches": [ + { + "name": "master", + "commit": { + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478" + }, + "protected": false + } + ], + "created_at": "2020-06-02T03:21:39+00:00", + "updated_at": "2020-06-02T03:21:39+00:00", + "repository": { + "id": 182, + "node_id": "MDEwOlJlcG9zaXRvcnkxODI=", + "name": "notabot_test", + "full_name": "ahrefs/notabot_test", + "private": true, + "html_url": "https://git.ahrefs.com/ahrefs/notabot_test", + "description": null, + "fork": false, + "commits_url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/commits{/sha}", + "contents_url": "https://git.ahrefs.com/api/v3/repos/ahrefs/notabot_test/contents/{+path}", + "created_at": "2020-06-01T18:44:17Z", + "updated_at": "2020-06-02T03:14:53Z", + "pushed_at": "2020-06-02T03:14:51Z" + } +} \ No newline at end of file diff --git a/mock_secrets/push.multi_repo_custom_repo_secrets.json b/mock_secrets/push.multi_repo_custom_repo_secrets.json new file mode 100644 index 00000000..19fd21ce --- /dev/null +++ b/mock_secrets/push.multi_repo_custom_repo_secrets.json @@ -0,0 +1,17 @@ +{ + "repositories": { + "https://github.com/acme/runner": { + "gh_token": "XXX" + }, + "https://git.ahrefs.com/ahrefs/notabot_test": { + "gh_token": "XXX", + "gh_hook_token": "XXX" + } + }, + "slack_hooks": [ + { + "url": "https://slack_webhook_url", + "channel": "all-push-events" + } + ] +} diff --git a/test/secrets.json b/mock_secrets/secrets.json similarity index 100% rename from test/secrets.json rename to mock_secrets/secrets.json diff --git a/mock_secrets/status.multi_repo_disallowed_repo.json b/mock_secrets/status.multi_repo_disallowed_repo.json new file mode 100644 index 00000000..8eaa981a --- /dev/null +++ b/mock_secrets/status.multi_repo_disallowed_repo.json @@ -0,0 +1,4 @@ +{ + "allowed_repositories": ["https://git.ahrefs.com/ahrefs/notabot_test"], + "slack_hooks": [] +} diff --git a/mock_states/status.multi_repo_independent_status_state.json b/mock_states/status.multi_repo_independent_status_state.json new file mode 100644 index 00000000..20b3c0bb --- /dev/null +++ b/mock_states/status.multi_repo_independent_status_state.json @@ -0,0 +1,16 @@ +{ + "https://github.com/acme/runner": { + "pipeline_statuses": { + "buildkite/notabot-test": { + "master": "success" + } + } + }, + "https://git.ahrefs.com/ahrefs/notabot_test": { + "pipeline_statuses": { + "buildkite/notabot-test": { + "master": "failure" + } + } + } +} diff --git a/mock_states/status.state_hide_success_test.json b/mock_states/status.state_hide_success_test.json index 6d188219..ffd4380f 100644 --- a/mock_states/status.state_hide_success_test.json +++ b/mock_states/status.state_hide_success_test.json @@ -1,11 +1,12 @@ - { - "pipeline_statuses": { - "default": { - "master": "failure" - }, - "buildkite/notabot-test": { - "master": "success" + "https://git.ahrefs.com/ahrefs/notabot_test": { + "pipeline_statuses": { + "default": { + "master": "failure" + }, + "buildkite/notabot-test": { + "master": "success" + } } } } \ No newline at end of file diff --git a/mock_states/status.state_hide_success_test_disallowed_pipeline.json b/mock_states/status.state_hide_success_test_disallowed_pipeline.json index b97c98e8..28ebd75f 100644 --- a/mock_states/status.state_hide_success_test_disallowed_pipeline.json +++ b/mock_states/status.state_hide_success_test_disallowed_pipeline.json @@ -1,7 +1,9 @@ { - "pipeline_statuses": { - "buildkite/notabot-test": { - "master": "failure" + "https://git.ahrefs.com/ahrefs/notabot_test": { + "pipeline_statuses": { + "buildkite/notabot-test": { + "master": "failure" + } } } } \ No newline at end of file diff --git a/src/notabot.ml b/src/notabot.ml index e66a146c..b4a5c2b2 100644 --- a/src/notabot.ml +++ b/src/notabot.ml @@ -21,7 +21,7 @@ let http_server_action addr port config secrets state = (** In check mode, instead of actually sending the message to slack, we simply print it in the console *) let check_gh_action file json config secrets state = - match Github.event_of_filename file with + match Github.event_of_filename (Caml.Filename.basename file) with | None -> log#error "aborting because payload %s is not named properly, named should be KIND.NAME_OF_PAYLOAD.json" file | Some kind -> @@ -30,14 +30,18 @@ let check_gh_action file json config secrets state = | Ok body -> let headers = [ "x-github-event", kind ] in let ctx = Context.make ~config_filename:config ~secrets_filepath:secrets ?state_filepath:state () in - Lwt_main.run - ( if json then - let module Action = Action.Action (Api_remote.Github) (Api_local.Slack_json) in - Action.process_github_notification ctx headers body - else - let module Action = Action.Action (Api_remote.Github) (Api_local.Slack_simple) in - Action.process_github_notification ctx headers body - ) + ( match Context.refresh_secrets ctx with + | Error e -> log#error "%s" e + | Ok ctx -> + Lwt_main.run + ( if json then + let module Action = Action.Action (Api_remote.Github) (Api_local.Slack_json) in + Action.process_github_notification ctx headers body + else + let module Action = Action.Action (Api_remote.Github) (Api_local.Slack_simple) in + Action.process_github_notification ctx headers body + ) + ) let check_slack_action url file = let data = Stdio.In_channel.read_all file in @@ -73,7 +77,7 @@ let secrets = let state = let doc = "state file" in - Arg.(value & opt (some file) None & info [ "state" ] ~docv:"STATE" ~doc) + Arg.(value & opt (some string) None & info [ "state" ] ~docv:"STATE" ~doc) let gh_payload = let doc = "JSON file containing a github webhook payload" in diff --git a/test/dune b/test/dune index 272bc3e0..b8bc0cd0 100644 --- a/test/dune +++ b/test/dune @@ -9,9 +9,9 @@ (deps (source_tree ../mock_states) (source_tree ../mock_payloads) - (source_tree github-api-cache) - notabot.json - secrets.json) + (source_tree ../mock_config) + (source_tree ../mock_secrets) + (source_tree github-api-cache)) (action (with-stdout-to slack_payloads.out diff --git a/test/slack_payloads.expected b/test/slack_payloads.expected index b641af69..d77ff6c7 100644 --- a/test/slack_payloads.expected +++ b/test/slack_payloads.expected @@ -405,6 +405,30 @@ will notify #backend } ===== file ../mock_payloads/pull_request_review_comment.deleted.json ===== ===== file ../mock_payloads/pull_request_review_comment.edited.json ===== +===== file ../mock_payloads/push.multi_repo_custom_repo_secrets.json ===== +using prefix routing: + any -> #all-push-events +using label routing: +signature checking disabled +personal access token enabled +will notify #all-push-events +{ + "text": + " pushed by wileecoyote", + "attachments": [ + { + "fallback": "Commit pushed notification", + "mrkdwn_in": [ "fields" ], + "color": "#ccc", + "fields": [ + { + "value": + "`` Update readme - wile e coyote\n`` Add TESTING.md - wile e coyote" + } + ] + } + ] +} ===== file ../mock_payloads/push.two_commits.json ===== will notify #all-push-events { @@ -484,6 +508,28 @@ will notify #default ] } ===== file ../mock_payloads/status.merge_develop.json ===== +===== file ../mock_payloads/status.multi_repo_disallowed_repo.json ===== +===== file ../mock_payloads/status.multi_repo_independent_status_state.json ===== +will notify #default +{ + "attachments": [ + { + "fallback": + " CI Build Status notification for : success", + "mrkdwn_in": [ "fields", "text" ], + "color": "good", + "pretext": + " CI Build Status notification for : success", + "text": "*Description*: Build #2 passed (5 minutes, 19 seconds).", + "fields": [ + { + "value": + "*Commit*: `` Update README.md - louis\n*Branch*: master" + } + ] + } + ] +} ===== file ../mock_payloads/status.pending_test.json ===== ===== file ../mock_payloads/status.state_hide_success_test.json ===== ===== file ../mock_payloads/status.state_hide_success_test_disallowed_pipeline.json ===== diff --git a/test/test.ml b/test/test.ml index d463b9cd..8a541b44 100644 --- a/test/test.ml +++ b/test/test.ml @@ -1,12 +1,19 @@ open Base open Lib +open Common let log = Devkit.Log.from "test" +let () = Devkit.Log.set_loglevels "error" + let mock_payload_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_payloads" let mock_state_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_states" +let mock_secrets_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_secrets" + +let mock_config_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_config" + module Action_local = Action.Action (Api_local.Github) (Api_local.Slack) let get_mock_payloads () = @@ -16,46 +23,46 @@ let get_mock_payloads () = |> List.filter_map ~f:(fun fn -> Github.event_of_filename fn |> Option.map ~f:(fun kind -> kind, fn)) |> List.map ~f:(fun (kind, fn) -> let payload_path = Caml.Filename.concat mock_payload_dir fn in - let state_path = Caml.Filename.concat mock_state_dir fn in - if Caml.Sys.file_exists state_path then kind, payload_path, Some state_path else kind, payload_path, None) - -let process ~(ctx : Context.t) (kind, path, state_path) = - let%lwt ctx = - match state_path with - | None -> Lwt.return ctx - | Some state_path -> - match Common.get_local_file state_path with - | Error e -> - log#error "failed to read %s: %s" state_path e; - Lwt.return ctx - | Ok file -> - let state = State_j.state_of_string file in - Lwt.return { ctx with state } + let state_filepath = + let path = Caml.Filename.concat mock_state_dir fn in + if Caml.Sys.file_exists path then Some path else None + in + let secrets_filepath = + let path = Caml.Filename.concat mock_secrets_dir fn in + if Caml.Sys.file_exists path then Some path else None + in + let config_filename = + let path = Caml.Filename.concat mock_config_dir fn in + if Caml.Sys.file_exists path then Some fn else None + in + kind, payload_path, state_filepath, secrets_filepath, config_filename) + +let process (kind, path, state_filepath, secrets_filepath, config_filename) = + let make_test_context () = + let verbose = Option.is_some secrets_filepath || Option.is_some config_filename in + let secrets_filepath = + Option.value ~default:(Caml.Filename.concat mock_secrets_dir Context.default.secrets_filepath) secrets_filepath + in + let config_filename = Option.value ~default:Context.default.config_filename config_filename in + let ctx = Context.make ~config_filename ~secrets_filepath ?state_filepath ~verbose () in + match Context.refresh_state ctx with + | Error e -> fmt_error "failed to read state: %s" e + | Ok ctx -> + match Context.refresh_secrets ctx with + | Error e -> fmt_error "failed to read secrets: %s" e + | Ok ctx -> Ok ctx in Stdio.printf "===== file %s =====\n" path; let headers = [ "x-github-event", kind ] in - match Common.get_local_file path with + match get_local_file path with | Error e -> Lwt.return @@ log#error "failed to read %s: %s" path e | Ok event -> + match make_test_context () with + | Error e -> Lwt.return @@ log#error "%s" e + | Ok ctx -> let%lwt _ctx = Action_local.process_github_notification ctx headers event in Lwt.return_unit let () = let payloads = get_mock_payloads () in - let repo : Github_t.repository = { name = ""; full_name = ""; url = ""; commits_url = ""; contents_url = "" } in - let ctx = Context.make ~state_filepath:"state.json" () in - Lwt_main.run - ( match%lwt Api_local.Github.get_config ~ctx ~repo with - | Error e -> - log#error "%s" e; - Lwt.return_unit - | Ok config -> - let ctx = { ctx with config = Some config } in - ( match Context.refresh_secrets ctx with - | Ok ctx -> Lwt_list.iter_s (process ~ctx) payloads - | Error e -> - log#error "failed to read secrets:"; - log#error "%s" e; - Lwt.return_unit - ) - ) + Lwt_main.run (Lwt_list.iter_s process payloads)