From 80be22dbc566df78e3f81ec785a0f4b64fa4f5cc Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 08:19:47 +0200 Subject: [PATCH 01/12] httpchecker: pass host header to http request The Go standard library net/http.Request struct exposes a special header, which takes effect when issuing http requests. The Host header isn't honored, so I set this Host field from a Host header if present. Closes #79 --- httpchecker.go | 4 ++++ httpchecker_test.go | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/httpchecker.go b/httpchecker.go index 026195d..8e81a96 100644 --- a/httpchecker.go +++ b/httpchecker.go @@ -88,6 +88,10 @@ func (c HTTPChecker) Check() (Result, error) { if c.Headers != nil { for key, header := range c.Headers { req.Header.Add(key, strings.Join(header, ", ")) + // net/http has special Host field which we'll fill out + if strings.ToLower(key) == "host" { + req.Host = header[0] + } } } diff --git a/httpchecker_test.go b/httpchecker_test.go index b7723a5..fa328f1 100644 --- a/httpchecker_test.go +++ b/httpchecker_test.go @@ -11,7 +11,7 @@ import ( func TestHTTPChecker(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("X-Checkup", r.Header.Get("X-Checkup")) - fmt.Fprintln(w, "I'm up") + fmt.Fprintln(w, "I'm up", "@"+r.Host) })) endpt := "http://" + srv.Listener.Addr().String() hc := HTTPChecker{Name: "Test", URL: endpt, Attempts: 2} @@ -107,6 +107,7 @@ func TestHTTPChecker(t *testing.T) { hc.Headers = http.Header{ "X-Checkup": []string{"Echo"}, } + hc.MustNotContain = "" hc.MustContain = "Echo" hc.ThresholdRTT = 0 result, err = hc.Check() @@ -117,6 +118,22 @@ func TestHTTPChecker(t *testing.T) { t.Errorf("Expected result.Down=%v, got %v", want, got) } + // Test with a Host header + hc.Headers = http.Header{ + "Host": []string{"http.check.local"}, + } + hc.MustContain = "@http.check.local" + hc.MustNotContain = "" + hc.ThresholdRTT = 0 + result, err = hc.Check() + + if err != nil { + t.Errorf("Didn't expect an error: %v", err) + } + if got, want := result.Down, false; got != want { + t.Errorf("Expected result.Down=%v, got %v", want, got) + } + // Try when the server is not even online srv.Listener.Close() result, err = hc.Check() From ba57107fde9c54c2ea619f40e5f35bc7bc20a86d Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 10:59:21 +0200 Subject: [PATCH 02/12] cleanup: A larger rework/cleanup - separated types into submodule, - defined interfaces into separate go file, - defined error messages in errors.go, - split checks into /check/[name] packages, - split storage into /storage/[name] packages, - split notifiers into /notifier/[name] packages, - test cleanups, move test data into relevant packages No new features have been added, the code base has just been cleaned up to be more readable, and that individual subpackages contain their implementations. With this change, new notifiers and checks can be added, without enroaching into the main checkup package. --- Makefile | 2 +- check.go | 50 +++ dnschecker.go => check/dns/dns.go | 28 +- dnschecker_test.go => check/dns/dns_test.go | 10 +- execchecker.go => check/exec/exec.go | 35 +- .../exec/exec_test.go | 14 +- {testdata => check/exec/testdata}/exec.sh | 0 httpchecker.go => check/http/http.go | 33 +- .../http/http_test.go | 6 +- tcpchecker.go => check/tcp/tcp.go | 28 +- tcpchecker_test.go => check/tcp/tcp_test.go | 22 +- {testdata => check/tcp/testdata}/Makefile | 0 .../tcp/testdata}/client.debug.crt | 0 {testdata => check/tcp/testdata}/client.key | 0 {testdata => check/tcp/testdata}/client.pem | 0 .../tcp/testdata}/leaf.debug.crt | 0 {testdata => check/tcp/testdata}/leaf.key | 0 {testdata => check/tcp/testdata}/leaf.pem | 0 .../tcp/testdata}/root.debug.crt | 0 {testdata => check/tcp/testdata}/root.key | 0 {testdata => check/tcp/testdata}/root.pem | 0 tlschecker.go => check/tls/tls.go | 40 +- tlschecker_test.go => check/tls/tls_test.go | 6 +- check_test.go | 17 + checkup.go | 412 ++---------------- checkup_test.go | 78 ++-- cmd/provision.go | 6 +- errors.go | 7 + go.mod | 2 +- go.sum | 4 +- interfaces.go | 51 +++ notifier.go | 30 ++ slack.go => notifier/slack/slack.go | 20 +- notifier_test.go | 17 + sql_disabled.go | 13 - storage.go | 45 ++ fs.go => storage/fs/fs.go | 37 +- fs_test.go => storage/fs/fs_test.go | 10 +- storage/fs/types.go | 25 ++ github.go => storage/github/github.go | 48 +- .../github/github_test.go | 12 +- s3.go => storage/s3/s3.go | 26 +- s3_test.go => storage/s3/s3_test.go | 10 +- sql.go => storage/sql/sql.go | 29 +- storage/sql/sql_disabled.go | 21 + sql_test.go => storage/sql/sql_test.go | 8 +- storage_test.go | 17 + types/attempt.go | 18 + types/provisioner.go | 41 ++ types/result.go | 122 ++++++ types/stats.go | 15 + types/status.go | 36 ++ types/util.go | 11 + 53 files changed, 865 insertions(+), 597 deletions(-) create mode 100644 check.go rename dnschecker.go => check/dns/dns.go (81%) rename dnschecker_test.go => check/dns/dns_test.go (93%) rename execchecker.go => check/exec/exec.go (86%) rename execchecker_test.go => check/exec/exec_test.go (68%) rename {testdata => check/exec/testdata}/exec.sh (100%) rename httpchecker.go => check/http/http.go (88%) rename httpchecker_test.go => check/http/http_test.go (97%) rename tcpchecker.go => check/tcp/tcp.go (84%) rename tcpchecker_test.go => check/tcp/tcp_test.go (94%) rename {testdata => check/tcp/testdata}/Makefile (100%) rename {testdata => check/tcp/testdata}/client.debug.crt (100%) rename {testdata => check/tcp/testdata}/client.key (100%) rename {testdata => check/tcp/testdata}/client.pem (100%) rename {testdata => check/tcp/testdata}/leaf.debug.crt (100%) rename {testdata => check/tcp/testdata}/leaf.key (100%) rename {testdata => check/tcp/testdata}/leaf.pem (100%) rename {testdata => check/tcp/testdata}/root.debug.crt (100%) rename {testdata => check/tcp/testdata}/root.key (100%) rename {testdata => check/tcp/testdata}/root.pem (100%) rename tlschecker.go => check/tls/tls.go (84%) rename tlschecker_test.go => check/tls/tls_test.go (98%) create mode 100644 check_test.go create mode 100644 errors.go create mode 100644 interfaces.go create mode 100644 notifier.go rename slack.go => notifier/slack/slack.go (67%) create mode 100644 notifier_test.go delete mode 100644 sql_disabled.go create mode 100644 storage.go rename fs.go => storage/fs/fs.go (72%) rename fs_test.go => storage/fs/fs_test.go (94%) create mode 100644 storage/fs/types.go rename github.go => storage/github/github.go (86%) rename github_test.go => storage/github/github_test.go (98%) rename s3.go => storage/s3/s3.go (90%) rename s3_test.go => storage/s3/s3_test.go (96%) rename sql.go => storage/sql/sql.go (86%) create mode 100644 storage/sql/sql_disabled.go rename sql_test.go => storage/sql/sql_test.go (95%) create mode 100644 storage_test.go create mode 100644 types/attempt.go create mode 100644 types/provisioner.go create mode 100644 types/result.go create mode 100644 types/stats.go create mode 100644 types/status.go create mode 100644 types/util.go diff --git a/Makefile b/Makefile index ee1e353..f65f27e 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ build: go build -o builds/ ./cmd/... test: - go test -race -count=1 -v ./... + /root/go/bin/gotest -race -count=1 -v ./... docker: docker build --no-cache . -t $(DOCKER_IMAGE) diff --git a/check.go b/check.go new file mode 100644 index 0000000..49ce00f --- /dev/null +++ b/check.go @@ -0,0 +1,50 @@ +package checkup + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/sourcegraph/checkup/check/dns" + "github.com/sourcegraph/checkup/check/exec" + "github.com/sourcegraph/checkup/check/http" + "github.com/sourcegraph/checkup/check/tcp" + "github.com/sourcegraph/checkup/check/tls" +) + +func checkerDecode(typeName string, config json.RawMessage) (Checker, error) { + switch typeName { + case "dns": + return dns.New(config) + case "exec": + return exec.New(config) + case "http": + return http.New(config) + case "tcp": + return tcp.New(config) + case "tls": + return tls.New(config) + default: + return nil, errors.New(strings.Replace(errUnknownCheckerType, "%T", typeName, -1)) + } +} + +func checkerType(ch interface{}) (string, error) { + var typeName string + switch ch.(type) { + case dns.Checker, *dns.Checker: + typeName = "dns" + case exec.Checker, *exec.Checker: + typeName = "exec" + case http.Checker, *http.Checker: + typeName = "http" + case tcp.Checker, *tcp.Checker: + typeName = "tcp" + case tls.Checker, *tls.Checker: + typeName = "tls" + default: + return "", fmt.Errorf(errUnknownCheckerType, ch) + } + return typeName, nil +} diff --git a/dnschecker.go b/check/dns/dns.go similarity index 81% rename from dnschecker.go rename to check/dns/dns.go index 01918e5..6e76439 100644 --- a/dnschecker.go +++ b/check/dns/dns.go @@ -1,15 +1,18 @@ -package checkup +package dns import ( + "encoding/json" "fmt" "net" "time" "github.com/miekg/dns" + + "github.com/sourcegraph/checkup/types" ) -// DNSChecker implements a Checker for TCP endpoints. -type DNSChecker struct { +// Checker implements a Checker for TCP endpoints. +type Checker struct { // Name is the name of the endpoint. Name string `json:"endpoint_name"` // This is the name of the DNS server you are testing. @@ -31,21 +34,30 @@ type DNSChecker struct { Attempts int `json:"attempts,omitempty"` } +// New creates a new Checker instance based on json config +func New(config json.RawMessage) (Checker, error) { + var checker Checker + err := json.Unmarshal(config, &checker) + return checker, err +} + // Check performs checks using c according to its configuration. // An error is only returned if there is a configuration error. -func (c DNSChecker) Check() (Result, error) { +func (c Checker) Check() (types.Result, error) { if c.Attempts < 1 { c.Attempts = 1 } - result := Result{Title: c.Name, Endpoint: c.URL, Timestamp: Timestamp()} + result := types.NewResult() + result.Title = c.Name + result.Endpoint = c.URL result.Times = c.doChecks() return c.conclude(result), nil } // doChecks executes and returns each attempt. -func (c DNSChecker) doChecks() Attempts { +func (c Checker) doChecks() types.Attempts { var conn net.Conn timeout := c.Timeout @@ -53,7 +65,7 @@ func (c DNSChecker) doChecks() Attempts { timeout = 1 * time.Second } - checks := make(Attempts, c.Attempts) + checks := make(types.Attempts, c.Attempts) for i := 0; i < c.Attempts; i++ { var err error start := time.Now() @@ -86,7 +98,7 @@ func (c DNSChecker) doChecks() Attempts { // computes remaining values needed to fill out the result. // It detects degraded (high-latency) responses and makes // the conclusion about the result's status. -func (c DNSChecker) conclude(result Result) Result { +func (c Checker) conclude(result types.Result) types.Result { result.ThresholdRTT = c.ThresholdRTT // Check errors (down) diff --git a/dnschecker_test.go b/check/dns/dns_test.go similarity index 93% rename from dnschecker_test.go rename to check/dns/dns_test.go index f91907f..b9cde5c 100644 --- a/dnschecker_test.go +++ b/check/dns/dns_test.go @@ -1,4 +1,4 @@ -package checkup +package dns import ( "net" @@ -6,7 +6,7 @@ import ( "time" ) -func TestDNSChecker(t *testing.T) { +func TestChecker(t *testing.T) { // Listen on localhost, random port srv, err := net.Listen("tcp", "localhost:8382") if err != nil { @@ -28,7 +28,7 @@ func TestDNSChecker(t *testing.T) { // Should know the host:port by now endpt := srv.Addr().String() testName := "TestDNS" - hc := DNSChecker{Name: testName, URL: endpt, Attempts: 2} + hc := Checker{Name: testName, URL: endpt, Attempts: 2} // Try an up server result, err := hc.Check() @@ -102,7 +102,7 @@ func TestDNSChecker(t *testing.T) { } } -func TestDNSCheckerWithAgressiveTimeout(t *testing.T) { +func TestCheckerWithAgressiveTimeout(t *testing.T) { // Listen on localhost, random port srv, err := net.Listen("tcp", "localhost:0") if err != nil { @@ -124,7 +124,7 @@ func TestDNSCheckerWithAgressiveTimeout(t *testing.T) { // Should know the host:port by now endpt := srv.Addr().String() testName := "TestTCP" - hc := DNSChecker{Name: testName, URL: endpt, Attempts: 2, Timeout: 1 * time.Nanosecond} + hc := Checker{Name: testName, URL: endpt, Attempts: 2, Timeout: 1 * time.Nanosecond} result, err := hc.Check() if err != nil { diff --git a/execchecker.go b/check/exec/exec.go similarity index 86% rename from execchecker.go rename to check/exec/exec.go index d976a24..4a67a5f 100644 --- a/execchecker.go +++ b/check/exec/exec.go @@ -1,15 +1,18 @@ -package checkup +package exec import ( "context" + "encoding/json" "fmt" "os/exec" "strings" "time" + + "github.com/sourcegraph/checkup/types" ) -// ExecChecker implements a Checker by running programs with os.Exec. -type ExecChecker struct { +// Checker implements a Checker by running programs with os.Exec. +type Checker struct { // Name is the name of the endpoint. Name string `json:"name"` @@ -60,27 +63,31 @@ type ExecChecker struct { AttemptSpacing time.Duration `json:"attempt_spacing,omitempty"` } +// New creates a new Checker instance based on json config +func New(config json.RawMessage) (Checker, error) { + var checker Checker + err := json.Unmarshal(config, &checker) + return checker, err +} + // Check performs checks using c according to its configuration. // An error is only returned if there is a configuration error. -func (c ExecChecker) Check() (Result, error) { +func (c Checker) Check() (types.Result, error) { if c.Attempts < 1 { c.Attempts = 1 } - result := Result{ - Title: c.Name, - Endpoint: c.Command, - Timestamp: Timestamp(), - } - + result := types.NewResult() + result.Title = c.Name + result.Endpoint = c.Command result.Times = c.doChecks() return c.conclude(result), nil } // doChecks executes command and returns each attempt. -func (c ExecChecker) doChecks() Attempts { - checks := make(Attempts, c.Attempts) +func (c Checker) doChecks() types.Attempts { + checks := make(types.Attempts, c.Attempts) for i := 0; i < c.Attempts; i++ { start := time.Now() @@ -118,7 +125,7 @@ func (c ExecChecker) doChecks() Attempts { // computes remaining values needed to fill out the result. // It detects degraded (high-latency) responses and makes // the conclusion about the result's status. -func (c ExecChecker) conclude(result Result) Result { +func (c Checker) conclude(result types.Result) types.Result { result.ThresholdRTT = c.ThresholdRTT warning := c.Raise == "warn" || c.Raise == "warning" @@ -153,7 +160,7 @@ func (c ExecChecker) conclude(result Result) Result { // checkDown checks whether the endpoint is down based on resp and // the configuration of c. It returns a non-nil error if down. // Note that it does not check for degraded response. -func (c ExecChecker) checkDown(body string) error { +func (c Checker) checkDown(body string) error { // Check response body if c.MustContain == "" && c.MustNotContain == "" { return nil diff --git a/execchecker_test.go b/check/exec/exec_test.go similarity index 68% rename from execchecker_test.go rename to check/exec/exec_test.go index cb0eb59..581a74c 100644 --- a/execchecker_test.go +++ b/check/exec/exec_test.go @@ -1,14 +1,14 @@ -package checkup +package exec import ( "testing" ) -func TestExecChecker(t *testing.T) { +func TestChecker(t *testing.T) { assert := func(ok bool, format string, args ...interface{}) { if !ok { - t.Fatalf(format, args...) + t.Errorf(format, args...) } } @@ -17,23 +17,23 @@ func TestExecChecker(t *testing.T) { // check non-zero exit code { testName := "Non-zero exit" - hc := ExecChecker{Name: testName, Command: command, Arguments: []string{"1", testName}, Attempts: 2} + hc := Checker{Name: testName, Command: command, Arguments: []string{"1", testName}, Attempts: 2} result, err := hc.Check() assert(err == nil, "expected no error, got %v, %#v", err, result) - assert(result.Title == testName, "expected result.Title == %s, got %s", testName, result.Title) + assert(result.Title == testName, "expected result.Title == %s, got '%s'", testName, result.Title) assert(result.Down == true, "expected result.Down = true, got %v", result.Down) } // check zero exit code { testName := "Non-zero exit" - hc := ExecChecker{Name: testName, Command: command, Arguments: []string{"0", testName}, Attempts: 2} + hc := Checker{Name: testName, Command: command, Arguments: []string{"0", testName}, Attempts: 2} result, err := hc.Check() t.Logf("%#v", result) assert(err == nil, "expected no error, got %v, %#v", err, result) - assert(result.Title == testName, "expected result.Title == %s, got %s", testName, result.Title) + assert(result.Title == testName, "expected result.Title == %s, got '%s'", testName, result.Title) assert(result.Down == false, "expected result.Down = false, got %v", result.Down) } } diff --git a/testdata/exec.sh b/check/exec/testdata/exec.sh similarity index 100% rename from testdata/exec.sh rename to check/exec/testdata/exec.sh diff --git a/httpchecker.go b/check/http/http.go similarity index 88% rename from httpchecker.go rename to check/http/http.go index 8e81a96..8b55c93 100644 --- a/httpchecker.go +++ b/check/http/http.go @@ -1,16 +1,19 @@ -package checkup +package http import ( + "encoding/json" "fmt" "io/ioutil" "net" "net/http" "strings" "time" + + "github.com/sourcegraph/checkup/types" ) -// HTTPChecker implements a Checker for HTTP endpoints. -type HTTPChecker struct { +// Checker implements a Checker for HTTP endpoints. +type Checker struct { // Name is the name of the endpoint. Name string `json:"endpoint_name"` @@ -66,9 +69,16 @@ type HTTPChecker struct { Headers http.Header `json:"headers,omitempty"` } +// New creates a new Checker instance based on json config +func New(config json.RawMessage) (Checker, error) { + var checker Checker + err := json.Unmarshal(config, &checker) + return checker, err +} + // Check performs checks using c according to its configuration. // An error is only returned if there is a configuration error. -func (c HTTPChecker) Check() (Result, error) { +func (c Checker) Check() (types.Result, error) { if c.Attempts < 1 { c.Attempts = 1 } @@ -79,7 +89,10 @@ func (c HTTPChecker) Check() (Result, error) { c.UpStatus = http.StatusOK } - result := Result{Title: c.Name, Endpoint: c.URL, Timestamp: Timestamp()} + result := types.NewResult() + result.Title = c.Name + result.Endpoint = c.URL + req, err := http.NewRequest("GET", c.URL, nil) if err != nil { return result, err @@ -101,8 +114,8 @@ func (c HTTPChecker) Check() (Result, error) { } // doChecks executes req using c.Client and returns each attempt. -func (c HTTPChecker) doChecks(req *http.Request) Attempts { - checks := make(Attempts, c.Attempts) +func (c Checker) doChecks(req *http.Request) types.Attempts { + checks := make(types.Attempts, c.Attempts) for i := 0; i < c.Attempts; i++ { start := time.Now() resp, err := c.Client.Do(req) @@ -127,7 +140,7 @@ func (c HTTPChecker) doChecks(req *http.Request) Attempts { // computes remaining values needed to fill out the result. // It detects degraded (high-latency) responses and makes // the conclusion about the result's status. -func (c HTTPChecker) conclude(result Result) Result { +func (c Checker) conclude(result types.Result) types.Result { result.ThresholdRTT = c.ThresholdRTT // Check errors (down) @@ -155,7 +168,7 @@ func (c HTTPChecker) conclude(result Result) Result { // checkDown checks whether the endpoint is down based on resp and // the configuration of c. It returns a non-nil error if down. // Note that it does not check for degraded response. -func (c HTTPChecker) checkDown(resp *http.Response) error { +func (c Checker) checkDown(resp *http.Response) error { // Check status code if resp.StatusCode != c.UpStatus { return fmt.Errorf("response status %s", resp.Status) @@ -181,7 +194,7 @@ func (c HTTPChecker) checkDown(resp *http.Response) error { } // DefaultHTTPClient is used when no other http.Client -// is specified on a HTTPChecker. +// is specified on a Checker. var DefaultHTTPClient = &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, diff --git a/httpchecker_test.go b/check/http/http_test.go similarity index 97% rename from httpchecker_test.go rename to check/http/http_test.go index fa328f1..fcb02a9 100644 --- a/httpchecker_test.go +++ b/check/http/http_test.go @@ -1,4 +1,4 @@ -package checkup +package http import ( "fmt" @@ -8,13 +8,13 @@ import ( "time" ) -func TestHTTPChecker(t *testing.T) { +func TestChecker(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("X-Checkup", r.Header.Get("X-Checkup")) fmt.Fprintln(w, "I'm up", "@"+r.Host) })) endpt := "http://" + srv.Listener.Addr().String() - hc := HTTPChecker{Name: "Test", URL: endpt, Attempts: 2} + hc := Checker{Name: "Test", URL: endpt, Attempts: 2} // Try an up server result, err := hc.Check() diff --git a/tcpchecker.go b/check/tcp/tcp.go similarity index 84% rename from tcpchecker.go rename to check/tcp/tcp.go index 0c3c891..6c9caf1 100644 --- a/tcpchecker.go +++ b/check/tcp/tcp.go @@ -1,16 +1,19 @@ -package checkup +package tcp import ( "crypto/tls" "crypto/x509" + "encoding/json" "fmt" "io/ioutil" "net" "time" + + "github.com/sourcegraph/checkup/types" ) -// TCPChecker implements a Checker for TCP endpoints. -type TCPChecker struct { +// Checker implements a Checker for TCP endpoints. +type Checker struct { // Name is the name of the endpoint. Name string `json:"endpoint_name"` @@ -48,19 +51,28 @@ type TCPChecker struct { // Check performs checks using c according to its configuration. // An error is only returned if there is a configuration error. -func (c TCPChecker) Check() (Result, error) { +func (c Checker) Check() (types.Result, error) { if c.Attempts < 1 { c.Attempts = 1 } - result := Result{Title: c.Name, Endpoint: c.URL, Timestamp: Timestamp()} + result := types.NewResult() + result.Title = c.Name + result.Endpoint = c.URL result.Times = c.doChecks() return c.conclude(result), nil } +// New creates a new Checker instance based on json config +func New(config json.RawMessage) (Checker, error) { + var checker Checker + err := json.Unmarshal(config, &checker) + return checker, err +} + // doChecks executes and returns each attempt. -func (c TCPChecker) doChecks() Attempts { +func (c Checker) doChecks() types.Attempts { var err error var conn net.Conn @@ -69,7 +81,7 @@ func (c TCPChecker) doChecks() Attempts { timeout = 1 * time.Second } - checks := make(Attempts, c.Attempts) + checks := make(types.Attempts, c.Attempts) for i := 0; i < c.Attempts; i++ { start := time.Now() @@ -116,7 +128,7 @@ func (c TCPChecker) doChecks() Attempts { // computes remaining values needed to fill out the result. // It detects degraded (high-latency) responses and makes // the conclusion about the result's status. -func (c TCPChecker) conclude(result Result) Result { +func (c Checker) conclude(result types.Result) types.Result { result.ThresholdRTT = c.ThresholdRTT // Check errors (down) diff --git a/tcpchecker_test.go b/check/tcp/tcp_test.go similarity index 94% rename from tcpchecker_test.go rename to check/tcp/tcp_test.go index 4016b3b..bad51da 100644 --- a/tcpchecker_test.go +++ b/check/tcp/tcp_test.go @@ -1,4 +1,4 @@ -package checkup +package tcp import ( "crypto/tls" @@ -7,7 +7,7 @@ import ( "time" ) -func TestTCPChecker(t *testing.T) { +func TestChecker(t *testing.T) { // Listen on localhost, random port srv, err := net.Listen("tcp", "localhost:0") if err != nil { @@ -29,7 +29,7 @@ func TestTCPChecker(t *testing.T) { // Should know the host:port by now endpt := srv.Addr().String() testName := "TestTCP" - hc := TCPChecker{Name: testName, URL: endpt, Attempts: 2} + hc := Checker{Name: testName, URL: endpt, Attempts: 2} // Try an up server result, err := hc.Check() @@ -103,7 +103,7 @@ func TestTCPChecker(t *testing.T) { } } -func TestTCPCheckerWithAgressiveTimeout(t *testing.T) { +func TestCheckerWithAgressiveTimeout(t *testing.T) { // Listen on localhost, random port srv, err := net.Listen("tcp", "localhost:0") if err != nil { @@ -125,7 +125,7 @@ func TestTCPCheckerWithAgressiveTimeout(t *testing.T) { // Should know the host:port by now endpt := srv.Addr().String() testName := "TestTCP" - hc := TCPChecker{Name: testName, URL: endpt, Attempts: 2, Timeout: 1 * time.Nanosecond} + hc := Checker{Name: testName, URL: endpt, Attempts: 2, Timeout: 1 * time.Nanosecond} result, err := hc.Check() if err != nil { @@ -142,7 +142,7 @@ func TestTCPCheckerWithAgressiveTimeout(t *testing.T) { } } -func TestTCPCheckerWithTLSNoVerify(t *testing.T) { +func TestCheckerWithTLSNoVerify(t *testing.T) { // Listen on localhost, random port certPair, err := tls.LoadX509KeyPair("testdata/leaf.pem", "testdata/leaf.key") if err != nil { @@ -175,7 +175,7 @@ func TestTCPCheckerWithTLSNoVerify(t *testing.T) { // Should know the host:port by now endpt := srv.Addr().String() testName := "TestWithTLSNoVerify" - hc := TCPChecker{Name: testName, URL: endpt, TLSEnabled: true, TLSSkipVerify: true, Attempts: 2} + hc := Checker{Name: testName, URL: endpt, TLSEnabled: true, TLSSkipVerify: true, Attempts: 2} // Try an up server result, err := hc.Check() @@ -247,7 +247,7 @@ func TestTCPCheckerWithTLSNoVerify(t *testing.T) { } } -func TestTCPCheckerWithTLSVerifySuccess(t *testing.T) { +func TestCheckerWithTLSVerifySuccess(t *testing.T) { // Listen on localhost, random port certPair, err := tls.LoadX509KeyPair("testdata/leaf.pem", "testdata/leaf.key") if err != nil { @@ -280,7 +280,7 @@ func TestTCPCheckerWithTLSVerifySuccess(t *testing.T) { // Should know the host:port by now endpt := srv.Addr().String() testName := "TestWithTLSNoVerify" - hc := TCPChecker{Name: testName, URL: endpt, TLSEnabled: true, TLSCAFile: "testdata/root.pem", Attempts: 2} + hc := Checker{Name: testName, URL: endpt, TLSEnabled: true, TLSCAFile: "testdata/root.pem", Attempts: 2} // Try an up server result, err := hc.Check() @@ -372,7 +372,7 @@ func TestTCPCheckerWithTLSVerifySuccess(t *testing.T) { } } -func TestTCPCheckerWithTLSVerifyError(t *testing.T) { +func TestCheckerWithTLSVerifyError(t *testing.T) { // Listen on localhost, random port certPair, err := tls.LoadX509KeyPair("testdata/leaf.pem", "testdata/leaf.key") if err != nil { @@ -405,7 +405,7 @@ func TestTCPCheckerWithTLSVerifyError(t *testing.T) { // Should know the host:port by now endpt := srv.Addr().String() testName := "TestWithTLSVerifyError" - hc := TCPChecker{Name: testName, URL: endpt, TLSEnabled: true, Attempts: 2} + hc := Checker{Name: testName, URL: endpt, TLSEnabled: true, Attempts: 2} // Try an up server result, err := hc.Check() diff --git a/testdata/Makefile b/check/tcp/testdata/Makefile similarity index 100% rename from testdata/Makefile rename to check/tcp/testdata/Makefile diff --git a/testdata/client.debug.crt b/check/tcp/testdata/client.debug.crt similarity index 100% rename from testdata/client.debug.crt rename to check/tcp/testdata/client.debug.crt diff --git a/testdata/client.key b/check/tcp/testdata/client.key similarity index 100% rename from testdata/client.key rename to check/tcp/testdata/client.key diff --git a/testdata/client.pem b/check/tcp/testdata/client.pem similarity index 100% rename from testdata/client.pem rename to check/tcp/testdata/client.pem diff --git a/testdata/leaf.debug.crt b/check/tcp/testdata/leaf.debug.crt similarity index 100% rename from testdata/leaf.debug.crt rename to check/tcp/testdata/leaf.debug.crt diff --git a/testdata/leaf.key b/check/tcp/testdata/leaf.key similarity index 100% rename from testdata/leaf.key rename to check/tcp/testdata/leaf.key diff --git a/testdata/leaf.pem b/check/tcp/testdata/leaf.pem similarity index 100% rename from testdata/leaf.pem rename to check/tcp/testdata/leaf.pem diff --git a/testdata/root.debug.crt b/check/tcp/testdata/root.debug.crt similarity index 100% rename from testdata/root.debug.crt rename to check/tcp/testdata/root.debug.crt diff --git a/testdata/root.key b/check/tcp/testdata/root.key similarity index 100% rename from testdata/root.key rename to check/tcp/testdata/root.key diff --git a/testdata/root.pem b/check/tcp/testdata/root.pem similarity index 100% rename from testdata/root.pem rename to check/tcp/testdata/root.pem diff --git a/tlschecker.go b/check/tls/tls.go similarity index 84% rename from tlschecker.go rename to check/tls/tls.go index 882588b..311ce5a 100644 --- a/tlschecker.go +++ b/check/tls/tls.go @@ -1,15 +1,18 @@ -package checkup +package tls import ( "crypto/tls" "crypto/x509" + "encoding/json" "fmt" "io/ioutil" "net" "time" + + "github.com/sourcegraph/checkup/types" ) -// TLSChecker implements a Checker for TLS endpoints. +// Checker implements a Checker for TLS endpoints. // // TODO: Implement more checks on the certificate and TLS configuration. // - Cipher suites @@ -17,7 +20,7 @@ import ( // - OCSP stapling // - Multiple SNIs // - Other things that you might see at SSL Labs or other TLS health checks -type TLSChecker struct { +type Checker struct { // Name is the name of the endpoint. Name string `json:"endpoint_name"` @@ -56,9 +59,16 @@ type TLSChecker struct { tlsConfig *tls.Config } +// New creates a new Checker instance based on json config +func New(config json.RawMessage) (Checker, error) { + var checker Checker + err := json.Unmarshal(config, &checker) + return checker, err +} + // Check performs checks using c according to its configuration. // An error is only returned if there is a configuration error. -func (c TLSChecker) Check() (Result, error) { +func (c Checker) Check() (types.Result, error) { if c.Attempts < 1 { c.Attempts = 1 } @@ -76,23 +86,21 @@ func (c TLSChecker) Check() (Result, error) { for _, fname := range c.TrustedRoots { pemData, err := ioutil.ReadFile(fname) if err != nil { - return Result{}, fmt.Errorf("error loading file: %v", err) + return types.Result{}, fmt.Errorf("error loading file: %v", err) } if !c.tlsConfig.RootCAs.AppendCertsFromPEM(pemData) { - return Result{}, fmt.Errorf("error appending certs from PEM %s: %v", fname, err) + return types.Result{}, fmt.Errorf("error appending certs from PEM %s: %v", fname, err) } } } attempts, conns := c.doChecks() - result := Result{ - Title: c.Name, - Endpoint: c.URL, - Timestamp: Timestamp(), - Times: attempts, - ThresholdRTT: c.ThresholdRTT, - } + result := types.NewResult() + result.Title = c.Name + result.Endpoint = c.URL + result.Times = attempts + result.ThresholdRTT = c.ThresholdRTT return c.conclude(conns, result), nil } @@ -102,8 +110,8 @@ func (c TLSChecker) Check() (Result, error) { // will be open, so it's vital that conclude() is called, // passing in the connections, so that they will be inspected // and closed properly. -func (c TLSChecker) doChecks() (Attempts, []*tls.Conn) { - checks := make(Attempts, c.Attempts) +func (c Checker) doChecks() (types.Attempts, []*tls.Conn) { + checks := make(types.Attempts, c.Attempts) conns := make([]*tls.Conn, c.Attempts) for i := 0; i < c.Attempts; i++ { dialer := &net.Dialer{Timeout: c.Timeout} @@ -124,7 +132,7 @@ func (c TLSChecker) doChecks() (Attempts, []*tls.Conn) { // It detects less-than-ideal (degraded) connections and // marks them as such. It closes the connections that are // passed in. -func (c TLSChecker) conclude(conns []*tls.Conn, result Result) Result { +func (c Checker) conclude(conns []*tls.Conn, result types.Result) types.Result { // close all connections when done defer func() { for _, conn := range conns { diff --git a/tlschecker_test.go b/check/tls/tls_test.go similarity index 98% rename from tlschecker_test.go rename to check/tls/tls_test.go index 3f0cd95..92ecfcd 100644 --- a/tlschecker_test.go +++ b/check/tls/tls_test.go @@ -1,4 +1,4 @@ -package checkup +package tls import ( "crypto/ecdsa" @@ -14,7 +14,7 @@ import ( "time" ) -func TestTLSChecker(t *testing.T) { +func TestChecker(t *testing.T) { selfSigned, err := makeSelfSignedCert("localhost", "", time.Hour*24*30) if err != nil { t.Fatal(err) @@ -40,7 +40,7 @@ func TestTLSChecker(t *testing.T) { } }() - tc := TLSChecker{ + tc := Checker{ Name: "Test", URL: endpt, Attempts: 2, diff --git a/check_test.go b/check_test.go new file mode 100644 index 0000000..6ae5a41 --- /dev/null +++ b/check_test.go @@ -0,0 +1,17 @@ +package checkup + +import ( + "strings" + "testing" +) + +func TestUnknownCheckerType(t *testing.T) { + kind, err := checkerType("") + if got, want := kind, ""; got != want { + t.Errorf("Expected type '%s', got '%s'", want, got) + } + want := strings.Replace(errUnknownCheckerType, "%T", "string", -1) + if got := err.Error(); got != want { + t.Errorf("Expected error '%s', got '%s'", want, got) + } +} diff --git a/checkup.go b/checkup.go index bae7d41..9c8af13 100644 --- a/checkup.go +++ b/checkup.go @@ -8,12 +8,11 @@ import ( "encoding/json" "fmt" "log" - "sort" "strings" "sync" "time" - "github.com/fatih/color" + "github.com/sourcegraph/checkup/types" ) // Checkup performs a routine checkup on endpoints or @@ -50,7 +49,7 @@ type Checkup struct { // Check performs the health checks. An error is only // returned in the case of a misconfiguration or if // any one of the Checkers returns an error. -func (c Checkup) Check() ([]Result, error) { +func (c Checkup) Check() ([]types.Result, error) { if c.ConcurrentChecks == 0 { c.ConcurrentChecks = DefaultConcurrentChecks } @@ -59,7 +58,7 @@ func (c Checkup) Check() ([]Result, error) { c.ConcurrentChecks) } - results := make([]Result, len(c.Checkers)) + results := make([]types.Result, len(c.Checkers)) errs := make(Errors, len(c.Checkers)) throttle := make(chan struct{}, c.ConcurrentChecks) wg := sync.WaitGroup{} @@ -173,21 +172,12 @@ func (c Checkup) MarshalJSON() ([]byte, error) { if err != nil { return result, err } - var typeName string - switch ch.(type) { - case ExecChecker: - typeName = "exec" - case HTTPChecker: - typeName = "http" - case TCPChecker: - typeName = "tcp" - case DNSChecker: - typeName = "dns" - case TLSChecker: - typeName = "tls" - default: - return result, fmt.Errorf("unknown Checker type") + + typeName, err := checkerType(ch) + if err != nil { + return result, err } + chb = []byte(fmt.Sprintf(`{"type":"%s",%s`, typeName, string(chb[1:]))) checkers = append(checkers, chb) } @@ -204,19 +194,12 @@ func (c Checkup) MarshalJSON() ([]byte, error) { if err != nil { return result, err } - var providerName string - switch c.Storage.(type) { - case *GitHub: - providerName = "github" - case S3: - providerName = "s3" - case FS: - providerName = "fs" - case SQL: - providerName = "sql" - default: - return result, fmt.Errorf("unknown Storage type: %T", c.Storage) + + providerName, err := storageType(c.Storage) + if err != nil { + return result, err } + sb = []byte(fmt.Sprintf(`{"provider":"%s",%s`, providerName, string(sb[1:]))) wrap("storage", sb) } @@ -227,13 +210,12 @@ func (c Checkup) MarshalJSON() ([]byte, error) { if err != nil { return result, err } - var notifierName string - switch c.Notifier.(type) { - case Slack: - notifierName = "slack" - default: - return result, fmt.Errorf("unknown Notifier type") + + notifierName, err := notifierType(c.Notifier) + if err != nil { + return result, err } + nb = []byte(fmt.Sprintf(`{"name":"%s",%s`, notifierName, string(nb[1:]))) wrap("notifier", nb) } @@ -288,320 +270,34 @@ func (c *Checkup) UnmarshalJSON(b []byte) error { // Finally, we unmarshal the remaining values using type // assertions with the help of the type information for i, t := range types.Checkers { - switch t.Type { - case "exec": - var checker ExecChecker - err = json.Unmarshal(raw.Checkers[i], &checker) - if err != nil { - return err - } - c.Checkers = append(c.Checkers, checker) - case "http": - var checker HTTPChecker - err = json.Unmarshal(raw.Checkers[i], &checker) - if err != nil { - return err - } - c.Checkers = append(c.Checkers, checker) - case "tcp": - var checker TCPChecker - err = json.Unmarshal(raw.Checkers[i], &checker) - if err != nil { - return err - } - c.Checkers = append(c.Checkers, checker) - case "dns": - var checker DNSChecker - err = json.Unmarshal(raw.Checkers[i], &checker) - if err != nil { - return err - } - c.Checkers = append(c.Checkers, checker) - case "tls": - var checker TLSChecker - err = json.Unmarshal(raw.Checkers[i], &checker) - if err != nil { - return err - } - c.Checkers = append(c.Checkers, checker) - default: - return fmt.Errorf("%s: unknown Checker type", t.Type) + checker, err := checkerDecode(t.Type, raw.Checkers[i]) + if err != nil { + return err } + c.Checkers = append(c.Checkers, checker) } if raw.Storage != nil { - switch types.Storage.Provider { - case "s3": - var storage S3 - err = json.Unmarshal(raw.Storage, &storage) - if err != nil { - return err - } - c.Storage = storage - case "fs": - var storage FS - err = json.Unmarshal(raw.Storage, &storage) - if err != nil { - return err - } - c.Storage = storage - case "github": - storage := &GitHub{} - err = json.Unmarshal(raw.Storage, storage) - if err != nil { - return err - } - c.Storage = storage - case "sql": - var storage SQL - err = json.Unmarshal(raw.Storage, &storage) - if err != nil { - return err - } - c.Storage = storage - default: - return fmt.Errorf("%s: unknown Storage type", types.Storage.Provider) + storage, err := storageDecode(types.Storage.Provider, raw.Storage) + if err != nil { + return err } + c.Storage = storage } if raw.Notifier != nil { - switch types.Notifier.Name { - case "slack": - var notifier Slack - err = json.Unmarshal(raw.Notifier, ¬ifier) - if err != nil { - return err - } - c.Notifier = notifier - default: - return fmt.Errorf("%s: unknown Notifier type", types.Notifier.Name) + notifier, err := notifierDecode(types.Notifier.Name, raw.Notifier) + if err != nil { + return err } + c.Notifier = notifier } return nil } -// Checker can create a Result. -type Checker interface { - Check() (Result, error) -} - -// Storage can store results. -type Storage interface { - Store([]Result) error -} - -// StorageReader can read results from the Storage. -type StorageReader interface { - // Fetch returns the contents of a check file. - Fetch(checkFile string) ([]Result, error) - // GetIndex returns the storage index, as a map where keys are check - // result filenames and values are the associated check timestamps. - GetIndex() (map[string]int64, error) -} - -// Maintainer can maintain a store of results by -// deleting old check files that are no longer -// needed or performing other required tasks. -type Maintainer interface { - Maintain() error -} - -// Notifier can notify ops or sysadmins of -// potential problems. A Notifier should keep -// state to avoid sending repeated notices -// more often than the admin would like. -type Notifier interface { - Notify([]Result) error -} - // DefaultConcurrentChecks is how many checks, // at most, to perform concurrently. var DefaultConcurrentChecks = 5 -// FilenameFormatString is the format string used -// by GenerateFilename to create a filename. -const FilenameFormatString = "%d-check.json" - -// Timestamp returns the UTC Unix timestamp in -// nanoseconds. -func Timestamp() int64 { - return time.Now().UTC().UnixNano() -} - -// GenerateFilename returns a filename that is ideal -// for storing the results file on a storage provider -// that relies on the filename for retrieval that is -// sorted by date/timeframe. It returns a string pointer -// to be used by the AWS SDK... -func GenerateFilename() *string { - s := fmt.Sprintf(FilenameFormatString, Timestamp()) - return &s -} - -// Result is the result of a health check. -type Result struct { - // Title is the title (or name) of the thing that was checked. - // It should be unique, as it acts like an identifier to users. - Title string `json:"title,omitempty"` - - // Endpoint is the URL/address/path/identifier/locator/whatever - // of what was checked. - Endpoint string `json:"endpoint,omitempty"` - - // Timestamp is when the check occurred; UTC UnixNano format. - Timestamp int64 `json:"timestamp,omitempty"` - - // Times is a list of each individual check attempt. - Times Attempts `json:"times,omitempty"` - - // ThresholdRTT is the maximum RTT that was tolerated before - // considering performance to be degraded. Leave 0 if irrelevant. - ThresholdRTT time.Duration `json:"threshold,omitempty"` - - // Healthy, Degraded, and Down contain the ultimate conclusion - // about the endpoint. Exactly one of these should be true; - // any more or less is a bug. - Healthy bool `json:"healthy,omitempty"` - Degraded bool `json:"degraded,omitempty"` - Down bool `json:"down,omitempty"` - - // Notice contains a description of some condition of this - // check that might have affected the result in some way. - // For example, that the median RTT is above the threshold. - Notice string `json:"notice,omitempty"` - - // Message is an optional message to show on the status page. - // For example, what you're doing to fix a problem. - Message string `json:"message,omitempty"` -} - -// ComputeStats computes basic statistics about r. -func (r Result) ComputeStats() Stats { - var s Stats - - for _, a := range r.Times { - s.Total += a.RTT - if a.RTT < s.Min || s.Min == 0 { - s.Min = a.RTT - } - if a.RTT > s.Max || s.Max == 0 { - s.Max = a.RTT - } - } - sorted := make(Attempts, len(r.Times)) - copy(sorted, r.Times) - sort.Sort(sorted) - - half := len(sorted) / 2 - if len(sorted)%2 == 0 { - s.Median = (sorted[half-1].RTT + sorted[half].RTT) / 2 - } else { - s.Median = sorted[half].RTT - } - - s.Mean = time.Duration(int64(s.Total) / int64(len(r.Times))) - - return s -} - -// String returns a human-readable rendering of r. -func (r Result) String() string { - stats := r.ComputeStats() - s := fmt.Sprintf("== %s - %s\n", r.Title, r.Endpoint) - s += fmt.Sprintf(" Threshold: %s\n", r.ThresholdRTT) - s += fmt.Sprintf(" Max: %s\n", stats.Max) - s += fmt.Sprintf(" Min: %s\n", stats.Min) - s += fmt.Sprintf(" Median: %s\n", stats.Median) - s += fmt.Sprintf(" Mean: %s\n", stats.Mean) - s += fmt.Sprintf(" All: %v\n", r.Times) - statusLine := fmt.Sprintf(" Assessment: %v\n", r.Status()) - switch r.Status() { - case Healthy: - statusLine = color.GreenString(statusLine) - case Degraded: - statusLine = color.YellowString(statusLine) - case Down: - statusLine = color.RedString(statusLine) - } - s += statusLine - return s -} - -// Status returns a text representation of the overall status -// indicated in r. -func (r Result) Status() StatusText { - if r.Down { - return Down - } else if r.Degraded { - return Degraded - } else if r.Healthy { - return Healthy - } - return Unknown -} - -// DisableColor disables ANSI colors in the Result default string. -func DisableColor() { - color.NoColor = true -} - -// StatusText is the textual representation of the -// result of a status check. -type StatusText string - -// PriorityOver returns whether s has priority over other. -// For example, a Down status has priority over Degraded. -func (s StatusText) PriorityOver(other StatusText) bool { - if s == other { - return false - } - switch s { - case Down: - return true - case Degraded: - if other == Down { - return false - } - return true - case Healthy: - if other == Unknown { - return true - } - return false - } - return false -} - -// Text representations for the status of a check. -const ( - Healthy StatusText = "healthy" - Degraded StatusText = "degraded" - Down StatusText = "down" - Unknown StatusText = "unknown" -) - -// Attempt is an attempt to communicate with the endpoint. -type Attempt struct { - RTT time.Duration `json:"rtt"` - Error string `json:"error,omitempty"` -} - -// Attempts is a list of Attempt that can be sorted by RTT. -type Attempts []Attempt - -func (a Attempts) Len() int { return len(a) } -func (a Attempts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a Attempts) Less(i, j int) bool { return a[i].RTT < a[j].RTT } - -// Stats is a type that holds information about a Result, -// especially its various Attempts. -type Stats struct { - Total time.Duration `json:"total,omitempty"` - Mean time.Duration `json:"mean,omitempty"` - Median time.Duration `json:"median,omitempty"` - Min time.Duration `json:"min,omitempty"` - Max time.Duration `json:"max,omitempty"` -} - // Errors is an error type that concatenates multiple errors. type Errors []error @@ -625,51 +321,3 @@ func (e Errors) Empty() bool { } return true } - -// Provisioner is a type of storage mechanism that can -// provision itself for use with checkup. Provisioning -// need only happen once and is merely a convenience -// so that the user can get up and running with their -// status page more quickly. Presumably, the info -// returned from Provision should be used on the status -// page side of things ot access the check files (like -// a key pair that is used for read-only access). -type Provisioner interface { - Provision() (ProvisionInfo, error) -} - -// ProvisionInfo contains the results of provisioning a new -// storage facility for check files. Its values should be -// used by the status page in order to obtain read-only -// access to the check files. -type ProvisionInfo struct { - // The ID of a user that was created for accessing checks. - UserID string `json:"user_id"` - - // The username of a user that was created for accessing checks. - Username string `json:"username"` - - // The ID or name of the ID/key used to access checks. Expect - // this value to be made public. (It should have read-only - // access to the checks.) - PublicAccessKeyID string `json:"public_access_key_id"` - - // The "secret" associated with the PublicAccessKeyID, but - // expect this value to be made public. (It should provide - // read-only access to the checks.) - PublicAccessKey string `json:"public_access_key"` -} - -// String returns the information in i in a human-readable format -// along with an important notice. -func (i ProvisionInfo) String() string { - s := "Provision successful\n\n" - s += fmt.Sprintf(" User ID: %s\n", i.UserID) - s += fmt.Sprintf(" Username: %s\n", i.Username) - s += fmt.Sprintf("Public Access Key ID: %s\n", i.PublicAccessKeyID) - s += fmt.Sprintf(" Public Access Key: %s\n\n", i.PublicAccessKey) - s += `IMPORTANT: Copy the Public Access Key ID and Public Access -Key into the config.js file for your status page. You will -not be shown these credentials again.` - return s -} diff --git a/checkup_test.go b/checkup_test.go index 3b966f8..5fe3b4c 100644 --- a/checkup_test.go +++ b/checkup_test.go @@ -7,6 +7,8 @@ import ( "sync" "testing" "time" + + "github.com/sourcegraph/checkup/types" ) func TestCheckAndStore(t *testing.T) { @@ -80,7 +82,7 @@ func TestCheckAndStoreEvery(t *testing.T) { } func TestComputeStats(t *testing.T) { - s := Result{Times: []Attempt{ + s := types.Result{Times: []types.Attempt{ {RTT: 7 * time.Second}, {RTT: 4 * time.Second}, {RTT: 4 * time.Second}, @@ -107,64 +109,64 @@ func TestComputeStats(t *testing.T) { } func TestResultStatus(t *testing.T) { - r := Result{Healthy: true} - if got, want := r.Status(), Healthy; got != want { + r := types.Result{Healthy: true} + if got, want := r.Status(), types.StatusHealthy; got != want { t.Errorf("Expected status '%s' but got: '%s'", want, got) } - r = Result{Degraded: true} - if got, want := r.Status(), Degraded; got != want { + r = types.Result{Degraded: true} + if got, want := r.Status(), types.StatusDegraded; got != want { t.Errorf("Expected status '%s' but got: '%s'", want, got) } - r = Result{Down: true} - if got, want := r.Status(), Down; got != want { + r = types.Result{Down: true} + if got, want := r.Status(), types.StatusDown; got != want { t.Errorf("Expected status '%s' but got: '%s'", want, got) } - r = Result{} - if got, want := r.Status(), Unknown; got != want { + r = types.Result{} + if got, want := r.Status(), types.StatusUnknown; got != want { t.Errorf("Expected status '%s' but got: '%s'", want, got) } // These are invalid states, but we need to test anyway in case a // checker is buggy. We expect the worst of the enabled fields. - r = Result{Down: true, Degraded: true} - if got, want := r.Status(), Down; got != want { + r = types.Result{Down: true, Degraded: true} + if got, want := r.Status(), types.StatusDown; got != want { t.Errorf("(INVALID RESULT CASE) Expected status '%s' but got: '%s'", want, got) } - r = Result{Degraded: true, Healthy: true} - if got, want := r.Status(), Degraded; got != want { + r = types.Result{Degraded: true, Healthy: true} + if got, want := r.Status(), types.StatusDegraded; got != want { t.Errorf("(INVALID RESULT CASE) Expected status '%s' but got: '%s'", want, got) } - r = Result{Down: true, Healthy: true} - if got, want := r.Status(), Down; got != want { + r = types.Result{Down: true, Healthy: true} + if got, want := r.Status(), types.StatusDown; got != want { t.Errorf("(INVALID RESULT CASE) Expected status '%s' but got: '%s'", want, got) } } func TestPriorityOver(t *testing.T) { for i, test := range []struct { - status StatusText - another StatusText + status types.StatusText + another types.StatusText expected bool }{ - {Down, Down, false}, - {Down, Degraded, true}, - {Down, Healthy, true}, - {Down, Unknown, true}, - {Degraded, Down, false}, - {Degraded, Degraded, false}, - {Degraded, Healthy, true}, - {Degraded, Unknown, true}, - {Healthy, Down, false}, - {Healthy, Degraded, false}, - {Healthy, Healthy, false}, - {Healthy, Unknown, true}, - {Unknown, Down, false}, - {Unknown, Degraded, false}, - {Unknown, Healthy, false}, - {Unknown, Unknown, false}, + {types.StatusDown, types.StatusDown, false}, + {types.StatusDown, types.StatusDegraded, true}, + {types.StatusDown, types.StatusHealthy, true}, + {types.StatusDown, types.StatusUnknown, true}, + {types.StatusDegraded, types.StatusDown, false}, + {types.StatusDegraded, types.StatusDegraded, false}, + {types.StatusDegraded, types.StatusHealthy, true}, + {types.StatusDegraded, types.StatusUnknown, true}, + {types.StatusHealthy, types.StatusDown, false}, + {types.StatusHealthy, types.StatusDegraded, false}, + {types.StatusHealthy, types.StatusHealthy, false}, + {types.StatusHealthy, types.StatusUnknown, true}, + {types.StatusUnknown, types.StatusDown, false}, + {types.StatusUnknown, types.StatusDegraded, false}, + {types.StatusUnknown, types.StatusHealthy, false}, + {types.StatusUnknown, types.StatusUnknown, false}, } { actual := test.status.PriorityOver(test.another) if actual != test.expected { @@ -200,24 +202,24 @@ type fake struct { returnErr bool checked int - stored []Result + stored []types.Result maintained int notified int } -func (f *fake) Check() (Result, error) { +func (f *fake) Check() (types.Result, error) { f.Lock() defer f.Unlock() f.checked++ - r := Result{Timestamp: time.Now().UTC().UnixNano()} + r := types.Result{Timestamp: time.Now().UTC().UnixNano()} if f.returnErr { return r, errTest } return r, nil } -func (f *fake) Store(results []Result) error { +func (f *fake) Store(results []types.Result) error { f.Lock() defer f.Unlock() @@ -236,7 +238,7 @@ func (f *fake) Maintain() error { return nil } -func (f *fake) Notify(results []Result) error { +func (f *fake) Notify(results []types.Result) error { f.Lock() defer f.Unlock() diff --git a/cmd/provision.go b/cmd/provision.go index 9f605c0..38d4b6c 100644 --- a/cmd/provision.go +++ b/cmd/provision.go @@ -6,8 +6,10 @@ import ( "os" "strings" - "github.com/sourcegraph/checkup" "github.com/spf13/cobra" + + "github.com/sourcegraph/checkup" + "github.com/sourcegraph/checkup/storage/s3" ) var provisionCmd = &cobra.Command{ @@ -92,7 +94,7 @@ func provisionerEnvVars(cmd *cobra.Command, args []string) (checkup.Provisioner, fmt.Println(cmd.Long) os.Exit(1) } - return checkup.S3{ + return s3.Storage{ AccessKeyID: keyID, SecretAccessKey: secretKey, Bucket: bucket, diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..00580ca --- /dev/null +++ b/errors.go @@ -0,0 +1,7 @@ +package checkup + +const ( + errUnknownCheckerType = "unknown checker type: %T" + errUnknownStorageType = "unknown storage type: %T" + errUnknownNotifierType = "unknown notifier type: %T" +) diff --git a/go.mod b/go.mod index 9395ccb..b7425a7 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/ashwanthkumar/slack-go-webhook v0.0.0-20200209025033-430dd4e66960 - github.com/aws/aws-sdk-go v1.30.2 + github.com/aws/aws-sdk-go v1.30.7 github.com/elazarl/goproxy v0.0.0-20200315184450-1f3cb6622dad // indirect github.com/fatih/color v1.9.0 github.com/google/go-github v17.0.0+incompatible diff --git a/go.sum b/go.sum index e84c2e9..095152c 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/ashwanthkumar/slack-go-webhook v0.0.0-20200209025033-430dd4e66960 h1:MIEURpsIpyLyy+dZ+GnL8T5P49Tco0ik9cYaUQNnAxE= github.com/ashwanthkumar/slack-go-webhook v0.0.0-20200209025033-430dd4e66960/go.mod h1:97O1qkjJBHSSaWJxsTShRIeFy0HWiygk+jnugO9aX3I= -github.com/aws/aws-sdk-go v1.30.2 h1:0vuroAsbPwVbP91MMaUmFLnrQcFBhmjQnnXaH1kcnPw= -github.com/aws/aws-sdk-go v1.30.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.30.7 h1:IaXfqtioP6p9SFAnNfsqdNczbR5UNbYqvcZUSsCAdTY= +github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= diff --git a/interfaces.go b/interfaces.go new file mode 100644 index 0000000..5e757c2 --- /dev/null +++ b/interfaces.go @@ -0,0 +1,51 @@ +package checkup + +import ( + "github.com/sourcegraph/checkup/types" +) + +// Checker can create a types.Result. +type Checker interface { + Check() (types.Result, error) +} + +// Storage can store results. +type Storage interface { + Store([]types.Result) error +} + +// StorageReader can read results from the Storage. +type StorageReader interface { + // Fetch returns the contents of a check file. + Fetch(checkFile string) ([]types.Result, error) + // GetIndex returns the storage index, as a map where keys are check + // result filenames and values are the associated check timestamps. + GetIndex() (map[string]int64, error) +} + +// Maintainer can maintain a store of results by +// deleting old check files that are no longer +// needed or performing other required tasks. +type Maintainer interface { + Maintain() error +} + +// Notifier can notify ops or sysadmins of +// potential problems. A Notifier should keep +// state to avoid sending repeated notices +// more often than the admin would like. +type Notifier interface { + Notify([]types.Result) error +} + +// Provisioner is a type of storage mechanism that can +// provision itself for use with checkup. Provisioning +// need only happen once and is merely a convenience +// so that the user can get up and running with their +// status page more quickly. Presumably, the info +// returned from Provision should be used on the status +// page side of things ot access the check files (like +// a key pair that is used for read-only access). +type Provisioner interface { + Provision() (types.ProvisionInfo, error) +} diff --git a/notifier.go b/notifier.go new file mode 100644 index 0000000..88d222e --- /dev/null +++ b/notifier.go @@ -0,0 +1,30 @@ +package checkup + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/sourcegraph/checkup/notifier/slack" +) + +func notifierDecode(typeName string, config json.RawMessage) (Notifier, error) { + switch typeName { + case "slack": + return slack.New(config) + default: + return nil, errors.New(strings.Replace(errUnknownNotifierType, "%T", typeName, -1)) + } +} + +func notifierType(ch interface{}) (string, error) { + var typeName string + switch ch.(type) { + case slack.Notifier, *slack.Notifier: + typeName = "slack" + default: + return "", fmt.Errorf(errUnknownNotifierType, ch) + } + return typeName, nil +} diff --git a/slack.go b/notifier/slack/slack.go similarity index 67% rename from slack.go rename to notifier/slack/slack.go index 5518965..a1642b7 100644 --- a/slack.go +++ b/notifier/slack/slack.go @@ -1,23 +1,33 @@ -package checkup +package slack import ( "fmt" "log" "strings" + "encoding/json" slack "github.com/ashwanthkumar/slack-go-webhook" + + "github.com/sourcegraph/checkup/types" ) -// Slack consist of all the sub components required to use Slack API -type Slack struct { +// Notifier consist of all the sub components required to use Slack API +type Notifier struct { Name string `json:"name"` Username string `json:"username"` Channel string `json:"channel"` Webhook string `json:"webhook"` } +// New creates a new Notifier instance based on json config +func New(config json.RawMessage) (Notifier, error) { + var notifier Notifier + err := json.Unmarshal(config, ¬ifier) + return notifier, err +} + // Notify implements notifier interface -func (s Slack) Notify(results []Result) error { +func (s Notifier) Notify(results []types.Result) error { for _, result := range results { if !result.Healthy { s.Send(result) @@ -27,7 +37,7 @@ func (s Slack) Notify(results []Result) error { } // Send request via Slack API to create incident -func (s Slack) Send(result Result) error { +func (s Notifier) Send(result types.Result) error { color := "danger" attach := slack.Attachment{} attach.AddField(slack.Field{Title: result.Title, Value: result.Endpoint}) diff --git a/notifier_test.go b/notifier_test.go new file mode 100644 index 0000000..1553af0 --- /dev/null +++ b/notifier_test.go @@ -0,0 +1,17 @@ +package checkup + +import ( + "strings" + "testing" +) + +func TestUnknownNotifierType(t *testing.T) { + kind, err := notifierType("") + if got, want := kind, ""; got != want { + t.Errorf("Expected type '%s', got '%s'", want, got) + } + want := strings.Replace(errUnknownNotifierType, "%T", "string", -1) + if got := err.Error(); got != want { + t.Errorf("Expected error '%s', got '%s'", want, got) + } +} diff --git a/sql_disabled.go b/sql_disabled.go deleted file mode 100644 index 7ae0026..0000000 --- a/sql_disabled.go +++ /dev/null @@ -1,13 +0,0 @@ -// +build !sql - -package checkup - -import ( - "errors" -) - -type SQL struct{} - -func (sql SQL) Store(results []Result) error { - return errors.New("sql data store is disabled") -} diff --git a/storage.go b/storage.go new file mode 100644 index 0000000..721215b --- /dev/null +++ b/storage.go @@ -0,0 +1,45 @@ +package checkup + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/sourcegraph/checkup/storage/s3" + "github.com/sourcegraph/checkup/storage/github" + "github.com/sourcegraph/checkup/storage/fs" + "github.com/sourcegraph/checkup/storage/sql" +) + +func storageDecode(typeName string, config json.RawMessage) (Storage, error) { + switch typeName { + case "s3": + return s3.New(config) + case "github": + return github.New(config) + case "fs": + return fs.New(config) + case "sql": + return sql.New(config) + default: + return nil, errors.New(strings.Replace(errUnknownStorageType, "%T", typeName, -1)) + } +} + +func storageType(ch interface{}) (string, error) { + var typeName string + switch ch.(type) { + case s3.Storage, *s3.Storage: + typeName = "s3" + case github.Storage, *github.Storage: + typeName = "github" + case fs.Storage, *fs.Storage: + typeName = "fs" + case sql.Storage, *sql.Storage: + typeName = "sql" + default: + return "", fmt.Errorf(errUnknownStorageType, ch) + } + return typeName, nil +} diff --git a/fs.go b/storage/fs/fs.go similarity index 72% rename from fs.go rename to storage/fs/fs.go index 4a98979..2974873 100644 --- a/fs.go +++ b/storage/fs/fs.go @@ -1,4 +1,4 @@ -package checkup +package fs import ( "encoding/json" @@ -6,12 +6,12 @@ import ( "os" "path/filepath" "time" -) -const indexName = "index.json" + "github.com/sourcegraph/checkup/types" +) -// FS is a way to store checkup results on the local filesystem. -type FS struct { +// Storage is a way to store checkup results on the local filesystem. +type Storage struct { // The path to the directory where check files will be stored. Dir string `json:"dir"` // The URL corresponding to fs.Dir. @@ -24,15 +24,22 @@ type FS struct { CheckExpiry time.Duration `json:"check_expiry,omitempty"` } +// New creates a new Storage instance based on json config +func New(config json.RawMessage) (Storage, error) { + var storage Storage + err := json.Unmarshal(config, &storage) + return storage, err +} + // GetIndex returns the index from filesystem. -func (fs FS) GetIndex() (map[string]int64, error) { +func (fs Storage) GetIndex() (map[string]int64, error) { return fs.readIndex() } -func (fs FS) readIndex() (map[string]int64, error) { +func (fs Storage) readIndex() (map[string]int64, error) { index := map[string]int64{} - f, err := os.Open(filepath.Join(fs.Dir, indexName)) + f, err := os.Open(filepath.Join(fs.Dir, IndexName)) if os.IsNotExist(err) { return index, nil } else if err != nil { @@ -44,8 +51,8 @@ func (fs FS) readIndex() (map[string]int64, error) { return index, err } -func (fs FS) writeIndex(index map[string]int64) error { - f, err := os.Create(filepath.Join(fs.Dir, indexName)) +func (fs Storage) writeIndex(index map[string]int64) error { + f, err := os.Create(filepath.Join(fs.Dir, IndexName)) if err != nil { return err } @@ -55,12 +62,12 @@ func (fs FS) writeIndex(index map[string]int64) error { } // Fetch fetches results from filesystem for the specified index. -func (fs FS) Fetch(name string) ([]Result, error) { +func (fs Storage) Fetch(name string) ([]types.Result, error) { f, err := os.Open(filepath.Join(fs.Dir, name)) if err != nil { return nil, err } - var results []Result + var results []types.Result err = json.NewDecoder(f).Decode(&results) f.Close() if err != nil { @@ -71,7 +78,7 @@ func (fs FS) Fetch(name string) ([]Result, error) { } // Store stores results on filesystem according to the configuration in fs. -func (fs FS) Store(results []Result) error { +func (fs Storage) Store(results []types.Result) error { // Write results to a new file name := *GenerateFilename() f, err := os.Create(filepath.Join(fs.Dir, name)) @@ -98,7 +105,7 @@ func (fs FS) Store(results []Result) error { } // Maintain deletes check files that are older than fs.CheckExpiry. -func (fs FS) Maintain() error { +func (fs Storage) Maintain() error { if fs.CheckExpiry == 0 { return nil } @@ -114,7 +121,7 @@ func (fs FS) Maintain() error { } for _, f := range files { - if f.Name() == indexName { + if f.Name() == IndexName { continue } diff --git a/fs_test.go b/storage/fs/fs_test.go similarity index 94% rename from fs_test.go rename to storage/fs/fs_test.go index bf813d8..d295c6f 100644 --- a/fs_test.go +++ b/storage/fs/fs_test.go @@ -1,4 +1,4 @@ -package checkup +package fs import ( "bytes" @@ -7,10 +7,12 @@ import ( "path/filepath" "testing" "time" + + "github.com/sourcegraph/checkup/types" ) -func TestFS(t *testing.T) { - results := []Result{{Title: "Testing"}} +func TestStorage(t *testing.T) { + results := []types.Result{{Title: "Testing"}} resultsBytes := []byte(`[{"title":"Testing"}]` + "\n") dir, err := ioutil.TempDir("", "checkup") @@ -19,7 +21,7 @@ func TestFS(t *testing.T) { } defer os.RemoveAll(dir) - specimen := FS{ + specimen := Storage{ Dir: dir, } diff --git a/storage/fs/types.go b/storage/fs/types.go new file mode 100644 index 0000000..9c79e31 --- /dev/null +++ b/storage/fs/types.go @@ -0,0 +1,25 @@ +package fs + +import ( + "fmt" + + "github.com/sourcegraph/checkup/types" +) + +const IndexName = "index.json" + +// FilenameFormatString is the format string used +// by GenerateFilename to create a filename. +const FilenameFormatString = "%d-check.json" + +// GenerateFilename returns a filename that is ideal +// for storing the results file on a storage provider +// that relies on the filename for retrieval that is +// sorted by date/timeframe. It returns a string pointer +// to be used by the AWS SDK... +func GenerateFilename() *string { + s := fmt.Sprintf(FilenameFormatString, types.Timestamp()) + return &s +} + + diff --git a/github.go b/storage/github/github.go similarity index 86% rename from github.go rename to storage/github/github.go index 46cd81b..4a6dbbf 100644 --- a/github.go +++ b/storage/github/github.go @@ -1,4 +1,4 @@ -package checkup +package github import ( "context" @@ -12,12 +12,15 @@ import ( "github.com/google/go-github/github" "golang.org/x/oauth2" + + "github.com/sourcegraph/checkup/types" + "github.com/sourcegraph/checkup/storage/fs" ) var errFileNotFound = fmt.Errorf("file not found on github") -// GitHub is a way to store checkup results in a GitHub repository. -type GitHub struct { +// Storage is a way to store checkup results in a GitHub repository. +type Storage struct { // AccessToken is the API token used to authenticate with GitHub (required). AccessToken string `json:"access_token"` @@ -53,8 +56,15 @@ type GitHub struct { client *github.Client `json:"-"` } +// New creates a new Storage instance based on json config +func New(config json.RawMessage) (*Storage, error) { + storage := new(Storage) + err := json.Unmarshal(config, &storage) + return storage, err +} + // ensureClient builds an GitHub API client if none exists and stores it on the struct. -func (gh *GitHub) ensureClient() error { +func (gh *Storage) ensureClient() error { if gh.client != nil { return nil } @@ -75,7 +85,7 @@ func (gh *GitHub) ensureClient() error { // fullPathName ensures the configured Dir value is present in the filename and // returns a filename with the Dir prefixed before the input filename if necessary. -func (gh *GitHub) fullPathName(filename string) string { +func (gh *Storage) fullPathName(filename string) string { if strings.HasPrefix(filename, gh.Dir) { return filename } else { @@ -86,7 +96,7 @@ func (gh *GitHub) fullPathName(filename string) string { // readFile reads a file from the Git repository at its latest revision. // This method returns the plaintext contents, the SHA associated with the contents // If an error occurs, the contents and sha will be nil & empty. -func (gh *GitHub) readFile(filename string) ([]byte, string, error) { +func (gh *Storage) readFile(filename string) ([]byte, string, error) { if err := gh.ensureClient(); err != nil { return nil, "", err } @@ -112,7 +122,7 @@ func (gh *GitHub) readFile(filename string) ([]byte, string, error) { // writeFile commits the contents to the Git repo at the given filename & revision. // If the Git repo does not yet have a file at this filename, it will create the file. // Otherwise, it will simply update the file with the new contents. -func (gh *GitHub) writeFile(filename string, sha string, contents []byte) error { +func (gh *Storage) writeFile(filename string, sha string, contents []byte) error { if err := gh.ensureClient(); err != nil { return err } @@ -155,7 +165,7 @@ func (gh *GitHub) writeFile(filename string, sha string, contents []byte) error // deleteFile deletes a file from a Git tree and returns any applicable errors. // If an empty SHA is passed as an argument, errFileNotFound is returned. -func (gh *GitHub) deleteFile(filename string, sha string) error { +func (gh *Storage) deleteFile(filename string, sha string) error { if err := gh.ensureClient(); err != nil { return err } @@ -192,10 +202,10 @@ func (gh *GitHub) deleteFile(filename string, sha string) error { // It returns the populated map & the Git SHA associated with the contents. // If the index file is not found in the Git repo, an empty index is returned with no error. // If any error occurs, a nil index and empty SHA are returned along with the error. -func (gh *GitHub) readIndex() (map[string]int64, string, error) { +func (gh *Storage) readIndex() (map[string]int64, string, error) { index := map[string]int64{} - contents, sha, err := gh.readFile(indexName) + contents, sha, err := gh.readFile(fs.IndexName) if err != nil && err != errFileNotFound { return nil, "", err } @@ -209,19 +219,19 @@ func (gh *GitHub) readIndex() (map[string]int64, string, error) { // writeIndex marshals the index into JSON and writes the file to the Git repo. // It returns any errors associated with marshaling the data or writing the file. -func (gh *GitHub) writeIndex(index map[string]int64, sha string) error { +func (gh *Storage) writeIndex(index map[string]int64, sha string) error { contents, err := json.Marshal(index) if err != nil { return err } - return gh.writeFile(indexName, sha, contents) + return gh.writeFile(fs.IndexName, sha, contents) } // Store stores results in the Git repo & updates the index. -func (gh *GitHub) Store(results []Result) error { +func (gh *Storage) Store(results []types.Result) error { // Write results to a new file - name := *GenerateFilename() + name := *fs.GenerateFilename() contents, err := json.Marshal(results) if err != nil { return err @@ -242,24 +252,24 @@ func (gh *GitHub) Store(results []Result) error { } // Fetch returns a checkup record -- Not tested! -func (gh *GitHub) Fetch(name string) ([]Result, error) { +func (gh *Storage) Fetch(name string) ([]types.Result, error) { contents, _, err := gh.readFile(name) if err != nil { return nil, err } - var r []Result + var r []types.Result err = json.Unmarshal(contents, &r) return r, err } // GetIndex returns the checkup index -func (gh *GitHub) GetIndex() (map[string]int64, error) { +func (gh *Storage) GetIndex() (map[string]int64, error) { m, _, e := gh.readIndex() return m, e } // Maintain deletes check files that are older than gh.CheckExpiry. -func (gh *GitHub) Maintain() error { +func (gh *Storage) Maintain() error { if gh.CheckExpiry == 0 { return nil } @@ -285,7 +295,7 @@ func (gh *GitHub) Maintain() error { for _, treeEntry := range tree.Entries { fileName := treeEntry.GetPath() - if fileName == filepath.Join(gh.Dir, indexName) { + if fileName == filepath.Join(gh.Dir, fs.IndexName) { continue } if gh.Dir != "" && !strings.HasPrefix(fileName, gh.Dir) { diff --git a/github_test.go b/storage/github/github_test.go similarity index 98% rename from github_test.go rename to storage/github/github_test.go index 6d84143..ce46df3 100644 --- a/github_test.go +++ b/storage/github/github_test.go @@ -1,4 +1,4 @@ -package checkup +package github import ( "bytes" @@ -16,10 +16,12 @@ import ( "time" "github.com/google/go-github/github" + + "github.com/sourcegraph/checkup/types" ) var ( - results = []Result{{Title: "Testing"}} + results = []types.Result{{Title: "Testing"}} resultsBytes = []byte(`[{"title":"Testing"}]`) ) @@ -65,7 +67,7 @@ func repositoryContent(path, serverSHAForRepo string, data interface{}) *github. } } -func withGitHubServer(t *testing.T, specimen GitHub, f func(*github.Client)) { +func withGitHubServer(t *testing.T, specimen Storage, f func(*github.Client)) { // test server mux := http.NewServeMux() server := httptest.NewServer(mux) @@ -291,7 +293,7 @@ func withGitHubServer(t *testing.T, specimen GitHub, f func(*github.Client)) { func TestGitHubWithoutSubdir(t *testing.T) { // Our subject, our specimen. - specimen := &GitHub{ + specimen := &Storage{ RepositoryOwner: "o", RepositoryName: "r", CommitterName: "John Appleseed", @@ -364,7 +366,7 @@ func TestGitHubWithoutSubdir(t *testing.T) { func TestGitHubWithSubdir(t *testing.T) { // Our subject, our specimen. - specimen := &GitHub{ + specimen := &Storage{ RepositoryOwner: "o", RepositoryName: "r", CommitterName: "John Appleseed", diff --git a/s3.go b/storage/s3/s3.go similarity index 90% rename from s3.go rename to storage/s3/s3.go index 167236c..917a932 100644 --- a/s3.go +++ b/storage/s3/s3.go @@ -1,4 +1,4 @@ -package checkup +package s3 import ( "bytes" @@ -12,10 +12,13 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/s3" + + "github.com/sourcegraph/checkup/types" + "github.com/sourcegraph/checkup/storage/fs" ) -// S3 is a way to store checkup results in an S3 bucket. -type S3 struct { +// Storage is a way to store checkup results in an S3 bucket. +type Storage struct { AccessKeyID string `json:"access_key_id"` SecretAccessKey string `json:"secret_access_key"` Region string `json:"region,omitempty"` @@ -28,8 +31,15 @@ type S3 struct { CheckExpiry time.Duration `json:"check_expiry,omitempty"` } +// New creates a new Storage instance based on json config +func New(config json.RawMessage) (Storage, error) { + var storage Storage + err := json.Unmarshal(config, &storage) + return storage, err +} + // Store stores results on S3 according to the configuration in s. -func (s S3) Store(results []Result) error { +func (s Storage) Store(results []types.Result) error { jsonBytes, err := json.Marshal(results) if err != nil { return err @@ -44,7 +54,7 @@ func (s S3) Store(results []Result) error { svc := newS3(session.New(), config) params := &s3.PutObjectInput{ Bucket: &s.Bucket, - Key: GenerateFilename(), + Key: fs.GenerateFilename(), Body: bytes.NewReader(jsonBytes), } _, err = svc.PutObject(params) @@ -52,7 +62,7 @@ func (s S3) Store(results []Result) error { } // Maintain deletes check files that are older than s.CheckExpiry. -func (s S3) Maintain() error { +func (s Storage) Maintain() error { if s.CheckExpiry == 0 { return nil } @@ -124,9 +134,9 @@ func (s S3) Maintain() error { // // Provision need only be called once per status page (bucket), // not once per endpoint. -func (s S3) Provision() (ProvisionInfo, error) { +func (s Storage) Provision() (types.ProvisionInfo, error) { const iamUser = "checkup-monitor-s3-public" - var info ProvisionInfo + var info types.ProvisionInfo // default region (required, but regions don't apply to S3, kinda weird) if s.Region == "" { diff --git a/s3_test.go b/storage/s3/s3_test.go similarity index 96% rename from s3_test.go rename to storage/s3/s3_test.go index b121873..34e34cb 100644 --- a/s3_test.go +++ b/storage/s3/s3_test.go @@ -1,4 +1,4 @@ -package checkup +package s3 import ( "bytes" @@ -11,12 +11,14 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/service/s3" + + "github.com/sourcegraph/checkup/types" ) func TestS3Store(t *testing.T) { keyID, accessKey, region, bucket := "fakeKeyID", "fakeKey", "fakeRegion", "fakeBucket" fakes3 := new(s3Mock) - results := []Result{{Title: "Testing"}} + results := []types.Result{{Title: "Testing"}} resultsBytes := []byte(`[{"title":"Testing"}]`) newS3 = func(p client.ConfigProvider, cfgs ...*aws.Config) s3svc { if len(cfgs) != 1 { @@ -38,7 +40,7 @@ func TestS3Store(t *testing.T) { return fakes3 } - specimen := S3{ + specimen := Storage{ AccessKeyID: keyID, SecretAccessKey: accessKey, Region: region, @@ -86,7 +88,7 @@ func TestS3Maintain(t *testing.T) { return fakes3 } - var specimen S3 + var specimen Storage err := specimen.Maintain() if err != nil { t.Fatalf("Expected no error, got %v", err) diff --git a/sql.go b/storage/sql/sql.go similarity index 86% rename from sql.go rename to storage/sql/sql.go index b338b79..9e90a1a 100644 --- a/sql.go +++ b/storage/sql/sql.go @@ -1,6 +1,6 @@ // +build sql -package checkup +package sql import ( "encoding/json" @@ -11,6 +11,8 @@ import ( "github.com/jmoiron/sqlx" _ "github.com/lib/pq" // Enable postgresql beckend _ "github.com/mattn/go-sqlite3" // Enable sqlite3 backend + + "github.com/sourcegraph/checkup/types" ) // schema is the table schema expected by the sqlite3 checkup storage. @@ -23,8 +25,8 @@ CREATE TABLE checks ( CREATE UNIQUE INDEX idx_checks_timestamp ON checks(timestamp); ` -// SQL is a way to store checkup results in a SQL database. -type SQL struct { +// Storage is a way to store checkup results in a SQL database. +type Storage struct { // SqliteDBFile is the sqlite3 DB where check results will be stored. SqliteDBFile string `json:"sqlite_db_file,omitempty"` @@ -45,7 +47,14 @@ type SQL struct { CheckExpiry time.Duration `json:"check_expiry,omitempty"` } -func (sql SQL) dbConnect() (*sqlx.DB, error) { +// New creates a new Storage instance based on json config +func New(config json.RawMessage) (Storage, error) { + var storage Storage + err := json.Unmarshal(config, &storage) + return storage, err +} + +func (sql Storage) dbConnect() (*sqlx.DB, error) { // Only one SQL backend can be present if sql.SqliteDBFile != "" && sql.PostgreSQL != nil { return nil, errors.New("several SQL backends are configured") @@ -88,7 +97,7 @@ func (sql SQL) dbConnect() (*sqlx.DB, error) { } // GetIndex returns the list of check results for the database. -func (sql SQL) GetIndex() (map[string]int64, error) { +func (sql Storage) GetIndex() (map[string]int64, error) { db, err := sql.dbConnect() if err != nil { return nil, err @@ -118,7 +127,7 @@ func (sql SQL) GetIndex() (map[string]int64, error) { } // Fetch fetches results of the check with given name. -func (sql SQL) Fetch(name string) ([]Result, error) { +func (sql Storage) Fetch(name string) ([]types.Result, error) { db, err := sql.dbConnect() if err != nil { return nil, err @@ -126,7 +135,7 @@ func (sql SQL) Fetch(name string) ([]Result, error) { defer db.Close() var checkResult []byte - var results []Result + var results []types.Result err = db.Get(&checkResult, `SELECT results FROM "checks" WHERE name=$1 LIMIT 1`, name) if err != nil { @@ -139,7 +148,7 @@ func (sql SQL) Fetch(name string) ([]Result, error) { } // Store stores results in the database. -func (sql SQL) Store(results []Result) error { +func (sql Storage) Store(results []types.Result) error { db, err := sql.dbConnect() if err != nil { return err @@ -159,7 +168,7 @@ func (sql SQL) Store(results []Result) error { } // Maintain deletes check files that are older than sql.CheckExpiry. -func (sql SQL) Maintain() error { +func (sql Storage) Maintain() error { if sql.CheckExpiry == 0 { return nil } @@ -177,7 +186,7 @@ func (sql SQL) Maintain() error { } // initialize creates the "checks" table in the database. -func (sql SQL) initialize() error { +func (sql Storage) initialize() error { db, err := sql.dbConnect() if err != nil { return err diff --git a/storage/sql/sql_disabled.go b/storage/sql/sql_disabled.go new file mode 100644 index 0000000..8920432 --- /dev/null +++ b/storage/sql/sql_disabled.go @@ -0,0 +1,21 @@ +// +build !sql + +package sql + +import ( + "errors" + "encoding/json" + + "github.com/sourcegraph/checkup/types" +) + +type Storage struct{} + +// New creates a new Storage instance based on json config +func New(_ json.RawMessage) (Storage, error) { + return Storage{}, errors.New("sql data store is disabled") +} + +func (_ Storage) Store(results []types.Result) error { + return errors.New("sql data store is disabled") +} diff --git a/sql_test.go b/storage/sql/sql_test.go similarity index 95% rename from sql_test.go rename to storage/sql/sql_test.go index 30955d0..e1e6e87 100644 --- a/sql_test.go +++ b/storage/sql/sql_test.go @@ -1,6 +1,6 @@ // +build sql -package checkup +package sql import ( "io/ioutil" @@ -8,10 +8,12 @@ import ( "path/filepath" "testing" "time" + + "github.com/sourcegraph/checkup/types" ) func TestSQL(t *testing.T) { - results := []Result{{Title: "Testing"}} + results := []types.Result{{Title: "Testing"}} // Create temporary directory for the tests dir, err := ioutil.TempDir("", "checkup") @@ -22,7 +24,7 @@ func TestSQL(t *testing.T) { dbFile := filepath.Join(dir, "checkuptest.db") - specimen := SQL{ + specimen := Storage{ SqliteDBFile: dbFile, } diff --git a/storage_test.go b/storage_test.go new file mode 100644 index 0000000..94386d9 --- /dev/null +++ b/storage_test.go @@ -0,0 +1,17 @@ +package checkup + +import ( + "strings" + "testing" +) + +func TestUnknownStorageType(t *testing.T) { + kind, err := storageType("") + if got, want := kind, ""; got != want { + t.Errorf("Expected type '%s', got '%s'", want, got) + } + want := strings.Replace(errUnknownStorageType, "%T", "string", -1) + if got := err.Error(); got != want { + t.Errorf("Expected error '%s', got '%s'", want, got) + } +} diff --git a/types/attempt.go b/types/attempt.go new file mode 100644 index 0000000..c9ff60e --- /dev/null +++ b/types/attempt.go @@ -0,0 +1,18 @@ +package types + +import ( + "time" +) + +// Attempt is an attempt to communicate with the endpoint. +type Attempt struct { + RTT time.Duration `json:"rtt"` + Error string `json:"error,omitempty"` +} + +// Attempts is a list of Attempt that can be sorted by RTT. +type Attempts []Attempt + +func (a Attempts) Len() int { return len(a) } +func (a Attempts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a Attempts) Less(i, j int) bool { return a[i].RTT < a[j].RTT } diff --git a/types/provisioner.go b/types/provisioner.go new file mode 100644 index 0000000..4ebf25a --- /dev/null +++ b/types/provisioner.go @@ -0,0 +1,41 @@ +package types + +import ( + "fmt" +) + +// ProvisionInfo contains the results of provisioning a new +// storage facility for check files. Its values should be +// used by the status page in order to obtain read-only +// access to the check files. +type ProvisionInfo struct { + // The ID of a user that was created for accessing checks. + UserID string `json:"user_id"` + + // The username of a user that was created for accessing checks. + Username string `json:"username"` + + // The ID or name of the ID/key used to access checks. Expect + // this value to be made public. (It should have read-only + // access to the checks.) + PublicAccessKeyID string `json:"public_access_key_id"` + + // The "secret" associated with the PublicAccessKeyID, but + // expect this value to be made public. (It should provide + // read-only access to the checks.) + PublicAccessKey string `json:"public_access_key"` +} + +// String returns the information in i in a human-readable format +// along with an important notice. +func (i ProvisionInfo) String() string { + s := "Provision successful\n\n" + s += fmt.Sprintf(" User ID: %s\n", i.UserID) + s += fmt.Sprintf(" Username: %s\n", i.Username) + s += fmt.Sprintf("Public Access Key ID: %s\n", i.PublicAccessKeyID) + s += fmt.Sprintf(" Public Access Key: %s\n\n", i.PublicAccessKey) + s += `IMPORTANT: Copy the Public Access Key ID and Public Access +Key into the config.js file for your status page. You will +not be shown these credentials again.` + return s +} diff --git a/types/result.go b/types/result.go new file mode 100644 index 0000000..6e5993f --- /dev/null +++ b/types/result.go @@ -0,0 +1,122 @@ +package types + +import ( + "fmt" + "sort" + "time" + + "github.com/fatih/color" +) + +// Result is the result of a health check. +type Result struct { + // Title is the title (or name) of the thing that was checked. + // It should be unique, as it acts like an identifier to users. + Title string `json:"title,omitempty"` + + // Endpoint is the URL/address/path/identifier/locator/whatever + // of what was checked. + Endpoint string `json:"endpoint,omitempty"` + + // Timestamp is when the check occurred; UTC UnixNano format. + Timestamp int64 `json:"timestamp,omitempty"` + + // Times is a list of each individual check attempt. + Times Attempts `json:"times,omitempty"` + + // ThresholdRTT is the maximum RTT that was tolerated before + // considering performance to be degraded. Leave 0 if irrelevant. + ThresholdRTT time.Duration `json:"threshold,omitempty"` + + // Healthy, Degraded, and Down contain the ultimate conclusion + // about the endpoint. Exactly one of these should be true; + // any more or less is a bug. + Healthy bool `json:"healthy,omitempty"` + Degraded bool `json:"degraded,omitempty"` + Down bool `json:"down,omitempty"` + + // Notice contains a description of some condition of this + // check that might have affected the result in some way. + // For example, that the median RTT is above the threshold. + Notice string `json:"notice,omitempty"` + + // Message is an optional message to show on the status page. + // For example, what you're doing to fix a problem. + Message string `json:"message,omitempty"` +} + +func NewResult() Result { + return Result{ + Timestamp: Timestamp(), + } +} + +// ComputeStats computes basic statistics about r. +func (r Result) ComputeStats() Stats { + var s Stats + + for _, a := range r.Times { + s.Total += a.RTT + if a.RTT < s.Min || s.Min == 0 { + s.Min = a.RTT + } + if a.RTT > s.Max || s.Max == 0 { + s.Max = a.RTT + } + } + sorted := make(Attempts, len(r.Times)) + copy(sorted, r.Times) + sort.Sort(sorted) + + half := len(sorted) / 2 + if len(sorted)%2 == 0 { + s.Median = (sorted[half-1].RTT + sorted[half].RTT) / 2 + } else { + s.Median = sorted[half].RTT + } + + s.Mean = time.Duration(int64(s.Total) / int64(len(r.Times))) + + return s +} + +// DisableColor disables ANSI colors in the Result default string. +func DisableColor() { + color.NoColor = true +} + +// String returns a human-readable rendering of r. +func (r Result) String() string { + stats := r.ComputeStats() + s := fmt.Sprintf("== %s - %s\n", r.Title, r.Endpoint) + s += fmt.Sprintf(" Threshold: %s\n", r.ThresholdRTT) + s += fmt.Sprintf(" Max: %s\n", stats.Max) + s += fmt.Sprintf(" Min: %s\n", stats.Min) + s += fmt.Sprintf(" Median: %s\n", stats.Median) + s += fmt.Sprintf(" Mean: %s\n", stats.Mean) + s += fmt.Sprintf(" All: %v\n", r.Times) + statusLine := fmt.Sprintf(" Assessment: %v\n", r.Status()) + switch r.Status() { + case StatusHealthy: + statusLine = color.GreenString(statusLine) + case StatusDegraded: + statusLine = color.YellowString(statusLine) + case StatusDown: + statusLine = color.RedString(statusLine) + } + s += statusLine + return s +} + +// Status returns a text representation of the overall status +// indicated in r. +func (r Result) Status() StatusText { + if r.Down { + return StatusDown + } else if r.Degraded { + return StatusDegraded + } else if r.Healthy { + return StatusHealthy + } + return StatusUnknown +} diff --git a/types/stats.go b/types/stats.go new file mode 100644 index 0000000..8eef341 --- /dev/null +++ b/types/stats.go @@ -0,0 +1,15 @@ +package types + +import ( + "time" +) + +// Stats is a type that holds information about a Result, +// especially its various Attempts. +type Stats struct { + Total time.Duration `json:"total,omitempty"` + Mean time.Duration `json:"mean,omitempty"` + Median time.Duration `json:"median,omitempty"` + Min time.Duration `json:"min,omitempty"` + Max time.Duration `json:"max,omitempty"` +} diff --git a/types/status.go b/types/status.go new file mode 100644 index 0000000..38c98ce --- /dev/null +++ b/types/status.go @@ -0,0 +1,36 @@ +package types + +// StatusText is the textual representation of the +// result of a status check. +type StatusText string + +// PriorityOver returns whether s has priority over other. +// For example, a Down status has priority over Degraded. +func (s StatusText) PriorityOver(other StatusText) bool { + if s == other { + return false + } + switch s { + case StatusDown: + return true + case StatusDegraded: + if other == StatusDown { + return false + } + return true + case StatusHealthy: + if other == StatusUnknown { + return true + } + return false + } + return false +} + +// Text representations for the status of a check. +const ( + StatusHealthy StatusText = "healthy" + StatusDegraded StatusText = "degraded" + StatusDown StatusText = "down" + StatusUnknown StatusText = "unknown" +) diff --git a/types/util.go b/types/util.go new file mode 100644 index 0000000..3542e65 --- /dev/null +++ b/types/util.go @@ -0,0 +1,11 @@ +package types + +import ( + "time" +) + +// Timestamp returns the UTC Unix timestamp in +// nanoseconds. +func Timestamp() int64 { + return time.Now().UTC().UnixNano() +} From 5ed18db77a7611f0392963b0818a1346674e7041 Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 11:25:12 +0200 Subject: [PATCH 03/12] notifier slack: prevent swallowing errors The notifier functions are set up to return errors. The slack notifier only issued log calls, but swallowed the error for handling in checkup runs. --- checkup.go | 29 +++-------------------------- notifier/slack/slack.go | 15 ++++++--------- types/errors.go | 29 +++++++++++++++++++++++++++++ types/errors_test.go | 19 +++++++++++++++++++ 4 files changed, 57 insertions(+), 35 deletions(-) create mode 100644 types/errors.go create mode 100644 types/errors_test.go diff --git a/checkup.go b/checkup.go index 9c8af13..ab49c8a 100644 --- a/checkup.go +++ b/checkup.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "log" - "strings" "sync" "time" @@ -59,7 +58,7 @@ func (c Checkup) Check() ([]types.Result, error) { } results := make([]types.Result, len(c.Checkers)) - errs := make(Errors, len(c.Checkers)) + errs := make(types.Errors, len(c.Checkers)) throttle := make(chan struct{}, c.ConcurrentChecks) wg := sync.WaitGroup{} @@ -87,8 +86,9 @@ func (c Checkup) Check() ([]types.Result, error) { if c.Notifier != nil { err := c.Notifier.Notify(results) if err != nil { - return results, err + log.Printf("ERROR sending notifications: %s", err) } + return results, nil } return results, nil @@ -298,26 +298,3 @@ func (c *Checkup) UnmarshalJSON(b []byte) error { // at most, to perform concurrently. var DefaultConcurrentChecks = 5 -// Errors is an error type that concatenates multiple errors. -type Errors []error - -// Error returns a string containing all the errors in e. -func (e Errors) Error() string { - var errs []string - for _, err := range e { - if err != nil { - errs = append(errs, err.Error()) - } - } - return strings.Join(errs, "; ") -} - -// Empty returns whether e has any non-nil errors in it. -func (e Errors) Empty() bool { - for _, err := range e { - if err != nil { - return false - } - } - return true -} diff --git a/notifier/slack/slack.go b/notifier/slack/slack.go index a1642b7..4a13715 100644 --- a/notifier/slack/slack.go +++ b/notifier/slack/slack.go @@ -2,7 +2,6 @@ package slack import ( "fmt" - "log" "strings" "encoding/json" @@ -28,12 +27,15 @@ func New(config json.RawMessage) (Notifier, error) { // Notify implements notifier interface func (s Notifier) Notify(results []types.Result) error { + errs := make(types.Errors, 0) for _, result := range results { if !result.Healthy { - s.Send(result) + if err := s.Send(result); err != nil { + errs = append(errs, err) + } } } - return nil + return errs } // Send request via Slack API to create incident @@ -50,10 +52,5 @@ func (s Notifier) Send(result types.Result) error { Attachments: []slack.Attachment{attach}, } - err := slack.Send(s.Webhook, "", payload) - if len(err) > 0 { - log.Printf("ERROR: %s", err) - } - log.Printf("Create request for %s", result.Endpoint) - return nil + return types.Errors(slack.Send(s.Webhook, "", payload)) } diff --git a/types/errors.go b/types/errors.go new file mode 100644 index 0000000..d39d66f --- /dev/null +++ b/types/errors.go @@ -0,0 +1,29 @@ +package types + +import ( + "strings" +) + +// Errors is an error type that concatenates multiple errors. +type Errors []error + +// Error returns a string containing all the errors in e. +func (e Errors) Error() string { + var errs []string + for _, err := range e { + if err != nil { + errs = append(errs, err.Error()) + } + } + return strings.Join(errs, "; ") +} + +// Empty returns whether e has any non-nil errors in it. +func (e Errors) Empty() bool { + for _, err := range e { + if err != nil { + return false + } + } + return true +} diff --git a/types/errors_test.go b/types/errors_test.go new file mode 100644 index 0000000..0b2682b --- /dev/null +++ b/types/errors_test.go @@ -0,0 +1,19 @@ +package types + +import ( + "errors" + "testing" +) + +func TestErrors(t *testing.T) { + errs := []error{ + errors.New("Err 1"), + errors.New("Err 2"), + } + errsT := Errors(errs) + + want := "Err 1; Err 2" + if got := errsT.Error(); want != got { + t.Errorf("Errors, wanted '%s', got '%s'", want, got) + } +} From ba94c0e6ca356d465502fafddae6c3c0f5bb8676 Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 11:37:31 +0200 Subject: [PATCH 04/12] chore: fixing formatting issues with gofmt --- checkup.go | 1 - errors.go | 4 ++-- notifier/slack/slack.go | 2 +- storage.go | 4 ++-- storage/fs/types.go | 2 -- storage/github/github.go | 2 +- storage/s3/s3.go | 2 +- storage/sql/sql_disabled.go | 2 +- 8 files changed, 8 insertions(+), 11 deletions(-) diff --git a/checkup.go b/checkup.go index ab49c8a..fcf1fdd 100644 --- a/checkup.go +++ b/checkup.go @@ -297,4 +297,3 @@ func (c *Checkup) UnmarshalJSON(b []byte) error { // DefaultConcurrentChecks is how many checks, // at most, to perform concurrently. var DefaultConcurrentChecks = 5 - diff --git a/errors.go b/errors.go index 00580ca..961a3aa 100644 --- a/errors.go +++ b/errors.go @@ -1,7 +1,7 @@ package checkup const ( - errUnknownCheckerType = "unknown checker type: %T" - errUnknownStorageType = "unknown storage type: %T" + errUnknownCheckerType = "unknown checker type: %T" + errUnknownStorageType = "unknown storage type: %T" errUnknownNotifierType = "unknown notifier type: %T" ) diff --git a/notifier/slack/slack.go b/notifier/slack/slack.go index 4a13715..26f3515 100644 --- a/notifier/slack/slack.go +++ b/notifier/slack/slack.go @@ -1,9 +1,9 @@ package slack import ( + "encoding/json" "fmt" "strings" - "encoding/json" slack "github.com/ashwanthkumar/slack-go-webhook" diff --git a/storage.go b/storage.go index 721215b..ebbaad9 100644 --- a/storage.go +++ b/storage.go @@ -6,9 +6,9 @@ import ( "fmt" "strings" - "github.com/sourcegraph/checkup/storage/s3" - "github.com/sourcegraph/checkup/storage/github" "github.com/sourcegraph/checkup/storage/fs" + "github.com/sourcegraph/checkup/storage/github" + "github.com/sourcegraph/checkup/storage/s3" "github.com/sourcegraph/checkup/storage/sql" ) diff --git a/storage/fs/types.go b/storage/fs/types.go index 9c79e31..5781c17 100644 --- a/storage/fs/types.go +++ b/storage/fs/types.go @@ -21,5 +21,3 @@ func GenerateFilename() *string { s := fmt.Sprintf(FilenameFormatString, types.Timestamp()) return &s } - - diff --git a/storage/github/github.go b/storage/github/github.go index 4a6dbbf..19d2661 100644 --- a/storage/github/github.go +++ b/storage/github/github.go @@ -13,8 +13,8 @@ import ( "github.com/google/go-github/github" "golang.org/x/oauth2" - "github.com/sourcegraph/checkup/types" "github.com/sourcegraph/checkup/storage/fs" + "github.com/sourcegraph/checkup/types" ) var errFileNotFound = fmt.Errorf("file not found on github") diff --git a/storage/s3/s3.go b/storage/s3/s3.go index 917a932..6cb343d 100644 --- a/storage/s3/s3.go +++ b/storage/s3/s3.go @@ -13,8 +13,8 @@ import ( "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/s3" - "github.com/sourcegraph/checkup/types" "github.com/sourcegraph/checkup/storage/fs" + "github.com/sourcegraph/checkup/types" ) // Storage is a way to store checkup results in an S3 bucket. diff --git a/storage/sql/sql_disabled.go b/storage/sql/sql_disabled.go index 8920432..2d19b72 100644 --- a/storage/sql/sql_disabled.go +++ b/storage/sql/sql_disabled.go @@ -3,8 +3,8 @@ package sql import ( - "errors" "encoding/json" + "errors" "github.com/sourcegraph/checkup/types" ) From 8ad096d76f64a04aa8dc539da33b5afe7277aa9f Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 12:31:31 +0200 Subject: [PATCH 05/12] Makefile: revert to go test, add go fmt ./... --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f65f27e..6e4fb0d 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,12 @@ all: build test DOCKER_IMAGE := checkup build: + go fmt ./... mkdir -p builds/ go build -o builds/ ./cmd/... test: - /root/go/bin/gotest -race -count=1 -v ./... + go test -race -count=1 -v ./... docker: docker build --no-cache . -t $(DOCKER_IMAGE) From 603b070022e3236e62b73c4a01be48bc157c3dd7 Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 12:39:41 +0200 Subject: [PATCH 06/12] checkup: update checks, notifiers and storage with types I added a Type field and Type functions on checks, notifiers and storage interfaces, to avoid awkward type casting/type detection from json data and go structs --- check.go | 33 ++++++--------------------------- check/dns/dns.go | 8 ++++++++ check/exec/exec.go | 8 ++++++++ check/exec/exec_test.go | 1 - check/http/http.go | 8 ++++++++ check/tcp/tcp.go | 8 ++++++++ check/tls/tls.go | 8 ++++++++ check_test.go | 17 ----------------- interfaces.go | 3 +++ notifier.go | 20 +++++--------------- notifier/slack/slack.go | 9 ++++++++- notifier_test.go | 17 ----------------- storage.go | 29 +++++------------------------ storage/fs/fs.go | 8 ++++++++ storage/github/github.go | 8 ++++++++ storage/s3/s3.go | 8 ++++++++ storage/sql/sql.go | 5 +++++ storage/sql/sql_disabled.go | 7 ++++++- storage/sql/types.go | 4 ++++ storage_test.go | 17 ----------------- 20 files changed, 106 insertions(+), 120 deletions(-) delete mode 100644 check_test.go delete mode 100644 notifier_test.go create mode 100644 storage/sql/types.go delete mode 100644 storage_test.go diff --git a/check.go b/check.go index 49ce00f..e8ccc72 100644 --- a/check.go +++ b/check.go @@ -2,9 +2,7 @@ package checkup import ( "encoding/json" - "errors" "fmt" - "strings" "github.com/sourcegraph/checkup/check/dns" "github.com/sourcegraph/checkup/check/exec" @@ -15,36 +13,17 @@ import ( func checkerDecode(typeName string, config json.RawMessage) (Checker, error) { switch typeName { - case "dns": + case dns.Type: return dns.New(config) - case "exec": + case exec.Type: return exec.New(config) - case "http": + case http.Type: return http.New(config) - case "tcp": + case tcp.Type: return tcp.New(config) - case "tls": + case tls.Type: return tls.New(config) default: - return nil, errors.New(strings.Replace(errUnknownCheckerType, "%T", typeName, -1)) + return nil, fmt.Errorf(errUnknownCheckerType, typeName) } } - -func checkerType(ch interface{}) (string, error) { - var typeName string - switch ch.(type) { - case dns.Checker, *dns.Checker: - typeName = "dns" - case exec.Checker, *exec.Checker: - typeName = "exec" - case http.Checker, *http.Checker: - typeName = "http" - case tcp.Checker, *tcp.Checker: - typeName = "tcp" - case tls.Checker, *tls.Checker: - typeName = "tls" - default: - return "", fmt.Errorf(errUnknownCheckerType, ch) - } - return typeName, nil -} diff --git a/check/dns/dns.go b/check/dns/dns.go index 6e76439..9d28fd1 100644 --- a/check/dns/dns.go +++ b/check/dns/dns.go @@ -11,6 +11,9 @@ import ( "github.com/sourcegraph/checkup/types" ) +// Type should match the package name +const Type = "dns" + // Checker implements a Checker for TCP endpoints. type Checker struct { // Name is the name of the endpoint. @@ -41,6 +44,11 @@ func New(config json.RawMessage) (Checker, error) { return checker, err } +// Type returns the checker package name +func (Checker) Type() string { + return Type +} + // Check performs checks using c according to its configuration. // An error is only returned if there is a configuration error. func (c Checker) Check() (types.Result, error) { diff --git a/check/exec/exec.go b/check/exec/exec.go index 4a67a5f..c133b69 100644 --- a/check/exec/exec.go +++ b/check/exec/exec.go @@ -11,6 +11,9 @@ import ( "github.com/sourcegraph/checkup/types" ) +// Type should match the package name +const Type = "exec" + // Checker implements a Checker by running programs with os.Exec. type Checker struct { // Name is the name of the endpoint. @@ -70,6 +73,11 @@ func New(config json.RawMessage) (Checker, error) { return checker, err } +// Type returns the checker package name +func (Checker) Type() string { + return Type +} + // Check performs checks using c according to its configuration. // An error is only returned if there is a configuration error. func (c Checker) Check() (types.Result, error) { diff --git a/check/exec/exec_test.go b/check/exec/exec_test.go index 581a74c..75d3138 100644 --- a/check/exec/exec_test.go +++ b/check/exec/exec_test.go @@ -31,7 +31,6 @@ func TestChecker(t *testing.T) { hc := Checker{Name: testName, Command: command, Arguments: []string{"0", testName}, Attempts: 2} result, err := hc.Check() - t.Logf("%#v", result) assert(err == nil, "expected no error, got %v, %#v", err, result) assert(result.Title == testName, "expected result.Title == %s, got '%s'", testName, result.Title) assert(result.Down == false, "expected result.Down = false, got %v", result.Down) diff --git a/check/http/http.go b/check/http/http.go index 8b55c93..ff5968f 100644 --- a/check/http/http.go +++ b/check/http/http.go @@ -12,6 +12,9 @@ import ( "github.com/sourcegraph/checkup/types" ) +// Type should match the package name +const Type = "http" + // Checker implements a Checker for HTTP endpoints. type Checker struct { // Name is the name of the endpoint. @@ -76,6 +79,11 @@ func New(config json.RawMessage) (Checker, error) { return checker, err } +// Type returns the checker package name +func (Checker) Type() string { + return Type +} + // Check performs checks using c according to its configuration. // An error is only returned if there is a configuration error. func (c Checker) Check() (types.Result, error) { diff --git a/check/tcp/tcp.go b/check/tcp/tcp.go index 6c9caf1..3ff768c 100644 --- a/check/tcp/tcp.go +++ b/check/tcp/tcp.go @@ -12,6 +12,9 @@ import ( "github.com/sourcegraph/checkup/types" ) +// Type should match the package name +const Type = "tcp" + // Checker implements a Checker for TCP endpoints. type Checker struct { // Name is the name of the endpoint. @@ -71,6 +74,11 @@ func New(config json.RawMessage) (Checker, error) { return checker, err } +// Type returns the checker package name +func (Checker) Type() string { + return Type +} + // doChecks executes and returns each attempt. func (c Checker) doChecks() types.Attempts { var err error diff --git a/check/tls/tls.go b/check/tls/tls.go index 311ce5a..c1693ac 100644 --- a/check/tls/tls.go +++ b/check/tls/tls.go @@ -12,6 +12,9 @@ import ( "github.com/sourcegraph/checkup/types" ) +// Type should match the package name +const Type = "tls" + // Checker implements a Checker for TLS endpoints. // // TODO: Implement more checks on the certificate and TLS configuration. @@ -66,6 +69,11 @@ func New(config json.RawMessage) (Checker, error) { return checker, err } +// Type returns the checker package name +func (Checker) Type() string { + return Type +} + // Check performs checks using c according to its configuration. // An error is only returned if there is a configuration error. func (c Checker) Check() (types.Result, error) { diff --git a/check_test.go b/check_test.go deleted file mode 100644 index 6ae5a41..0000000 --- a/check_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package checkup - -import ( - "strings" - "testing" -) - -func TestUnknownCheckerType(t *testing.T) { - kind, err := checkerType("") - if got, want := kind, ""; got != want { - t.Errorf("Expected type '%s', got '%s'", want, got) - } - want := strings.Replace(errUnknownCheckerType, "%T", "string", -1) - if got := err.Error(); got != want { - t.Errorf("Expected error '%s', got '%s'", want, got) - } -} diff --git a/interfaces.go b/interfaces.go index 5e757c2..4ce29c2 100644 --- a/interfaces.go +++ b/interfaces.go @@ -6,11 +6,13 @@ import ( // Checker can create a types.Result. type Checker interface { + Type() string Check() (types.Result, error) } // Storage can store results. type Storage interface { + Type() string Store([]types.Result) error } @@ -35,6 +37,7 @@ type Maintainer interface { // state to avoid sending repeated notices // more often than the admin would like. type Notifier interface { + Type() string Notify([]types.Result) error } diff --git a/notifier.go b/notifier.go index 88d222e..5f72ba5 100644 --- a/notifier.go +++ b/notifier.go @@ -2,29 +2,19 @@ package checkup import ( "encoding/json" - "errors" "fmt" - "strings" + "github.com/sourcegraph/checkup/notifier/mail" "github.com/sourcegraph/checkup/notifier/slack" ) func notifierDecode(typeName string, config json.RawMessage) (Notifier, error) { switch typeName { - case "slack": + case mail.Type: + return mail.New(config) + case slack.Type: return slack.New(config) default: - return nil, errors.New(strings.Replace(errUnknownNotifierType, "%T", typeName, -1)) + return nil, fmt.Errorf(errUnknownNotifierType, typeName) } } - -func notifierType(ch interface{}) (string, error) { - var typeName string - switch ch.(type) { - case slack.Notifier, *slack.Notifier: - typeName = "slack" - default: - return "", fmt.Errorf(errUnknownNotifierType, ch) - } - return typeName, nil -} diff --git a/notifier/slack/slack.go b/notifier/slack/slack.go index 26f3515..e6f6847 100644 --- a/notifier/slack/slack.go +++ b/notifier/slack/slack.go @@ -10,9 +10,11 @@ import ( "github.com/sourcegraph/checkup/types" ) +// Type should match the package name +const Type = "slack" + // Notifier consist of all the sub components required to use Slack API type Notifier struct { - Name string `json:"name"` Username string `json:"username"` Channel string `json:"channel"` Webhook string `json:"webhook"` @@ -25,6 +27,11 @@ func New(config json.RawMessage) (Notifier, error) { return notifier, err } +// Type returns the notifier package name +func (Notifier) Type() string { + return Type +} + // Notify implements notifier interface func (s Notifier) Notify(results []types.Result) error { errs := make(types.Errors, 0) diff --git a/notifier_test.go b/notifier_test.go deleted file mode 100644 index 1553af0..0000000 --- a/notifier_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package checkup - -import ( - "strings" - "testing" -) - -func TestUnknownNotifierType(t *testing.T) { - kind, err := notifierType("") - if got, want := kind, ""; got != want { - t.Errorf("Expected type '%s', got '%s'", want, got) - } - want := strings.Replace(errUnknownNotifierType, "%T", "string", -1) - if got := err.Error(); got != want { - t.Errorf("Expected error '%s', got '%s'", want, got) - } -} diff --git a/storage.go b/storage.go index ebbaad9..8efe730 100644 --- a/storage.go +++ b/storage.go @@ -2,9 +2,7 @@ package checkup import ( "encoding/json" - "errors" "fmt" - "strings" "github.com/sourcegraph/checkup/storage/fs" "github.com/sourcegraph/checkup/storage/github" @@ -14,32 +12,15 @@ import ( func storageDecode(typeName string, config json.RawMessage) (Storage, error) { switch typeName { - case "s3": + case s3.Type: return s3.New(config) - case "github": + case github.Type: return github.New(config) - case "fs": + case fs.Type: return fs.New(config) - case "sql": + case sql.Type: return sql.New(config) default: - return nil, errors.New(strings.Replace(errUnknownStorageType, "%T", typeName, -1)) + return nil, fmt.Errorf(errUnknownStorageType, typeName) } } - -func storageType(ch interface{}) (string, error) { - var typeName string - switch ch.(type) { - case s3.Storage, *s3.Storage: - typeName = "s3" - case github.Storage, *github.Storage: - typeName = "github" - case fs.Storage, *fs.Storage: - typeName = "fs" - case sql.Storage, *sql.Storage: - typeName = "sql" - default: - return "", fmt.Errorf(errUnknownStorageType, ch) - } - return typeName, nil -} diff --git a/storage/fs/fs.go b/storage/fs/fs.go index 2974873..684c5b5 100644 --- a/storage/fs/fs.go +++ b/storage/fs/fs.go @@ -10,6 +10,9 @@ import ( "github.com/sourcegraph/checkup/types" ) +// Type should match the package name +const Type = "fs" + // Storage is a way to store checkup results on the local filesystem. type Storage struct { // The path to the directory where check files will be stored. @@ -31,6 +34,11 @@ func New(config json.RawMessage) (Storage, error) { return storage, err } +// Type returns the storage driver package name +func (Storage) Type() string { + return Type +} + // GetIndex returns the index from filesystem. func (fs Storage) GetIndex() (map[string]int64, error) { return fs.readIndex() diff --git a/storage/github/github.go b/storage/github/github.go index 19d2661..312d025 100644 --- a/storage/github/github.go +++ b/storage/github/github.go @@ -17,6 +17,9 @@ import ( "github.com/sourcegraph/checkup/types" ) +// Type should match the package name +const Type = "github" + var errFileNotFound = fmt.Errorf("file not found on github") // Storage is a way to store checkup results in a GitHub repository. @@ -63,6 +66,11 @@ func New(config json.RawMessage) (*Storage, error) { return storage, err } +// Type returns the storage driver package name +func (Storage) Type() string { + return Type +} + // ensureClient builds an GitHub API client if none exists and stores it on the struct. func (gh *Storage) ensureClient() error { if gh.client != nil { diff --git a/storage/s3/s3.go b/storage/s3/s3.go index 6cb343d..23c1e15 100644 --- a/storage/s3/s3.go +++ b/storage/s3/s3.go @@ -17,6 +17,9 @@ import ( "github.com/sourcegraph/checkup/types" ) +// Type should match the package name +const Type = "s3" + // Storage is a way to store checkup results in an S3 bucket. type Storage struct { AccessKeyID string `json:"access_key_id"` @@ -38,6 +41,11 @@ func New(config json.RawMessage) (Storage, error) { return storage, err } +// Type returns the storage driver package name +func (Storage) Type() string { + return Type +} + // Store stores results on S3 according to the configuration in s. func (s Storage) Store(results []types.Result) error { jsonBytes, err := json.Marshal(results) diff --git a/storage/sql/sql.go b/storage/sql/sql.go index 9e90a1a..b9ac2d4 100644 --- a/storage/sql/sql.go +++ b/storage/sql/sql.go @@ -54,6 +54,11 @@ func New(config json.RawMessage) (Storage, error) { return storage, err } +// Type returns the storage driver package name +func (Storage) Type() string { + return Type +} + func (sql Storage) dbConnect() (*sqlx.DB, error) { // Only one SQL backend can be present if sql.SqliteDBFile != "" && sql.PostgreSQL != nil { diff --git a/storage/sql/sql_disabled.go b/storage/sql/sql_disabled.go index 2d19b72..b4ea6ae 100644 --- a/storage/sql/sql_disabled.go +++ b/storage/sql/sql_disabled.go @@ -16,6 +16,11 @@ func New(_ json.RawMessage) (Storage, error) { return Storage{}, errors.New("sql data store is disabled") } -func (_ Storage) Store(results []types.Result) error { +// Type returns the storage driver package name +func (Storage) Type() string { + return Type +} + +func (Storage) Store(results []types.Result) error { return errors.New("sql data store is disabled") } diff --git a/storage/sql/types.go b/storage/sql/types.go new file mode 100644 index 0000000..e0dcca2 --- /dev/null +++ b/storage/sql/types.go @@ -0,0 +1,4 @@ +package sql + +// Type should match the package name +const Type = "sql" diff --git a/storage_test.go b/storage_test.go deleted file mode 100644 index 94386d9..0000000 --- a/storage_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package checkup - -import ( - "strings" - "testing" -) - -func TestUnknownStorageType(t *testing.T) { - kind, err := storageType("") - if got, want := kind, ""; got != want { - t.Errorf("Expected type '%s', got '%s'", want, got) - } - want := strings.Replace(errUnknownStorageType, "%T", "string", -1) - if got := err.Error(); got != want { - t.Errorf("Expected error '%s', got '%s'", want, got) - } -} From 687e65d3f89d8fe8e48f9f97c1d6e7f7127369ba Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 12:42:52 +0200 Subject: [PATCH 07/12] feature: add mail notifier and docs --- README.md | 19 ++++++++++ checkup_test.go | 4 ++ errors.go | 6 +-- go.mod | 1 + go.sum | 2 + notifier/mail/mail.go | 86 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 notifier/mail/mail.go diff --git a/README.md b/README.md index b96e489..035b678 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,25 @@ Enable notifications in Slack with this Notifier configuration: Follow these instructions to [create a webhook](https://get.slack.help/hc/en-us/articles/115005265063-Incoming-WebHooks-for-Slack). +#### Mail notifier + +Enable E-mail notifications with this Notifier configuration: +```js +{ + "from": "from@example.com", + "to": [ "support1@examiple.com", "support2@example.com" ], + "subject": "Custom subject line", + "smtp": { + "server": "smtp.example.com", + "port": 25, + "username": "username", + "password": "password" + } +} +``` + +The settings for `subject`, `smtp.port` (default to 25), `smtp.username` and `smtp.password` are optional. + ## Setting up storage on S3 The easiest way to do this is to give an IAM user these two privileges (keep the credentials secret): diff --git a/checkup_test.go b/checkup_test.go index 5fe3b4c..5d97ec8 100644 --- a/checkup_test.go +++ b/checkup_test.go @@ -207,6 +207,10 @@ type fake struct { notified int } +func (f *fake) Type() string { + return "fake" +} + func (f *fake) Check() (types.Result, error) { f.Lock() defer f.Unlock() diff --git a/errors.go b/errors.go index 961a3aa..ee339dd 100644 --- a/errors.go +++ b/errors.go @@ -1,7 +1,7 @@ package checkup const ( - errUnknownCheckerType = "unknown checker type: %T" - errUnknownStorageType = "unknown storage type: %T" - errUnknownNotifierType = "unknown notifier type: %T" + errUnknownCheckerType = "unknown checker type: %s" + errUnknownStorageType = "unknown storage type: %s" + errUnknownNotifierType = "unknown notifier type: %s" ) diff --git a/go.mod b/go.mod index b7425a7..7b4a307 100644 --- a/go.mod +++ b/go.mod @@ -17,5 +17,6 @@ require ( github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v0.0.7 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d + gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df moul.io/http2curl v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 095152c..bf23a4e 100644 --- a/go.sum +++ b/go.sum @@ -203,6 +203,8 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/notifier/mail/mail.go b/notifier/mail/mail.go new file mode 100644 index 0000000..430e491 --- /dev/null +++ b/notifier/mail/mail.go @@ -0,0 +1,86 @@ +package mail + +import ( + "encoding/json" + "fmt" + "strings" + + "gopkg.in/gomail.v2" + + "github.com/sourcegraph/checkup/types" +) + +// Type should match the package name +const Type = "mail" + +// Notifier consist of all the sub components required to send E-mail notifications +type Notifier struct { + // From contains the e-mail address notifications are sent from + From string `json:"from"` + + // To contains a list of e-mail address destinations + To []string `json:"to"` + + // Subject contains customizable subject line + Subject string `json:"subject,omitempty"` + + // SMTP contains all relevant mail server settings + SMTP struct { + Server string `json:"server"` + Port int `json:"port,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + } `json:"smtp"` +} + +// New creates a new Notifier instance based on json config +func New(config json.RawMessage) (Notifier, error) { + var notifier Notifier + err := json.Unmarshal(config, ¬ifier) + // Fall back to port 25 if not defined + if notifier.SMTP.Port == 0 { + notifier.SMTP.Port = 25 + } + if strings.TrimSpace(notifier.Subject) == "" { + notifier.Subject = "Checkup: Service Unavailable" + } + return notifier, err +} + +// Type returns the notifier package name +func (Notifier) Type() string { + return Type +} + +// Notify implements notifier interface +func (m Notifier) Notify(results []types.Result) error { + issues := []types.Result{} + for _, result := range results { + if !result.Healthy { + issues = append(issues, result) + } + } + + if len(issues) == 0 { + return nil + } + + message := gomail.NewMessage() + message.SetHeader("From", m.From) + message.SetHeader("To", m.To...) + message.SetHeader("Subject", m.Subject) + message.SetBody("text/html", renderMessage(issues)) + + dialer := gomail.NewDialer(m.SMTP.Server, m.SMTP.Port, m.SMTP.Username, m.SMTP.Password) + return dialer.DialAndSend(message) +} + +func renderMessage(issues []types.Result) string { + body := []string{"Checkup has detected the following issues:", "

", "
    "} + for _, issue := range issues { + format := "
  • %s - Status %s
    • " + body = append(body, fmt.Sprintf(format, issue.Title, issue.Status())) + } + body = append(body, "
") + return strings.Join(body, "\n") +} From 31bd327b65a2bfb0614b42feb11a97ddd67a9a4d Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 12:43:08 +0200 Subject: [PATCH 08/12] Makefile: tune test verbosity to non-verbose --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6e4fb0d..6d61a2c 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ build: go build -o builds/ ./cmd/... test: - go test -race -count=1 -v ./... + go test -race -count=1 ./... docker: docker build --no-cache . -t $(DOCKER_IMAGE) From 19203b6dc0dd6a104d4e5ca108da5feb172fca12 Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 12:46:09 +0200 Subject: [PATCH 09/12] checkup: Fix type names in configs for consistency This commit fixes the issue of inconsistent type detection over storage (had 'provider'), notifier (had 'name') and check (had 'type') - not it always uses 'type'. This commit also adds support for multiple checkers via in checkup.json. --- checkup.go | 97 +++++++++++++++++++++++++------------------------ checkup_test.go | 6 +-- 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/checkup.go b/checkup.go index fcf1fdd..bc7b44f 100644 --- a/checkup.go +++ b/checkup.go @@ -38,11 +38,11 @@ type Checkup struct { // method will be called by c.CheckAndStore(). Storage Storage `json:"storage,omitempty"` - // Notifier is a notifier that will be passed the - // results after checks from all checkers have + // Notifiers are list of notifiers to invoke with + // the results after checks from all checkers have // completed. Notifier may evaluate and choose to // send a notification of potential problems. - Notifier Notifier `json:"notifier,omitempty"` + Notifiers []Notifier `json:"notifiers,omitempty"` } // Check performs the health checks. An error is only @@ -83,12 +83,11 @@ func (c Checkup) Check() ([]types.Result, error) { return results, errs } - if c.Notifier != nil { - err := c.Notifier.Notify(results) + for _, service := range c.Notifiers { + err := service.Notify(results) if err != nil { - log.Printf("ERROR sending notifications: %s", err) + log.Printf("ERROR sending notifications for %s: %s", service.Type(), err) } - return results, nil } return results, nil @@ -172,13 +171,7 @@ func (c Checkup) MarshalJSON() ([]byte, error) { if err != nil { return result, err } - - typeName, err := checkerType(ch) - if err != nil { - return result, err - } - - chb = []byte(fmt.Sprintf(`{"type":"%s",%s`, typeName, string(chb[1:]))) + chb = []byte(fmt.Sprintf(`{"type":"%s",%s`, ch.Type(), string(chb[1:]))) checkers = append(checkers, chb) } @@ -194,30 +187,27 @@ func (c Checkup) MarshalJSON() ([]byte, error) { if err != nil { return result, err } - - providerName, err := storageType(c.Storage) - if err != nil { - return result, err - } - - sb = []byte(fmt.Sprintf(`{"provider":"%s",%s`, providerName, string(sb[1:]))) + sb = []byte(fmt.Sprintf(`{"type":"%s",%s`, c.Storage.Type(), string(sb[1:]))) wrap("storage", sb) } - // Notifier - if c.Notifier != nil { - nb, err := json.Marshal(c.Notifier) - if err != nil { - return result, err - } + // Notifiers + if len(c.Notifiers) > 0 { + var checkers [][]byte + for _, ch := range c.Notifiers { + chb, err := json.Marshal(ch) + if err != nil { + return result, err + } - notifierName, err := notifierType(c.Notifier) - if err != nil { - return result, err + chb = []byte(fmt.Sprintf(`{"type":"%s",%s`, ch.Type(), string(chb[1:]))) + checkers = append(checkers, chb) } - nb = []byte(fmt.Sprintf(`{"name":"%s",%s`, notifierName, string(nb[1:]))) - wrap("notifier", nb) + allNotifiers := []byte{'['} + allNotifiers = append([]byte{'['}, bytes.Join(checkers, []byte(","))...) + allNotifiers = append(allNotifiers, ']') + wrap("notifiers", allNotifiers) } return result, nil @@ -233,14 +223,18 @@ func (c *Checkup) UnmarshalJSON(b []byte) error { // but we can ignore it because we handle it below. type checkup2 *Checkup json.Unmarshal(b, checkup2(c)) - c.Checkers = []Checker{} // clean the slate + + // clean the slate + c.Checkers = []Checker{} + c.Notifiers = []Notifier{} // Begin unmarshaling interface values by // collecting the raw JSON raw := struct { - Checkers []json.RawMessage `json:"checkers"` - Storage json.RawMessage `json:"storage"` - Notifier json.RawMessage `json:"notifier"` + Checkers []json.RawMessage `json:"checkers"` + Storage json.RawMessage `json:"storage"` + Notifier json.RawMessage `json:"notifier"` + Notifiers []json.RawMessage `json:"notifiers"` }{} err := json.Unmarshal([]byte(b), &raw) if err != nil { @@ -248,28 +242,28 @@ func (c *Checkup) UnmarshalJSON(b []byte) error { } // Then collect the concrete type information - types := struct { + configTypes := struct { Checkers []struct { Type string `json:"type"` } Storage struct { - Provider string `json:"provider"` + Type string `json:"type"` } Notifier struct { - Name string `json:"name"` - Username string `json:"username"` - Channel string `json:"channel"` - Webhook string `json:"webhook"` + Type string `json:"type"` + } + Notifiers []struct { + Type string `json:"type"` } }{} - err = json.Unmarshal([]byte(b), &types) + err = json.Unmarshal([]byte(b), &configTypes) if err != nil { return err } // Finally, we unmarshal the remaining values using type // assertions with the help of the type information - for i, t := range types.Checkers { + for i, t := range configTypes.Checkers { checker, err := checkerDecode(t.Type, raw.Checkers[i]) if err != nil { return err @@ -277,20 +271,27 @@ func (c *Checkup) UnmarshalJSON(b []byte) error { c.Checkers = append(c.Checkers, checker) } if raw.Storage != nil { - storage, err := storageDecode(types.Storage.Provider, raw.Storage) + storage, err := storageDecode(configTypes.Storage.Type, raw.Storage) if err != nil { return err } c.Storage = storage } if raw.Notifier != nil { - notifier, err := notifierDecode(types.Notifier.Name, raw.Notifier) + notifier, err := notifierDecode(configTypes.Notifier.Type, raw.Notifier) if err != nil { return err } - c.Notifier = notifier + // Move `notifier` into `notifiers[]` + c.Notifiers = append(c.Notifiers, notifier) + } + for i, n := range configTypes.Notifiers { + notifier, err := notifierDecode(n.Type, raw.Notifiers[i]) + if err != nil { + return err + } + c.Notifiers = append(c.Notifiers, notifier) } - return nil } diff --git a/checkup_test.go b/checkup_test.go index 5d97ec8..4560a9c 100644 --- a/checkup_test.go +++ b/checkup_test.go @@ -18,7 +18,7 @@ func TestCheckAndStore(t *testing.T) { Checkers: []Checker{f, f}, ConcurrentChecks: 1, Timestamp: time.Now(), - Notifier: f, + Notifiers: []Notifier{f, f}, } err := c.CheckAndStore() @@ -36,7 +36,7 @@ func TestCheckAndStore(t *testing.T) { t.Error("Expected timestamps to be the same, but they weren't") } } - if got, want := f.notified, 1; got != want { + if got, want := f.notified, 2; got != want { t.Errorf("Expected Notify() to be called %d time, called %d times", want, got) } if got, want := f.maintained, 1; got != want { @@ -177,7 +177,7 @@ func TestPriorityOver(t *testing.T) { } func TestJSON(t *testing.T) { - jsonBytes := []byte(`{"storage":{"provider":"s3","access_key_id":"AAAAAA6WVZYYANEAFL6Q","secret_access_key":"DbvNDdKHaN4n8n3qqqXwvUVqVQTcHVmNYtvcJfTd","region":"us-east-1","bucket":"test","check_expiry":604800000000000},"checkers":[{"type":"http","endpoint_name":"Example (HTTP)","endpoint_url":"http://www.example.com","attempts":5},{"type":"http","endpoint_name":"Example (HTTPS)","endpoint_url":"https://example.com","threshold_rtt":500000000,"attempts":5},{"type":"http","endpoint_name":"localhost","endpoint_url":"http://localhost:2015","threshold_rtt":1000000,"attempts":5}],"timestamp":"0001-01-01T00:00:00Z"}`) + jsonBytes := []byte(`{"storage":{"type":"s3","access_key_id":"AAAAAA6WVZYYANEAFL6Q","secret_access_key":"DbvNDdKHaN4n8n3qqqXwvUVqVQTcHVmNYtvcJfTd","region":"us-east-1","bucket":"test","check_expiry":604800000000000},"checkers":[{"type":"http","endpoint_name":"Example (HTTP)","endpoint_url":"http://www.example.com","attempts":5},{"type":"http","endpoint_name":"Example (HTTPS)","endpoint_url":"https://example.com","threshold_rtt":500000000,"attempts":5},{"type":"http","endpoint_name":"localhost","endpoint_url":"http://localhost:2015","threshold_rtt":1000000,"attempts":5}],"timestamp":"0001-01-01T00:00:00Z"}`) var c Checkup err := json.Unmarshal(jsonBytes, &c) From fdb460863866f1f651d6708bee278c6bdbc0442c Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 13:01:58 +0200 Subject: [PATCH 10/12] sql storage: fix compile error, add build-sql target into makefile --- Makefile | 6 +++++- storage/sql/sql.go | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6d61a2c..9a13e01 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build test docker +.PHONY: all build build-sql test docker all: build test @@ -9,6 +9,10 @@ build: mkdir -p builds/ go build -o builds/ ./cmd/... +build-sql: + go fmt ./... + go build -o builds/ -tags sql ./cmd/... + test: go test -race -count=1 ./... diff --git a/storage/sql/sql.go b/storage/sql/sql.go index b9ac2d4..fb97367 100644 --- a/storage/sql/sql.go +++ b/storage/sql/sql.go @@ -12,6 +12,7 @@ import ( _ "github.com/lib/pq" // Enable postgresql beckend _ "github.com/mattn/go-sqlite3" // Enable sqlite3 backend + "github.com/sourcegraph/checkup/storage/fs" "github.com/sourcegraph/checkup/types" ) @@ -160,7 +161,7 @@ func (sql Storage) Store(results []types.Result) error { } defer db.Close() - name := *GenerateFilename() + name := *fs.GenerateFilename() contents, err := json.Marshal(results) if err != nil { return err From f5a941e22f23b1505f0e89ab049dc0a5a1e951c9 Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 13:08:58 +0200 Subject: [PATCH 11/12] documentation: update to reflect recent changes and additions --- README.md | 78 +++++++++++++++++++++++++++---------------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 035b678..398246a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,20 @@ Checkup was created by Matt Holt, author of the [Caddy web server](https://caddy This tool is a work-in-progress. Please use liberally (with discretion) and report any bugs! +## Recent changes +Due to recent development, some breaking changes have been introduced: + +- providers: the json config field `provider` was renamed to `type` for consistency, +- notifiers: the json config field `name` was renamed to `type` for consistency, +- sql: by default the sqlite storage engine is disabled (needs build with `-tags sql` to enable), + +If you want to build the latest version, it's best to run: + +- `make build` - builds checkup without sql support, +- `make build-sql` - builds checkup with pgsql/sqlite; + +The resulting binary will be placed into `builds/checkup`. ## Intro @@ -76,9 +89,9 @@ You can configure Checkup entirely with a simple JSON document. You should confi // storage configuration goes here }, - "notifier": { + "notifiers": [ // notifier configuration goes here - } + ] } ``` @@ -90,7 +103,7 @@ Here are the configuration structures you can use, which are explained fully [in #### HTTP Checkers -**[godoc: HTTPChecker](https://godoc.org/github.com/sourcegraph/checkup#HTTPChecker)** +**[godoc: HTTPChecker](https://godoc.org/github.com/sourcegraph/checkup/check/http)** ```js { @@ -104,7 +117,7 @@ Here are the configuration structures you can use, which are explained fully [in #### TCP Checkers -**[godoc: TCPChecker](https://godoc.org/github.com/sourcegraph/checkup#TCPChecker)** +**[godoc: TCPChecker](https://godoc.org/github.com/sourcegraph/checkup/check/tcp)** ```js { @@ -116,7 +129,7 @@ Here are the configuration structures you can use, which are explained fully [in #### DNS Checkers -**[godoc: DNSChecker](https://godoc.org/github.com/sourcegraph/checkup#DNSChecker)** +**[godoc: DNSChecker](https://godoc.org/github.com/sourcegraph/checkup/check/dns)** ```js { @@ -129,7 +142,7 @@ Here are the configuration structures you can use, which are explained fully [in #### TLS Checkers -**[godoc: TLSChecker](https://godoc.org/github.com/sourcegraph/checkup#TLSChecker)** +**[godoc: TLSChecker](https://godoc.org/github.com/sourcegraph/checkup/check/tls)** ```js { @@ -142,11 +155,11 @@ Here are the configuration structures you can use, which are explained fully [in #### Amazon S3 Storage -**[godoc: S3](https://godoc.org/github.com/sourcegraph/checkup#S3)** +**[godoc: S3](https://godoc.org/github.com/sourcegraph/checkup/check/s3)** ```js { - "provider": "s3", + "type": "s3", "access_key_id": "", "secret_access_key": "", "bucket": "", @@ -159,11 +172,11 @@ S3 is the default storage provider assumed by the status page, so the only chang #### File System Storage -**[godoc: FS](https://godoc.org/github.com/sourcegraph/checkup#FS)** +**[godoc: FS](https://godoc.org/github.com/sourcegraph/checkup/storage/fs)** ```js { - "provider": "fs", + "type": "fs", "dir": "/path/to/your/check_files", "url": "http://127.0.0.1:2015/check_files" } @@ -180,11 +193,11 @@ Then fill out [config.js](https://github.com/sourcegraph/checkup/blob/master/sta #### GitHub Storage -**[godoc: GitHub](https://godoc.org/github.com/sourcegraph/checkup#GitHub)** +**[godoc: GitHub](https://godoc.org/github.com/sourcegraph/checkup/storage/github)** ```js { - "provider": "github", + "type": "github", "access_token": "some_api_access_token_with_repo_scope", "repository_owner": "owner", "repository_name": "repo", @@ -209,14 +222,14 @@ Where "dir" is a subdirectory within the repo to push all the check files. Setup #### SQL Storage (sqlite3/PostgreSQL) -**[godoc: SQL](https://godoc.org/github.com/sourcegraph/checkup#SQL)** +**[godoc: SQL](https://godoc.org/github.com/sourcegraph/checkup/storage/sql)** Postgres or sqlite3 databases can be used as storage backends. sqlite database file configuration: ```js { - "provider": "sql", + "type": "sql", "sqlite_db_file": "/path/to/your/sqlite.db" } ``` @@ -224,7 +237,7 @@ sqlite database file configuration: postgresql database file configuration: ```js { - "provider": "sql", + "type": "sql", "postgresql": { "user": "postgres", "dbname": "dbname", @@ -251,7 +264,7 @@ Currently the status page does not support SQL storage. Enable notifications in Slack with this Notifier configuration: ```js { - "name": "slack", + "type": "slack", "username": "username", "channel": "#channel-name", "webhook": "webhook-url" @@ -265,6 +278,7 @@ Follow these instructions to [create a webhook](https://get.slack.help/hc/en-us/ Enable E-mail notifications with this Notifier configuration: ```js { + "type": "mail", "from": "from@example.com", "to": [ "support1@examiple.com", "support2@example.com" ], "subject": "Custom subject line", @@ -472,34 +486,18 @@ You can implement your own Checker and Storage types. If it's general enough, fe ### Building Locally -Requires Go v1.10 or newer. +Requires Go v1.13 or newer. ```bash git clone git@github.com:sourcegraph/checkup.git -cd checkup/cmd/checkup/ - -# Install dependencies -go get -v -d - -# Build binary -go build -v -ldflags '-s' -o ../../checkup - -# Run tests -go test -race ../../ +cd checkup +make ``` -### Building with Docker +Building the SQL enabled version is done with `make build-sql`. -Linux binary: - -```bash -git clone git@github.com:sourcegraph/checkup.git -cd checkup -docker pull golang:latest -docker run --net=host --rm \ --v `pwd`:/project \ --w /project golang bash \ --c "cd cmd/checkup; go get -v -d; go build -v -ldflags '-s' -o ../../checkup" -``` +### Building a Docker image -This will create a checkup binary in the root project folder. +If you would like to run checkup in a docker container, building it is done by running `make docker`. +It will build the version without sql support. An SQL supported docker image is currently not provided, +but there's a plan to do that in the future. From b17f4c18c5482bd675f06ff243cd0c7cb7ceb3f0 Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Fri, 10 Apr 2020 13:24:22 +0200 Subject: [PATCH 12/12] chore: added go mod tidy as a build step to clean up go.mod/sum files --- Makefile | 2 ++ go.mod | 1 + go.sum | 2 ++ 3 files changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 9a13e01..7bbb2a3 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,13 @@ DOCKER_IMAGE := checkup build: go fmt ./... + go mod tidy mkdir -p builds/ go build -o builds/ ./cmd/... build-sql: go fmt ./... + go mod tidy go build -o builds/ -tags sql ./cmd/... test: diff --git a/go.mod b/go.mod index 7b4a307..0e01477 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v0.0.7 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df moul.io/http2curl v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index bf23a4e..80b2de4 100644 --- a/go.sum +++ b/go.sum @@ -200,6 +200,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=