Split out detector types into separate proto file in order to narrow CODEOWNERS scope (#4871)

We want to move the detector types out of the Scanning team purview. So I split off detector types into its own proto file (so that file detector_type.proto can be owned by the Integrations team), regenerated the pb files with "make protos", and made the detector files use the new generated detector_type.pb.go. Included the new detector_type.proto file in CODEOWNERS and made CODEOWNERS categories that contain larger teams be towards the top so that more fine grained ownership is filtered properly.

Author: casey-tran

CODEOWNERS

@@ -1,6 +1,16 @@
 # catch-all
 * @trufflesecurity/product-eng
 
+# Shared
+pkg/decoders/ @trufflesecurity/Scanning @trufflesecurity/OSS
+pkg/engine/ @trufflesecurity/Scanning  @trufflesecurity/OSS
+pkg/gitparse/ @trufflesecurity/Scanning  @trufflesecurity/OSS
+pkg/giturl/ @trufflesecurity/Scanning  @trufflesecurity/OSS
+pkg/handlers/ @trufflesecurity/Scanning  @trufflesecurity/OSS
+pkg/iobuf/ @trufflesecurity/Scanning  @trufflesecurity/OSS
+pkg/sanitizer/ @trufflesecurity/Scanning  @trufflesecurity/OSS
+proto/ @trufflesecurity/Scanning  @trufflesecurity/Integrations
+
 # Scanning
 pkg/sources/ @trufflesecurity/Scanning
 pkg/writers/ @trufflesecurity/Scanning
@@ -18,16 +28,7 @@ pkg/sources/jenkins/ @trufflesecurity/Integrations
 pkg/sources/postman/ @trufflesecurity/Integrations
 pkg/sources/s3/ @trufflesecurity/Integrations
 pkg/sources/travisci/ @trufflesecurity/Integrations
-
-# Shared
-pkg/decoders/ @trufflesecurity/Scanning @trufflesecurity/OSS
-pkg/engine/ @trufflesecurity/Scanning  @trufflesecurity/OSS
-pkg/gitparse/ @trufflesecurity/Scanning  @trufflesecurity/OSS
-pkg/giturl/ @trufflesecurity/Scanning  @trufflesecurity/OSS
-pkg/handlers/ @trufflesecurity/Scanning  @trufflesecurity/OSS
-pkg/iobuf/ @trufflesecurity/Scanning  @trufflesecurity/OSS
-pkg/sanitizer/ @trufflesecurity/Scanning  @trufflesecurity/OSS
-proto/ @trufflesecurity/Scanning  @trufflesecurity/Integrations
+proto/detector_type.proto @trufflesecurity/Integrations
 
 # OSS
 pkg/detectors/ @trufflesecurity/OSS

hack/docs/Adding_Detectors_Internal.md

@@ -63,7 +63,7 @@ Note: Be sure to update the tests to reference the new secret values in GSM, or
 
 ### Creating a new Secret Scanner
 
-1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum.
+1. Identify the Secret Detector name from the [/proto/detector_type.proto](/proto/detector_type.proto) `DetectorType` enum.
 
 2. Generate the Secret Detector
 
@@ -154,7 +154,7 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
    ```bash
    dos2unix ./scripts/gen_proto.sh
    ```
-5. Open [/proto/detectors.proto](/proto/detectors.proto) file and add new detectors then save it. Make sure Docker is running and run this in Ubuntu command line.
+5. Open [/proto/detector_type.proto](/proto/detector_type.proto) file and add new detectors then save it. Make sure Docker is running and run this in Ubuntu command line.
    ```bash
    make protos
    ```

hack/docs/Adding_Detectors_external.md

@@ -59,7 +59,7 @@ Note: Be sure to update the tests to reference the new secret values in GSM, or
 
 ### Creating a new Secret Detector
 
-1. Add a new Secret Detector enum to the [`DetectorType` list here](/proto/detectors.proto).
+1. Add a new Secret Detector enum to the [`DetectorType` list here](/proto/detector_type.proto).
 
 2. Run `make protos` to update the `.pb` files.
 
@@ -71,7 +71,7 @@ Note: Be sure to update the tests to reference the new secret values in GSM, or
    ```
 4. Add the Secret Detector to TruffleHog's Default Detectors
 
-   Add the secret scanner to the [`pkg/engine/defaults/defaults.go`](https://github.com/trufflesecurity/trufflehog/blob/main/pkg/engine/defaults/defaults.go) file like [`github.com/trufflesecurity/trufflehog/v3/pkg/detectors/<detector_name>`](https://github.com/trufflesecurity/trufflehog/blob/b71ea27a696bdf1c3141f637fda4ee4936c2f2d6/pkg/engine/defaults/defaults.go#L9) and 
+   Add the secret scanner to the [`pkg/engine/defaults/defaults.go`](https://github.com/trufflesecurity/trufflehog/blob/main/pkg/engine/defaults/defaults.go) file like [`github.com/trufflesecurity/trufflehog/v3/pkg/detectors/<detector_name>`](https://github.com/trufflesecurity/trufflehog/blob/b71ea27a696bdf1c3141f637fda4ee4936c2f2d6/pkg/engine/defaults/defaults.go#L9) and
    [`<detector_name>.Scanner{}`](https://github.com/trufflesecurity/trufflehog/blob/b71ea27a696bdf1c3141f637fda4ee4936c2f2d6/pkg/engine/defaults/defaults.go#L1546)
 
 5. Complete the Secret Detector.
@@ -117,15 +117,15 @@ To ensure the quality of your PR, make sure your tests are passing with verified
    4. Found and unverified (indeterminately due to an unexpected API response)
    5. Not found
 
-    Make any necessary updates to the tests. Note there might not be any changes required as the tests generated by the `go run hack/generate/generate.go` command are pretty good. 
+    Make any necessary updates to the tests. Note there might not be any changes required as the tests generated by the `go run hack/generate/generate.go` command are pretty good.
    [Here is an exemplary test file for a detector which covers all 5 test cases](https://github.com/trufflesecurity/trufflehog/blob/6f9065b0aae981133a7fa3431c17a5c6213be226/pkg/detectors/browserstack/browserstack_test.go).
 
 4. Now run the tests and check to make sure they are passing ✔️!
 ```bash
    go test ./pkg/detectors/<detector> -tags=detectors
    ```
 
-If the tests are passing, feel free to open a PR! 
+If the tests are passing, feel free to open a PR!
 
 
 
@@ -157,8 +157,7 @@ For example, consider a hypothetical authentication endpoint that returns `200 O
    ```bash
    dos2unix ./scripts/gen_proto.sh
    ```
-5. Open [/proto/detectors.proto](/proto/detectors.proto) file and add new detectors then save it. Make sure Docker is running and run this in Ubuntu command line.
+5. Open [/proto/detector_type.proto](/proto/detector_type.proto) file and add new detectors then save it. Make sure Docker is running and run this in Ubuntu command line.
    ```bash
    make protos
    ```
-

pkg/config/detectors.go

@@ -8,23 +8,23 @@ import (
 	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
 )
 
 var (
 	specialGroups = map[string][]DetectorID{
 		"all": allDetectors(),
 	}
 
-	detectorTypeValue = make(map[string]dpb.DetectorType, len(dpb.DetectorType_value))
-	validDetectors    = make(map[dpb.DetectorType]struct{}, len(dpb.DetectorType_value))
-	maxDetectorType   dpb.DetectorType
+	detectorTypeValue = make(map[string]detector_typepb.DetectorType, len(detector_typepb.DetectorType_value))
+	validDetectors    = make(map[detector_typepb.DetectorType]struct{}, len(detector_typepb.DetectorType_value))
+	maxDetectorType   detector_typepb.DetectorType
 )
 
 // Setup package local global variables.
 func init() {
-	for k, v := range dpb.DetectorType_value {
-		dt := dpb.DetectorType(v)
+	for k, v := range detector_typepb.DetectorType_value {
+		dt := detector_typepb.DetectorType(v)
 		detectorTypeValue[strings.ToLower(k)] = dt
 		validDetectors[dt] = struct{}{}
 		if dt > maxDetectorType {
@@ -37,7 +37,7 @@ func init() {
 // way for users to identify detectors, whether unique or not. A DetectorID
 // with Version = 0 indicates all possible versions of a detector.
 type DetectorID struct {
-	ID      dpb.DetectorType
+	ID      detector_typepb.DetectorType
 	Version int
 }
 
@@ -117,7 +117,7 @@ func ParseVerifierEndpoints(verifierURLs map[string]string) (map[DetectorID][]st
 }
 
 func (id DetectorID) String() string {
-	name := dpb.DetectorType_name[int32(id.ID)]
+	name := detector_typepb.DetectorType_name[int32(id.ID)]
 	if name == "" {
 		name = "<invalid ID>"
 	}
@@ -129,9 +129,9 @@ func (id DetectorID) String() string {
 
 // allDetectors returns an ordered slice of all detector types.
 func allDetectors() []DetectorID {
-	all := make([]DetectorID, 0, len(dpb.DetectorType_name))
-	for id := range dpb.DetectorType_name {
-		all = append(all, DetectorID{ID: dpb.DetectorType(id)})
+	all := make([]DetectorID, 0, len(detector_typepb.DetectorType_name))
+	for id := range detector_typepb.DetectorType_name {
+		all = append(all, DetectorID{ID: detector_typepb.DetectorType(id)})
 	}
 	sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID })
 	return all
@@ -173,7 +173,7 @@ func asRange(input string) ([]DetectorID, error) {
 		return nil, fmt.Errorf("versions within ranges are not supported: %s", input)
 	}
 
-	step := dpb.DetectorType(1)
+	step := detector_typepb.DetectorType(1)
 	if dtStart.ID > dtEnd.ID {
 		step = -1
 	}
@@ -212,7 +212,7 @@ func asDetectorID(input string) (DetectorID, error) {
 	}
 	// Check if it's a detector ID.
 	if i, err := strconv.ParseInt(input, 10, 32); err == nil {
-		dt := dpb.DetectorType(i)
+		dt := detector_typepb.DetectorType(i)
 		if _, ok := validDetectors[dt]; !ok {
 			return DetectorID{}, fmt.Errorf("invalid detector ID: %s", input)
 		}

pkg/config/detectors_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
 )
 
 func TestDetectorParsing(t *testing.T) {
@@ -15,22 +15,22 @@ func TestDetectorParsing(t *testing.T) {
 		"all":                       {"AlL", allDetectors()},
 		"trailing range":            {"0-", allDetectors()},
 		"all after 1":               {"1-", allDetectors()[1:]},
-		"named and valid range":     {"aWs,8-9", []DetectorID{{ID: dpb.DetectorType_AWS}, {ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
+		"named and valid range":     {"aWs,8-9", []DetectorID{{ID: detector_typepb.DetectorType_AWS}, {ID: detector_typepb.DetectorType_Github}, {ID: detector_typepb.DetectorType_Gitlab}}},
 		"duplicate order preserved": {"9, 8, 9", []DetectorID{{ID: 9}, {ID: 8}}},
-		"named range":               {"github - gitlab", []DetectorID{{ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
+		"named range":               {"github - gitlab", []DetectorID{{ID: detector_typepb.DetectorType_Github}, {ID: detector_typepb.DetectorType_Gitlab}}},
 		"range preserved":           {"8-9, 7-10", []DetectorID{{ID: 8}, {ID: 9}, {ID: 7}, {ID: 10}}},
 		"reverse range":             {"9-8", []DetectorID{{ID: 9}, {ID: 8}}},
 		"range preserved with all":  {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
 		"empty list item":           {"8, ,9", []DetectorID{{ID: 8}, {ID: 9}}},
 		"invalid end range":         {"0-1337", nil},
 		"invalid name":              {"foo", nil},
 		"negative":                  {"-1", nil},
-		"github.v1":                 {"github.v1", []DetectorID{{ID: dpb.DetectorType_Github, Version: 1}}},
-		"gitlab.v100":               {"gitlab.v100", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 100}}},
+		"github.v1":                 {"github.v1", []DetectorID{{ID: detector_typepb.DetectorType_Github, Version: 1}}},
+		"gitlab.v100":               {"gitlab.v100", []DetectorID{{ID: detector_typepb.DetectorType_Gitlab, Version: 100}}},
 		"range with versions":       {"github.v2 - gitlab.v1", nil},
 		"invalid version no v":      {"gitlab.2", nil},
 		"invalid version no number": {"gitlab.github", nil},
-		"capital V is fine":         {"GiTlAb.V2", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 2}}},
+		"capital V is fine":         {"GiTlAb.V2", []DetectorID{{ID: detector_typepb.DetectorType_Gitlab, Version: 2}}},
 		"id number with version":    {"8.v2", []DetectorID{{ID: 8, Version: 2}}},
 	}
 

pkg/custom_detectors/custom_detectors.go

@@ -16,7 +16,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
 )
 
 // The maximum number of matches from one chunk. This const is used when
@@ -222,7 +222,7 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string
 	}
 
 	result := detectors.Result{
-		DetectorType: detectorspb.DetectorType_CustomRegex,
+		DetectorType: detector_typepb.DetectorType_CustomRegex,
 		DetectorName: c.GetName(),
 		ExtraData:    map[string]string{},
 	}
@@ -398,8 +398,8 @@ func permutateMatches(regexMatches map[string][][]string) []map[string][]string
 	return matches
 }
 
-func (c *CustomRegexWebhook) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_CustomRegex
+func (c *CustomRegexWebhook) Type() detector_typepb.DetectorType {
+	return detector_typepb.DetectorType_CustomRegex
 }
 
 const defaultDescription = "This is a user-defined detector with no description provided."

pkg/custom_detectors/custom_detectors_test.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/protoyaml"
 )
 
@@ -317,7 +317,7 @@ func TestDetectorValidations(t *testing.T) {
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_CustomRegex,
+					DetectorType: detector_typepb.DetectorType_CustomRegex,
 					DetectorName: "test",
 					Verified:     false,
 					Raw:          []byte("MyStr0ngP@ssword!"),
@@ -364,7 +364,7 @@ func TestDetectorValidations(t *testing.T) {
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_CustomRegex,
+					DetectorType: detector_typepb.DetectorType_CustomRegex,
 					DetectorName: "test",
 					Verified:     false,
 					Raw:          []byte("MyStrongPassword!"),
@@ -411,7 +411,7 @@ func TestDetectorValidations(t *testing.T) {
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_CustomRegex,
+					DetectorType: detector_typepb.DetectorType_CustomRegex,
 					DetectorName: "test",
 					Verified:     false,
 					Raw:          []byte("MyStrongPassword!"),
@@ -458,7 +458,7 @@ func TestDetectorValidations(t *testing.T) {
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_CustomRegex,
+					DetectorType: detector_typepb.DetectorType_CustomRegex,
 					DetectorName: "test",
 					Verified:     false,
 					Raw:          []byte("MyStr@ngP@ssword!"),
@@ -506,7 +506,7 @@ func TestDetectorValidations(t *testing.T) {
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_CustomRegex,
+					DetectorType: detector_typepb.DetectorType_CustomRegex,
 					DetectorName: "test",
 					Verified:     false,
 					Raw:          []byte("MyStrongP@ssword"),
@@ -554,7 +554,7 @@ func TestDetectorValidations(t *testing.T) {
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_CustomRegex,
+					DetectorType: detector_typepb.DetectorType_CustomRegex,
 					DetectorName: "test",
 					Verified:     false,
 					Raw:          []byte("mystrongp@ssword"),
@@ -590,7 +590,7 @@ func TestDetectorValidations(t *testing.T) {
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_CustomRegex,
+					DetectorType: detector_typepb.DetectorType_CustomRegex,
 					DetectorName: "test",
 					Verified:     false,
 					Raw:          []byte("c392c9837d69b44c764cbf260b-e6184MyStrongP@ssword"),

pkg/detectors/abstract/abstract.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
 )
 
 type Scanner struct {
@@ -51,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
-			DetectorType: detectorspb.DetectorType_Abstract,
+			DetectorType: detector_typepb.DetectorType_Abstract,
 			Raw:          []byte(resMatch),
 		}
 
@@ -92,8 +92,8 @@ func verifyAbstract(ctx context.Context, client *http.Client, resMatch string) (
 	}
 }
 
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Abstract
+func (s Scanner) Type() detector_typepb.DetectorType {
+	return detector_typepb.DetectorType_Abstract
 }
 
 func (s Scanner) Description() string {

pkg/detectors/abstract/abstract_integration_test.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
 )
 
 func TestAbstract_FromChunk(t *testing.T) {
@@ -49,7 +49,7 @@ func TestAbstract_FromChunk(t *testing.T) {
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_Abstract,
+					DetectorType: detector_typepb.DetectorType_Abstract,
 					Verified:     true,
 				},
 			},
@@ -65,7 +65,7 @@ func TestAbstract_FromChunk(t *testing.T) {
 			},
 			want: func() []detectors.Result {
 				r := detectors.Result{
-					DetectorType: detectorspb.DetectorType_Abstract,
+					DetectorType: detector_typepb.DetectorType_Abstract,
 					Verified:     false,
 				}
 				r.SetVerificationError(context.DeadlineExceeded)
@@ -83,7 +83,7 @@ func TestAbstract_FromChunk(t *testing.T) {
 			},
 			want: func() []detectors.Result {
 				r := detectors.Result{
-					DetectorType: detectorspb.DetectorType_Abstract,
+					DetectorType: detector_typepb.DetectorType_Abstract,
 					Verified:     false,
 				}
 				r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
@@ -101,7 +101,7 @@ func TestAbstract_FromChunk(t *testing.T) {
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_Abstract,
+					DetectorType: detector_typepb.DetectorType_Abstract,
 					Verified:     false,
 				},
 			},