runner: allow to create plugin-specific exceptions at rule level

README.md

@@ -158,6 +158,35 @@ Severity levels could be: `error`, `warning`, or `ok`.
 
 > Note: Grafana Labs enforces its own configuration for plugins submissions and your own config file can't change these rules.
 
+#### Excluding a plugin from an analyzer or rule
+
+It's also possible to exclude a specific plugin from an analyzer or a specific rule within an analyzer. This is useful when a particular check is not applicable to your plugin.
+
+To disable an entire analyzer for a plugin, add an `exceptions` list with the plugin ID.
+
+```yaml
+analyzers:
+  some-analyzer:
+    enabled: true
+    # This entire analyzer will be skipped for 'my-plugin-id'
+    exceptions:
+      - my-plugin-id
+```
+
+To disable a single rule for a plugin, add the `exceptions` list to the rule's configuration.
+
+```yaml
+analyzers:
+  some-analyzer:
+    rules:
+      some-rule:
+        enabled: true
+        # This rule will be skipped for 'my-plugin-id'
+        exceptions:
+          - my-plugin-id
+```
+
+
 ### Source code
 
 You can specify the location of the plugin source code to the validator with the `-sourceCodeUri` option. Doing so allows for additional [analyzers](#analyzers) to be run and for a more complete scan.

pkg/analysis/analysis.go

@@ -20,7 +20,7 @@ type Pass struct {
 	AnalyzerName string
 	RootDir      string
 	CheckParams  CheckParams
-	ResultOf     map[*Analyzer]interface{}
+	ResultOf     map[*Analyzer]any
 	Report       func(string, Diagnostic)
 }
 

pkg/runner/runner.go

@@ -2,6 +2,7 @@ package runner
 
 import (
 	"fmt"
+	"slices"
 
 	"github.com/grafana/plugin-validator/pkg/analysis"
 	"github.com/grafana/plugin-validator/pkg/logme"
@@ -29,8 +30,9 @@ type AnalyzerConfig struct {
 }
 
 type RuleConfig struct {
-	Enabled  *bool              `yaml:"enabled"`
-	Severity *analysis.Severity `yaml:"severity"`
+	Enabled    *bool              `yaml:"enabled"`
+	Severity   *analysis.Severity `yaml:"severity"`
+	Exceptions []string           `yaml:"exceptions"`
 }
 
 var defaultSeverity = analysis.Warning
@@ -54,11 +56,25 @@ func Check(
 	pass := &analysis.Pass{
 		RootDir:     params.ArchiveDir,
 		CheckParams: params,
-		ResultOf:    make(map[*analysis.Analyzer]interface{}),
-		Report: func(name string, d analysis.Diagnostic) {
-			// Collect all diagnostics for presenting at the end.
-			diagnostics[name] = append(diagnostics[name], d)
-		},
+		ResultOf:    make(map[*analysis.Analyzer]any),
+	}
+
+	pass.Report = func(ruleName string, d analysis.Diagnostic) {
+		// Check for exceptions at the rule level
+		if analyzerConfig, ok := cfg.Analyzers[pass.AnalyzerName]; ok {
+			if ruleConfig, ok := analyzerConfig.Rules[ruleName]; ok {
+				if slices.Contains(ruleConfig.Exceptions, pluginId) {
+					logme.DebugFln(
+						"Diagnostic for rule '%s' skipped for plugin '%s' due to a rule-level exception.",
+						ruleName,
+						pluginId,
+					)
+					return
+				}
+			}
+		}
+		// Collect all diagnostics for presenting at the end.
+		diagnostics[ruleName] = append(diagnostics[ruleName], d)
 	}
 
 	seen := make(map[*analysis.Analyzer]bool)
@@ -169,10 +185,8 @@ func initAnalyzers(
 
 func isExcepted(pluginId string, cfg *AnalyzerConfig) bool {
 	if len(pluginId) > 0 && cfg != nil && len(cfg.Exceptions) > 0 {
-		for _, exception := range cfg.Exceptions {
-			if exception == pluginId {
-				return true
-			}
+		if slices.Contains(cfg.Exceptions, pluginId) {
+			return true
 		}
 	}
 	return false

pkg/runner/runner_test.go

@@ -111,6 +111,64 @@ func contains(strs []string, str string) bool {
 	return false
 }
 
+func TestRuleExceptions(t *testing.T) {
+	analyzer := &analysis.Analyzer{
+		Name: "testanalyzer",
+		Rules: []*analysis.Rule{
+			{Name: "rule1"},
+			{Name: "rule2"},
+		},
+		Run: func(pass *analysis.Pass) (interface{}, error) {
+			pass.Report("rule1", analysis.Diagnostic{
+				Title: "diagnostic from rule1",
+			})
+			pass.Report("rule2", analysis.Diagnostic{
+				Title: "diagnostic from rule2",
+			})
+			return nil, nil
+		},
+	}
+
+	config := Config{
+		Global: GlobalConfig{Enabled: true},
+		Analyzers: map[string]AnalyzerConfig{
+			"testanalyzer": {
+				Rules: map[string]RuleConfig{
+					"rule1": {
+						Exceptions: []string{"myorg-plugin-panel"},
+					},
+				},
+			},
+		},
+	}
+
+	pluginDir := filepath.Join("testdata", "RuleExceptions", "myorg-plugin-panel")
+
+	ds, err := Check([]*analysis.Analyzer{analyzer},
+		analysis.CheckParams{
+			ArchiveDir: pluginDir,
+		},
+		config,
+		analysis.Severity(""),
+	)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var diagnostics []string
+	for name := range ds {
+		for _, d := range ds[name] {
+			diagnostics = append(diagnostics, d.Title)
+		}
+	}
+
+	// rule1 should be skipped
+	assert.NotContains(t, diagnostics, "diagnostic from rule1")
+	// rule2 should be reported
+	assert.Contains(t, diagnostics, "diagnostic from rule2")
+}
+
 func TestLinearDependencies(t *testing.T) {
 	res := make(map[string]bool)
 	first := &analysis.Analyzer{

pkg/runner/testdata/RuleExceptions/myorg-plugin-panel/plugin.json

@@ -0,0 +1,5 @@
+{
+  "id": "myorg-plugin-panel",
+  "name": "my-plugin",
+  "type": "panel"
+}