Merge pull request #1156 from wakatime/feature/hide-imports-config

New config hide_dependencies

Author: alanhamlett

USAGE.md

@@ -41,7 +41,8 @@ api_key_vault_cmd = command arg arg ... (space-separated, no shell syntax)
 api_url = https://api.wakatime.com/api/v1
 hide_file_names = false
 hide_project_names = false
-hide_branch_names =
+hide_branch_names = false
+hide_dependencies = false
 hide_project_folder = false
 exclude =
     ^COMMIT_EDITMSG$
@@ -95,6 +96,7 @@ some/submodule/name = new project name
 | hide_file_names                | Obfuscate filenames. Will not send file names to api. | _bool_;_list_ | `false` |
 | hide_project_names             | Obfuscate project names. When a project folder is detected instead of using the folder name as the project, a `.wakatime-project file` is created with a random project name. | _bool_;_list_ | `false` |
 | hide_branch_names              | Obfuscate branch names. Will not send revision control branch names to api. | _bool_;_list_ | `false` |
+| hide_dependencies              | Prevent sending imports/libraries/dependencies used in currently focused file to the api.  | _bool_;_list_ | `false` |
 | hide_project_folder            | When set, send the file's path relative to the project folder. For ex: `/User/me/projects/bar/src/file.ts` is sent as `src/file.ts` so the server never sees the full path. When the project folder cannot be detected, only the file name is sent. For ex: `file.ts`. | _bool_ | `false` |
 | exclude                        | Filename patterns to exclude from logging. POSIX regex syntax. | _bool_;_list_ | |
 | include                        | Filename patterns to log. When used in combination with `exclude`, files matching `include` will still be logged. POSIX regex syntax | _bool_;_list_ | |
@@ -111,7 +113,7 @@ some/submodule/name = new project name
 | hostname                       | Optional name of local machine. By default, auto-detects the local machine’s hostname. | _string_ | |
 | log_file                       | Optional log file path. | _filepath_ | `~/.wakatime/wakatime.log` |
 | import_cfg                     | Optional path to another wakatime.cfg file to import. If set it will overwrite values loaded from $WAKATIME_HOME/.wakatime.cfg file. | _filepath_ | |
-| metrics                        | When set, collects metrics usage in `~/.wakatime/metrics` folder. For further reference visit <https://go.dev/blog/pprof>. | _bool_ | `false` |
+| metrics                        | When set, collects metrics usage in '~/.wakatime/metrics' folder. For further reference visit <https://go.dev/blog/pprof>. | _bool_ | `false` |
 | guess_language                 | When `true`, enables detecting programming language from file contents. | _bool_ | `false` |
 
 ### Project Map Section
@@ -142,9 +144,9 @@ However, if an api key exists in your `~/.wakatime.cfg` file then it takes prece
 
 ### Git Section
 
-| option                         | description | type | default value |
-| ---                            | ---         | ---  | ---           |
-| submodules_disabled            | It will be matched against the submodule path and if matching, will skip it. | _bool_;_list_ | false |
+| option              | description | type | default value |
+| ---                 | ---         | ---  | ---           |
+| submodules_disabled | It will be matched against the submodule path and if matching, will skip it. | _bool_;_list_ | false |
 
 ### Git Submodule Project Map Section
 

cmd/fileexperts/fileexperts.go

@@ -131,10 +131,11 @@ func initHandleOptions(params paramscmd.Params) []heartbeat.HandleOption {
 			ExcludeUnknownProject: params.Heartbeat.Filter.ExcludeUnknownProject,
 		}),
 		heartbeat.WithSanitization(heartbeat.SanitizeConfig{
-			BranchPatterns:    params.Heartbeat.Sanitize.HideBranchNames,
-			FilePatterns:      params.Heartbeat.Sanitize.HideFileNames,
-			HideProjectFolder: params.Heartbeat.Sanitize.HideProjectFolder,
-			ProjectPatterns:   params.Heartbeat.Sanitize.HideProjectNames,
+			BranchPatterns:     params.Heartbeat.Sanitize.HideBranchNames,
+			DependencyPatterns: params.Heartbeat.Sanitize.HideDependencies,
+			FilePatterns:       params.Heartbeat.Sanitize.HideFileNames,
+			HideProjectFolder:  params.Heartbeat.Sanitize.HideProjectFolder,
+			ProjectPatterns:    params.Heartbeat.Sanitize.HideProjectNames,
 		}),
 		fileexperts.WithValidation(),
 		filter.WithLengthValidator(),

cmd/heartbeat/heartbeat.go

@@ -326,10 +326,11 @@ func initHandleOptions(params paramscmd.Params) []heartbeat.HandleOption {
 			ExcludeUnknownProject: params.Heartbeat.Filter.ExcludeUnknownProject,
 		}),
 		heartbeat.WithSanitization(heartbeat.SanitizeConfig{
-			BranchPatterns:    params.Heartbeat.Sanitize.HideBranchNames,
-			FilePatterns:      params.Heartbeat.Sanitize.HideFileNames,
-			HideProjectFolder: params.Heartbeat.Sanitize.HideProjectFolder,
-			ProjectPatterns:   params.Heartbeat.Sanitize.HideProjectNames,
+			BranchPatterns:     params.Heartbeat.Sanitize.HideBranchNames,
+			DependencyPatterns: params.Heartbeat.Sanitize.HideDependencies,
+			FilePatterns:       params.Heartbeat.Sanitize.HideFileNames,
+			HideProjectFolder:  params.Heartbeat.Sanitize.HideProjectFolder,
+			ProjectPatterns:    params.Heartbeat.Sanitize.HideProjectNames,
 		}),
 		remote.WithCleanup(),
 		filter.WithLengthValidator(),

cmd/heartbeat/heartbeat_test.go

@@ -1061,7 +1061,12 @@ func TestSendHeartbeats_ObfuscateProject(t *testing.T) {
 		lines, err := project.ReadFile(ctx, filepath.Join(fp, "wakatime-cli", ".wakatime-project"), 1)
 		require.NoError(t, err)
 
-		expectedBodyStr := fmt.Sprintf(string(expectedBody), entity.Entity, lines[0], heartbeat.UserAgent(ctx, plugin))
+		expectedBodyStr := fmt.Sprintf(
+			string(expectedBody),
+			entity.Entity,
+			lines[0],
+			heartbeat.UserAgent(ctx, plugin),
+		)
 
 		assert.True(t, strings.HasSuffix(entity.Entity, "src/pkg/file.go"))
 		assert.JSONEq(t, expectedBodyStr, string(body))

cmd/heartbeat/testdata/api_heartbeats_request_template_obfuscated_project.json

@@ -1,5 +1,6 @@
 [
     {
+        "branch": "master",
         "category": "debugging",
         "entity": "%s",
         "is_write": true,

cmd/offline/offline.go

@@ -166,10 +166,11 @@ func initHandleOptions(params paramscmd.Params) []heartbeat.HandleOption {
 			ExcludeUnknownProject: params.Heartbeat.Filter.ExcludeUnknownProject,
 		}),
 		heartbeat.WithSanitization(heartbeat.SanitizeConfig{
-			BranchPatterns:    params.Heartbeat.Sanitize.HideBranchNames,
-			FilePatterns:      params.Heartbeat.Sanitize.HideFileNames,
-			HideProjectFolder: params.Heartbeat.Sanitize.HideProjectFolder,
-			ProjectPatterns:   params.Heartbeat.Sanitize.HideProjectNames,
+			BranchPatterns:     params.Heartbeat.Sanitize.HideBranchNames,
+			DependencyPatterns: params.Heartbeat.Sanitize.HideDependencies,
+			FilePatterns:       params.Heartbeat.Sanitize.HideFileNames,
+			HideProjectFolder:  params.Heartbeat.Sanitize.HideProjectFolder,
+			ProjectPatterns:    params.Heartbeat.Sanitize.HideProjectNames,
 		}),
 		remote.WithCleanup(),
 		filter.WithLengthValidator(),

cmd/params/params.go

@@ -154,6 +154,7 @@ type (
 	// SanitizeParams params for heartbeat sanitization.
 	SanitizeParams struct {
 		HideBranchNames     []regex.Regex
+		HideDependencies    []regex.Regex
 		HideFileNames       []regex.Regex
 		HideProjectFolder   bool
 		HideProjectNames    []regex.Regex
@@ -555,6 +556,22 @@ func loadSanitizeParams(ctx context.Context, v *viper.Viper) (SanitizeParams, er
 		)
 	}
 
+	// hide dependencies
+	hideDependenciesStr := vipertools.FirstNonEmptyString(
+		v,
+		"hide-dependencies",
+		"settings.hide_dependencies",
+	)
+
+	hideDependenciesPatterns, err := parseBoolOrRegexList(ctx, hideDependenciesStr)
+	if err != nil {
+		return SanitizeParams{}, fmt.Errorf(
+			"failed to parse regex hide dependencies param %q: %s",
+			hideDependenciesStr,
+			err,
+		)
+	}
+
 	// hide project names
 	hideProjectNamesStr := vipertools.FirstNonEmptyString(
 		v,
@@ -595,6 +612,7 @@ func loadSanitizeParams(ctx context.Context, v *viper.Viper) (SanitizeParams, er
 
 	return SanitizeParams{
 		HideBranchNames:     hideBranchNamesPatterns,
+		HideDependencies:    hideDependenciesPatterns,
 		HideFileNames:       hideFileNamesPatterns,
 		HideProjectFolder:   vipertools.FirstNonEmptyBool(v, "hide-project-folder", "settings.hide_project_folder"),
 		HideProjectNames:    hideProjectNamesPatterns,
@@ -1146,11 +1164,12 @@ func (p ProjectParams) String() string {
 func (p SanitizeParams) String() string {
 	return fmt.Sprintf(
 		"hide branch names: '%s', hide project folder: %t, hide file names: '%s',"+
-			" hide project names: '%s', project path override: '%s'",
+			" hide project names: '%s', hide dependencies: '%s', project path override: '%s'",
 		p.HideBranchNames,
 		p.HideProjectFolder,
 		p.HideFileNames,
 		p.HideProjectNames,
+		p.HideDependencies,
 		p.ProjectPathOverride,
 	)
 }

cmd/params/params_test.go

@@ -1118,7 +1118,7 @@ func TestLoadHeartbeatParams_SanitizeParams_HideBranchNames_FlagTakesPrecedence(
 func TestLoadHeartbeatParams_SanitizeParams_HideBranchNames_ConfigTakesPrecedence(t *testing.T) {
 	v := viper.New()
 	v.Set("entity", "/path/to/file")
-	v.Set("settings.hide_branch_names", "true")
+	v.Set("settings.hide_branch_names", true)
 	v.Set("settings.hide_branchnames", "ignored")
 	v.Set("settings.hidebranchnames", "ignored")
 
@@ -1133,7 +1133,7 @@ func TestLoadHeartbeatParams_SanitizeParams_HideBranchNames_ConfigTakesPrecedenc
 func TestLoadHeartbeatParams_SanitizeParams_HideBranchNames_ConfigDeprecatedOneTakesPrecedence(t *testing.T) {
 	v := viper.New()
 	v.Set("entity", "/path/to/file")
-	v.Set("settings.hide_branchnames", "true")
+	v.Set("settings.hide_branchnames", true)
 	v.Set("settings.hidebranchnames", "ignored")
 
 	params, err := cmdparams.LoadHeartbeatParams(context.Background(), v)
@@ -1147,7 +1147,7 @@ func TestLoadHeartbeatParams_SanitizeParams_HideBranchNames_ConfigDeprecatedOneT
 func TestLoadHeartbeatParams_SanitizeParams_HideBranchNames_ConfigDeprecatedTwo(t *testing.T) {
 	v := viper.New()
 	v.Set("entity", "/path/to/file")
-	v.Set("settings.hidebranchnames", "true")
+	v.Set("settings.hidebranchnames", true)
 
 	params, err := cmdparams.LoadHeartbeatParams(context.Background(), v)
 	require.NoError(t, err)
@@ -1186,6 +1186,163 @@ func TestLoadHeartbeatParams_SanitizeParams_HideBranchNames_InvalidRegex(t *test
 	assert.Contains(t, string(output), "failed to compile regex pattern \\\"(?i)[0-9+\\\", it will be ignored")
 }
 
+func TestLoadHeartbeatParams_SanitizeParams_HideDependencies_Flag(t *testing.T) {
+	v := viper.New()
+	v.Set("entity", "/path/to/file")
+	v.Set("settings.hide_dependencies", true)
+
+	params, err := cmdparams.LoadHeartbeatParams(context.Background(), v)
+	require.NoError(t, err)
+
+	assert.Equal(t, cmdparams.SanitizeParams{
+		HideDependencies: []regex.Regex{regex.NewRegexpWrap(regexp.MustCompile(".*"))},
+	}, params.Sanitize)
+}
+
+func TestLoadHeartbeatParams_SanitizeParams_HideDependencies_True(t *testing.T) {
+	ctx := context.Background()
+
+	tests := map[string]string{
+		"lowercase":       "true",
+		"uppercase":       "TRUE",
+		"first uppercase": "True",
+	}
+
+	for name, viperValue := range tests {
+		t.Run(name, func(t *testing.T) {
+			v := viper.New()
+			v.Set("entity", "/path/to/file")
+			v.Set("hide-dependencies", viperValue)
+
+			params, err := cmdparams.LoadHeartbeatParams(ctx, v)
+			require.NoError(t, err)
+
+			assert.Equal(t, cmdparams.SanitizeParams{
+				HideDependencies: []regex.Regex{regex.NewRegexpWrap(regexp.MustCompile(".*"))},
+			}, params.Sanitize)
+		})
+	}
+}
+
+func TestLoadHeartbeatParams_SanitizeParams_HideDependencies_False(t *testing.T) {
+	ctx := context.Background()
+
+	tests := map[string]string{
+		"lowercase":       "false",
+		"uppercase":       "FALSE",
+		"first uppercase": "False",
+	}
+
+	for name, viperValue := range tests {
+		t.Run(name, func(t *testing.T) {
+			v := viper.New()
+			v.Set("entity", "/path/to/file")
+			v.Set("hide-dependencies", viperValue)
+
+			params, err := cmdparams.LoadHeartbeatParams(ctx, v)
+			require.NoError(t, err)
+
+			assert.Equal(t, cmdparams.SanitizeParams{
+				HideDependencies: []regex.Regex{regex.NewRegexpWrap(regexp.MustCompile("a^"))},
+			}, params.Sanitize)
+		})
+	}
+}
+
+func TestLoadHeartbeatParams_SanitizeParams_HideDependencies_List(t *testing.T) {
+	ctx := context.Background()
+
+	tests := map[string]struct {
+		ViperValue string
+		Expected   []regex.Regex
+	}{
+		"regex": {
+			ViperValue: "fix.*",
+			Expected: []regex.Regex{
+				regex.NewRegexpWrap(regexp.MustCompile("(?i)fix.*")),
+			},
+		},
+		"regex list": {
+			ViperValue: ".*secret.*\nfix.*",
+			Expected: []regex.Regex{
+				regex.NewRegexpWrap(regexp.MustCompile("(?i).*secret.*")),
+				regex.NewRegexpWrap(regexp.MustCompile("(?i)fix.*")),
+			},
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			v := viper.New()
+			v.Set("entity", "/path/to/file")
+			v.Set("hide-dependencies", test.ViperValue)
+
+			params, err := cmdparams.LoadHeartbeatParams(ctx, v)
+			require.NoError(t, err)
+
+			assert.Equal(t, cmdparams.SanitizeParams{
+				HideDependencies: test.Expected,
+			}, params.Sanitize)
+		})
+	}
+}
+
+func TestLoadHeartbeatParams_SanitizeParams_HideDependencies_FlagTakesPrecedence(t *testing.T) {
+	v := viper.New()
+	v.Set("entity", "/path/to/file")
+	v.Set("hide-dependencies", true)
+	v.Set("settings.hide_dependencies", "ignored")
+
+	params, err := cmdparams.LoadHeartbeatParams(context.Background(), v)
+	require.NoError(t, err)
+
+	assert.Equal(t, cmdparams.SanitizeParams{
+		HideDependencies: []regex.Regex{regex.NewRegexpWrap(regexp.MustCompile(".*"))},
+	}, params.Sanitize)
+}
+
+func TestLoadHeartbeatParams_SanitizeParams_HideDependencies_FromConfig(t *testing.T) {
+	v := viper.New()
+	v.Set("entity", "/path/to/file")
+	v.Set("settings.hide_dependencies", true)
+
+	params, err := cmdparams.LoadHeartbeatParams(context.Background(), v)
+	require.NoError(t, err)
+
+	assert.Equal(t, cmdparams.SanitizeParams{
+		HideDependencies: []regex.Regex{regex.NewRegexpWrap(regexp.MustCompile(".*"))},
+	}, params.Sanitize)
+}
+
+func TestLoadHeartbeatParams_SanitizeParams_HideDependencies_InvalidRegex(t *testing.T) {
+	logFile, err := os.CreateTemp(t.TempDir(), "")
+	require.NoError(t, err)
+
+	defer logFile.Close()
+
+	ctx := context.Background()
+
+	v := viper.New()
+	v.Set("entity", "/path/to/file")
+	v.Set("hide-dependencies", ".*secret.*\n[0-9+")
+	v.Set("log-file", logFile.Name())
+
+	logger, err := cmd.SetupLogging(ctx, v)
+	require.NoError(t, err)
+
+	defer logger.Flush()
+
+	ctx = log.ToContext(ctx, logger)
+
+	_, err = cmdparams.LoadHeartbeatParams(ctx, v)
+	require.NoError(t, err)
+
+	output, err := io.ReadAll(logFile)
+	require.NoError(t, err)
+
+	assert.Contains(t, string(output), "failed to compile regex pattern \\\"(?i)[0-9+\\\", it will be ignored")
+}
+
 func TestLoadHeartbeatParams_SanitizeParams_HideProjectNames_True(t *testing.T) {
 	ctx := context.Background()
 
@@ -2689,7 +2846,7 @@ func TestHeartbeat_String(t *testing.T) {
 			" project file: false), project params: (alternate: '', branch alternate: '', map patterns:"+
 			" '[]', override: '', git submodules disabled: '[]', git submodule project map: '[]'), sanitize"+
 			" params: (hide branch names: '[]', hide project folder: false, hide file names: '[]',"+
-			" hide project names: '[]', project path override: '')",
+			" hide project names: '[]', hide dependencies: '[]', project path override: '')",
 		heartbeat.String(),
 	)
 }
@@ -2753,6 +2910,7 @@ func TestLoadHeartbeatParams_ProjectFromGitRemote(t *testing.T) {
 func TestSanitizeParams_String(t *testing.T) {
 	sanitizeparams := cmdparams.SanitizeParams{
 		HideBranchNames:     []regex.Regex{regex.NewRegexpWrap(regexp.MustCompile("^/hide"))},
+		HideDependencies:    []regex.Regex{regex.NewRegexpWrap(regexp.MustCompile("^/hide"))},
 		HideProjectFolder:   true,
 		HideFileNames:       []regex.Regex{regex.NewRegexpWrap(regexp.MustCompile("^/hide"))},
 		HideProjectNames:    []regex.Regex{regex.NewRegexpWrap(regexp.MustCompile("^/hide"))},
@@ -2762,7 +2920,7 @@ func TestSanitizeParams_String(t *testing.T) {
 	assert.Equal(
 		t,
 		"hide branch names: '[^/hide]', hide project folder: true, hide file names: '[^/hide]',"+
-			" hide project names: '[^/hide]', project path override: 'path/to/project'",
+			" hide project names: '[^/hide]', hide dependencies: '[^/hide]', project path override: 'path/to/project'",
 		sanitizeparams.String(),
 	)
 }

pkg/deps/deps.go

@@ -52,7 +52,12 @@ func WithDetection(c Config) heartbeat.HandleOption {
 					continue
 				}
 
-				if heartbeat.ShouldSanitize(ctx, h.Entity, c.FilePatterns) {
+				if heartbeat.ShouldSanitize(ctx, heartbeat.SanitizeCheck{
+					Entity:              h.Entity,
+					ProjectPath:         h.ProjectPath,
+					ProjectPathOverride: h.ProjectPathOverride,
+					Patterns:            c.FilePatterns,
+				}) {
 					continue
 				}