- Notifications
You must be signed in to change notification settings - Fork928
fix(coderd): use stable sorting for insights and improve test coverage#9250
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Merged
Uh oh!
There was an error while loading.Please reload this page.
Merged
Changes fromall commits
Commits
Show all changes
33 commits Select commitHold shift + click to select a range
ffc7391 test(coderd): add extended template insights test suite
mafredri95ad848 add initial golden files
mafredrid4d58cf add some more usage and verify golden
mafredrie4f901a fixes
mafredrif333170 add an hour of app usage
mafredri6fb49a9 add 15 min of app usage -> active user on last day
mafredrie9a967e add placeholder data
mafredri9879462 add identical usage in two templates for one user
mafredrifc1a98d test app merging
mafredrif14ddf6 fix(codersdk): use url values for query params
mafredri8a60568 improve stability of template id
mafredri7d93d13 add sao paulo tz test (invalid interval times)
mafredrif9fdb47 fix start/end time tz returned in interval report
mafredri54638b0 cleanup
mafredri7b0ac4c refactor template creation with stable IDs
mafredri1e7d860 add parameter test
mafredri9d96052 refactor
mafredri930d7ef add parameter no data test
mafredri07cdc0b add 1h usage to other user
mafredri73367d1 add usage outside of utc timeframe
mafredric2a0496 add third user with ssh and web term usage
mafredrib738799 add third user app usage
mafredri79bdbf9 add webterminal app usage
mafredrice2bb78 use stable sorting in Go and minor query tweaks and fakedb fixes
mafredri0896439 use github.com/google/go-cmp/cmp for diffs
mafredri5b4733a restructure
mafredri3607a80 more fakedb fixes
mafredri688e1d0 fix fakedb logic bugs
mafredri8c294d0 fix commented code
mafredridecbdfe fix test bug
mafredri870aff1 add test for tpl1 param4 option2
mafredri75a96d7 add outside timeframe tests for apps
mafredrie72e265 remove previous insights test in favor of new one
mafredriFile filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
6 changes: 5 additions & 1 deletionMakefile
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
4 changes: 4 additions & 0 deletionscoderd/coderd_test.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
48 changes: 35 additions & 13 deletionscoderd/database/db2sdk/db2sdk.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -3,9 +3,10 @@ package db2sdk | ||
| import ( | ||
| "encoding/json" | ||
| "strings" | ||
| "github.com/google/uuid" | ||
| "golang.org/x/exp/slices" | ||
| "github.com/coder/coder/v2/coderd/database" | ||
| "github.com/coder/coder/v2/coderd/parameter" | ||
| @@ -125,9 +126,34 @@ func Role(role rbac.Role) codersdk.Role { | ||
| } | ||
| func TemplateInsightsParameters(parameterRows []database.GetTemplateParameterInsightsRow) ([]codersdk.TemplateParameterUsage, error) { | ||
| // Use a stable sort, similarly to how we would sort in the query, note that | ||
| // we don't sort in the query because order varies depending on the table | ||
| // collation. | ||
| // | ||
| // ORDER BY utp.name, utp.type, utp.display_name, utp.description, utp.options, wbp.value | ||
mafredri marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| slices.SortFunc(parameterRows, func(a, b database.GetTemplateParameterInsightsRow) int { | ||
| if a.Name != b.Name { | ||
| return strings.Compare(a.Name, b.Name) | ||
| } | ||
| if a.Type != b.Type { | ||
| return strings.Compare(a.Type, b.Type) | ||
| } | ||
| if a.DisplayName != b.DisplayName { | ||
| return strings.Compare(a.DisplayName, b.DisplayName) | ||
| } | ||
| if a.Description != b.Description { | ||
| return strings.Compare(a.Description, b.Description) | ||
| } | ||
| if string(a.Options) != string(b.Options) { | ||
| return strings.Compare(string(a.Options), string(b.Options)) | ||
| } | ||
| return strings.Compare(a.Value, b.Value) | ||
| }) | ||
| parametersUsage := []codersdk.TemplateParameterUsage{} | ||
| indexByNum := make(map[int64]int) | ||
| for _, param := range parameterRows { | ||
| if _, ok :=indexByNum[param.Num]; !ok { | ||
| var opts []codersdk.TemplateVersionParameterOption | ||
| err := json.Unmarshal(param.Options, &opts) | ||
| if err != nil { | ||
| @@ -139,28 +165,24 @@ func TemplateInsightsParameters(parameterRows []database.GetTemplateParameterIns | ||
| return nil, err | ||
| } | ||
| parametersUsage =append(parametersUsage,codersdk.TemplateParameterUsage{ | ||
| TemplateIDs: param.TemplateIDs, | ||
| Name: param.Name, | ||
| Type: param.Type, | ||
| DisplayName: param.DisplayName, | ||
| Description: plaintextDescription, | ||
| Options: opts, | ||
| }) | ||
| indexByNum[param.Num] = len(parametersUsage) - 1 | ||
| } | ||
| i := indexByNum[param.Num] | ||
| parametersUsage[i].Values = append(parametersUsage[i].Values, codersdk.TemplateParameterValue{ | ||
| Value: param.Value, | ||
| Count: param.Count, | ||
| }) | ||
| } | ||
| return parametersUsage, nil | ||
| } | ||
33 changes: 22 additions & 11 deletionscoderd/database/dbfake/dbfake.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -2018,6 +2018,10 @@ func (q *FakeQuerier) GetTemplateAppInsights(ctx context.Context, arg database.G | ||
| return nil, err | ||
| } | ||
| if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, w.TemplateID) { | ||
| continue | ||
| } | ||
| app, _ := q.getWorkspaceAppByAgentIDAndSlugNoLock(ctx, database.GetWorkspaceAppByAgentIDAndSlugParams{ | ||
| AgentID: s.AgentID, | ||
| Slug: s.SlugOrPort, | ||
| @@ -2095,6 +2099,8 @@ func (q *FakeQuerier) GetTemplateAppInsights(ctx context.Context, arg database.G | ||
| }) | ||
| } | ||
| // NOTE(mafredri): Add sorting if we decide on how to handle PostgreSQL collations. | ||
| // ORDER BY access_method, slug_or_port, display_name, icon, is_app | ||
| return rows, nil | ||
| } | ||
| @@ -2264,7 +2270,6 @@ func (q *FakeQuerier) GetTemplateDailyInsights(ctx context.Context, arg database | ||
| } | ||
| ds.userSet[s.UserID] = struct{}{} | ||
| ds.templateIDSet[s.TemplateID] = struct{}{} | ||
| } | ||
| } | ||
| @@ -2278,24 +2283,27 @@ func (q *FakeQuerier) GetTemplateDailyInsights(ctx context.Context, arg database | ||
| continue | ||
| } | ||
| w, err := q.getWorkspaceByIDNoLock(ctx, s.WorkspaceID) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, w.TemplateID) { | ||
| continue | ||
| } | ||
| for _, ds := range dailyStats { | ||
| // (was.session_started_at >= ts.from_ AND was.session_started_at < ts.to_) | ||
| // OR (was.session_ended_at > ts.from_ AND was.session_ended_at < ts.to_) | ||
| // OR (was.session_started_at < ts.from_ AND was.session_ended_at >= ts.to_) | ||
| if !(((s.SessionStartedAt.After(ds.startTime) || s.SessionStartedAt.Equal(ds.startTime)) && s.SessionStartedAt.Before(ds.endTime)) || | ||
| (s.SessionEndedAt.After(ds.startTime) && s.SessionEndedAt.Before(ds.endTime)) || | ||
| (s.SessionStartedAt.Before(ds.startTime) && (s.SessionEndedAt.After(ds.endTime) || s.SessionEndedAt.Equal(ds.endTime)))) { | ||
| continue | ||
| } | ||
| ds.userSet[s.UserID] = struct{}{} | ||
| ds.templateIDSet[w.TemplateID] = struct{}{} | ||
| } | ||
| } | ||
| @@ -2430,7 +2438,8 @@ func (q *FakeQuerier) GetTemplateParameterInsights(ctx context.Context, arg data | ||
| if tvp.TemplateVersionID != tv.ID { | ||
| continue | ||
| } | ||
| // GROUP BY tvp.name, tvp.type, tvp.display_name, tvp.description, tvp.options | ||
| key := fmt.Sprintf("%s:%s:%s:%s:%s", tvp.Name, tvp.Type, tvp.DisplayName, tvp.Description, tvp.Options) | ||
| if _, ok := uniqueTemplateParams[key]; !ok { | ||
| num++ | ||
| uniqueTemplateParams[key] = &database.GetTemplateParameterInsightsRow{ | ||
| @@ -2480,6 +2489,8 @@ func (q *FakeQuerier) GetTemplateParameterInsights(ctx context.Context, arg data | ||
| } | ||
| } | ||
| // NOTE(mafredri): Add sorting if we decide on how to handle PostgreSQL collations. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. out of curiosity: what is the default PostgreSQL behavior considering the "most default" configuration? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. It will depend on the systems locale. Essentially if e.g. if | ||
| // ORDER BY utp.name, utp.type, utp.display_name, utp.description, utp.options, wbp.value | ||
| return rows, nil | ||
| } | ||
8 changes: 4 additions & 4 deletionscoderd/database/queries.sql.go
Some generated files are not rendered by default. Learn more abouthow customized files appear on GitHub.
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
8 changes: 4 additions & 4 deletionscoderd/database/queries/insights.sql
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
33 changes: 31 additions & 2 deletionscoderd/insights.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -4,6 +4,7 @@ import ( | ||
| "context" | ||
| "fmt" | ||
| "net/http" | ||
| "strings" | ||
| "time" | ||
| "github.com/google/uuid" | ||
| @@ -288,8 +289,10 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) { | ||
| } | ||
| for _, row := range dailyUsage { | ||
| resp.IntervalReports = append(resp.IntervalReports, codersdk.TemplateInsightsIntervalReport{ | ||
| // NOTE(mafredri): This might not be accurate over DST since the | ||
| // parsed location only contains the offset. | ||
| StartTime: row.StartTime.In(startTime.Location()), | ||
| EndTime: row.EndTime.In(startTime.Location()), | ||
| Interval: interval, | ||
| TemplateIDs: row.TemplateIDs, | ||
| ActiveUsers: row.ActiveUsers, | ||
| @@ -377,6 +380,32 @@ func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage | ||
| }, | ||
| } | ||
| // Use a stable sort, similarly to how we would sort in the query, note that | ||
| // we don't sort in the query because order varies depending on the table | ||
| // collation. | ||
| // | ||
| // ORDER BY access_method, slug_or_port, display_name, icon, is_app | ||
mafredri marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| slices.SortFunc(appUsage, func(a, b database.GetTemplateAppInsightsRow) int { | ||
| if a.AccessMethod != b.AccessMethod { | ||
| return strings.Compare(a.AccessMethod, b.AccessMethod) | ||
| } | ||
| if a.SlugOrPort != b.SlugOrPort { | ||
| return strings.Compare(a.SlugOrPort, b.SlugOrPort) | ||
| } | ||
| if a.DisplayName.String != b.DisplayName.String { | ||
| return strings.Compare(a.DisplayName.String, b.DisplayName.String) | ||
| } | ||
| if a.Icon.String != b.Icon.String { | ||
| return strings.Compare(a.Icon.String, b.Icon.String) | ||
| } | ||
| if !a.IsApp && b.IsApp { | ||
| return -1 | ||
| } else if a.IsApp && !b.IsApp { | ||
| return 1 | ||
| } | ||
| return 0 | ||
| }) | ||
| // Template apps. | ||
| for _, app := range appUsage { | ||
| if !app.IsApp { | ||
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.