- Notifications
You must be signed in to change notification settings - Fork1k
feat(cli): addcoder stat command#8005
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Merged
Uh oh!
There was an error while loading.Please reload this page.
Merged
Changes fromall commits
Commits
Show all changes
48 commits Select commitHold shift + click to select a range
5db9006 add stat command
johnstcnd6029b4 cpu working on mac
johnstcn18f4942 add stat memory
johnstcn251fdda support values with no total
johnstcn4c081dc move clistats to its own package
johnstcn2ba7392 fix container detection to work with sysbox containers
johnstcn0e1c96a add cross-platform declaration for IsContainerized()
johnstcn0f9859e add a sync.Once to IsContainerized()
johnstcna220c7f make uptime minutes
johnstcn89f7e8d lint
johnstcnc51e245 extract nproc to variable
johnstcn3528c00 add skeleton of cgroup stuff
johnstcn7108c6e initial cgroupv2 cpu implementation
johnstcn4ef5f24 fix disk_windows
johnstcnf0f7b6a add tests for clistats
johnstcn6a878b9 improve testing
johnstcnbe7ba72 remove unnecessary os-specific implementations now that we have abstr…
johnstcn3643407 remove uptime stat as it is trivial to implement in bash
johnstcn1c8943e implement cgroupv1 cpu
johnstcn95b8d1f unskip container memory tests
johnstcn495b5b0 flesh out tests
johnstcnfa0c4c6 cgroupv1 memory
johnstcn70ef79b improve tests to allow testing cpu used
johnstcn7eeefc1 refactor cpu usage calc
johnstcn305675f fix tests
johnstcnd1bb322 fix off-by-10 error
johnstcneb2bcf6 remove --sample-interval and collect CPU stats in parallel
johnstcn44edcf3 fmt; gen
johnstcn0f3254a make default_cols consistent to avoid ci surprises
johnstcnedd99f4 fix race condition
johnstcn49b6861 remove UPTIME from test
johnstcn69b1904 update golden files
johnstcn7eb526d add stat subcommands
johnstcn665bf7f allow modifying unit prefixes
johnstcn6b11a5c update docs and examples
johnstcnc1467f0 fix NaN issue for HostCPU
johnstcn789c6de avoid blocking on err chan
johnstcn482db10 add percentages
johnstcn0775082 remove outdated comments
johnstcn73debf8 handle counter reset
johnstcnd0c992a add test for large difference between used and total
johnstcnef7460a auto-scale precision, limiting to 3 digits
johnstcnbec527f automatically scale precision, remove --prefix arg
johnstcn08adba7 make gen
johnstcn78f76e7 improve cli tests
johnstcn9a82882 update go.mod
johnstcn19c8a80 Merge remote-tracking branch 'origin/main' into cj/coder-stat
johnstcneab2530 update go.sum
johnstcnFile filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
311 changes: 311 additions & 0 deletionscli/clistat/cgroup.go
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. Happy that you split a lot of this logic into its own package. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,311 @@ | ||
| package clistat | ||
| import ( | ||
| "bufio" | ||
| "bytes" | ||
| "strconv" | ||
| "strings" | ||
| "github.com/spf13/afero" | ||
| "golang.org/x/xerrors" | ||
| "tailscale.com/types/ptr" | ||
| ) | ||
| // Paths for CGroupV1. | ||
| // Ref: https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt | ||
| const ( | ||
| // CPU usage of all tasks in cgroup in nanoseconds. | ||
| cgroupV1CPUAcctUsage = "/sys/fs/cgroup/cpu/cpuacct.usage" | ||
| // Alternate path | ||
| cgroupV1CPUAcctUsageAlt = "/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage" | ||
| // CFS quota and period for cgroup in MICROseconds | ||
| cgroupV1CFSQuotaUs = "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us" | ||
| cgroupV1CFSPeriodUs = "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us" | ||
| // Maximum memory usable by cgroup in bytes | ||
| cgroupV1MemoryMaxUsageBytes = "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" | ||
| // Current memory usage of cgroup in bytes | ||
| cgroupV1MemoryUsageBytes = "/sys/fs/cgroup/memory/memory.usage_in_bytes" | ||
| // Other memory stats - we are interested in total_inactive_file | ||
| cgroupV1MemoryStat = "/sys/fs/cgroup/memory/memory.stat" | ||
| ) | ||
| // Paths for CGroupV2. | ||
| // Ref: https://docs.kernel.org/admin-guide/cgroup-v2.html | ||
| const ( | ||
| // Contains quota and period in microseconds separated by a space. | ||
| cgroupV2CPUMax = "/sys/fs/cgroup/cpu.max" | ||
| // Contains current CPU usage under usage_usec | ||
| cgroupV2CPUStat = "/sys/fs/cgroup/cpu.stat" | ||
| // Contains current cgroup memory usage in bytes. | ||
| cgroupV2MemoryUsageBytes = "/sys/fs/cgroup/memory.current" | ||
| // Contains max cgroup memory usage in bytes. | ||
| cgroupV2MemoryMaxBytes = "/sys/fs/cgroup/memory.max" | ||
| // Other memory stats - we are interested in total_inactive_file | ||
| cgroupV2MemoryStat = "/sys/fs/cgroup/memory.stat" | ||
| ) | ||
| // ContainerCPU returns the CPU usage of the container cgroup. | ||
| // This is calculated as difference of two samples of the | ||
| // CPU usage of the container cgroup. | ||
| // The total is read from the relevant path in /sys/fs/cgroup. | ||
| // If there is no limit set, the total is assumed to be the | ||
| // number of host cores multiplied by the CFS period. | ||
| // If the system is not containerized, this always returns nil. | ||
| func (s *Statter) ContainerCPU() (*Result, error) { | ||
| // Firstly, check if we are containerized. | ||
| if ok, err := IsContainerized(s.fs); err != nil || !ok { | ||
| return nil, nil //nolint: nilnil | ||
| } | ||
| total, err := s.cGroupCPUTotal() | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("get total cpu: %w", err) | ||
| } | ||
| used1, err := s.cGroupCPUUsed() | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("get cgroup CPU usage: %w", err) | ||
| } | ||
| // The measurements in /sys/fs/cgroup are counters. | ||
| // We need to wait for a bit to get a difference. | ||
| // Note that someone could reset the counter in the meantime. | ||
| // We can't do anything about that. | ||
johnstcn marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| s.wait(s.sampleInterval) | ||
johnstcn marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| used2, err := s.cGroupCPUUsed() | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("get cgroup CPU usage: %w", err) | ||
| } | ||
| if used2 < used1 { | ||
| // Someone reset the counter. Best we can do is count from zero. | ||
| used1 = 0 | ||
| } | ||
| r := &Result{ | ||
| Unit: "cores", | ||
| Used: used2 - used1, | ||
| Total: ptr.To(total), | ||
| } | ||
| return r, nil | ||
| } | ||
| func (s *Statter) cGroupCPUTotal() (used float64, err error) { | ||
| if s.isCGroupV2() { | ||
| return s.cGroupV2CPUTotal() | ||
| } | ||
| // Fall back to CGroupv1 | ||
| return s.cGroupV1CPUTotal() | ||
| } | ||
| func (s *Statter) cGroupCPUUsed() (used float64, err error) { | ||
| if s.isCGroupV2() { | ||
| return s.cGroupV2CPUUsed() | ||
| } | ||
| return s.cGroupV1CPUUsed() | ||
| } | ||
| func (s *Statter) isCGroupV2() bool { | ||
| // Check for the presence of /sys/fs/cgroup/cpu.max | ||
| _, err := s.fs.Stat(cgroupV2CPUMax) | ||
| return err == nil | ||
| } | ||
| func (s *Statter) cGroupV2CPUUsed() (used float64, err error) { | ||
| usageUs, err := readInt64Prefix(s.fs, cgroupV2CPUStat, "usage_usec") | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("get cgroupv2 cpu used: %w", err) | ||
| } | ||
| periodUs, err := readInt64SepIdx(s.fs, cgroupV2CPUMax, " ", 1) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("get cpu period: %w", err) | ||
| } | ||
| return float64(usageUs) / float64(periodUs), nil | ||
| } | ||
| func (s *Statter) cGroupV2CPUTotal() (total float64, err error) { | ||
| var quotaUs, periodUs int64 | ||
| periodUs, err = readInt64SepIdx(s.fs, cgroupV2CPUMax, " ", 1) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("get cpu period: %w", err) | ||
| } | ||
| quotaUs, err = readInt64SepIdx(s.fs, cgroupV2CPUMax, " ", 0) | ||
| if err != nil { | ||
| // Fall back to number of cores | ||
| quotaUs = int64(s.nproc) * periodUs | ||
| } | ||
| return float64(quotaUs) / float64(periodUs), nil | ||
| } | ||
| func (s *Statter) cGroupV1CPUTotal() (float64, error) { | ||
| periodUs, err := readInt64(s.fs, cgroupV1CFSPeriodUs) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("read cpu period: %w", err) | ||
| } | ||
| quotaUs, err := readInt64(s.fs, cgroupV1CFSQuotaUs) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("read cpu quota: %w", err) | ||
| } | ||
| if quotaUs < 0 { | ||
| // Fall back to the number of cores | ||
| quotaUs = int64(s.nproc) * periodUs | ||
| } | ||
| return float64(quotaUs) / float64(periodUs), nil | ||
| } | ||
| func (s *Statter) cGroupV1CPUUsed() (float64, error) { | ||
| usageNs, err := readInt64(s.fs, cgroupV1CPUAcctUsage) | ||
| if err != nil { | ||
| // try alternate path | ||
| usageNs, err = readInt64(s.fs, cgroupV1CPUAcctUsageAlt) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("read cpu used: %w", err) | ||
| } | ||
| } | ||
| // usage is in ns, convert to us | ||
| usageNs /= 1000 | ||
| periodUs, err := readInt64(s.fs, cgroupV1CFSPeriodUs) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("get cpu period: %w", err) | ||
| } | ||
| return float64(usageNs) / float64(periodUs), nil | ||
| } | ||
| // ContainerMemory returns the memory usage of the container cgroup. | ||
| // If the system is not containerized, this always returns nil. | ||
| func (s *Statter) ContainerMemory() (*Result, error) { | ||
| if ok, err := IsContainerized(s.fs); err != nil || !ok { | ||
| return nil, nil //nolint:nilnil | ||
| } | ||
| if s.isCGroupV2() { | ||
| return s.cGroupV2Memory() | ||
| } | ||
| // Fall back to CGroupv1 | ||
| return s.cGroupV1Memory() | ||
| } | ||
| func (s *Statter) cGroupV2Memory() (*Result, error) { | ||
| maxUsageBytes, err := readInt64(s.fs, cgroupV2MemoryMaxBytes) | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("read memory total: %w", err) | ||
| } | ||
| currUsageBytes, err := readInt64(s.fs, cgroupV2MemoryUsageBytes) | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("read memory usage: %w", err) | ||
| } | ||
| inactiveFileBytes, err := readInt64Prefix(s.fs, cgroupV2MemoryStat, "inactive_file") | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("read memory stats: %w", err) | ||
| } | ||
| return &Result{ | ||
| Total: ptr.To(float64(maxUsageBytes)), | ||
| Used: float64(currUsageBytes - inactiveFileBytes), | ||
| Unit: "B", | ||
| }, nil | ||
| } | ||
| func (s *Statter) cGroupV1Memory() (*Result, error) { | ||
| maxUsageBytes, err := readInt64(s.fs, cgroupV1MemoryMaxUsageBytes) | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("read memory total: %w", err) | ||
| } | ||
| // need a space after total_rss so we don't hit something else | ||
| usageBytes, err := readInt64(s.fs, cgroupV1MemoryUsageBytes) | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("read memory usage: %w", err) | ||
| } | ||
| totalInactiveFileBytes, err := readInt64Prefix(s.fs, cgroupV1MemoryStat, "total_inactive_file") | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("read memory stats: %w", err) | ||
| } | ||
| // Total memory used is usage - total_inactive_file | ||
| return &Result{ | ||
| Total: ptr.To(float64(maxUsageBytes)), | ||
| Used: float64(usageBytes - totalInactiveFileBytes), | ||
| Unit: "B", | ||
| }, nil | ||
| } | ||
| // read an int64 value from path | ||
| func readInt64(fs afero.Fs, path string) (int64, error) { | ||
| data, err := afero.ReadFile(fs, path) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("read %s: %w", path, err) | ||
| } | ||
| val, err := strconv.ParseInt(string(bytes.TrimSpace(data)), 10, 64) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("parse %s: %w", path, err) | ||
| } | ||
| return val, nil | ||
| } | ||
| // read an int64 value from path at field idx separated by sep | ||
| func readInt64SepIdx(fs afero.Fs, path, sep string, idx int) (int64, error) { | ||
| data, err := afero.ReadFile(fs, path) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("read %s: %w", path, err) | ||
| } | ||
| parts := strings.Split(string(data), sep) | ||
| if len(parts) < idx { | ||
| return 0, xerrors.Errorf("expected line %q to have at least %d parts", string(data), idx+1) | ||
| } | ||
| val, err := strconv.ParseInt(strings.TrimSpace(parts[idx]), 10, 64) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("parse %s: %w", path, err) | ||
| } | ||
| return val, nil | ||
| } | ||
| // read the first int64 value from path prefixed with prefix | ||
| func readInt64Prefix(fs afero.Fs, path, prefix string) (int64, error) { | ||
| data, err := afero.ReadFile(fs, path) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("read %s: %w", path, err) | ||
| } | ||
| scn := bufio.NewScanner(bytes.NewReader(data)) | ||
| for scn.Scan() { | ||
| line := scn.Text() | ||
| if !strings.HasPrefix(line, prefix) { | ||
| continue | ||
| } | ||
| parts := strings.Fields(line) | ||
| if len(parts) != 2 { | ||
| return 0, xerrors.Errorf("parse %s: expected two fields but got %s", path, line) | ||
| } | ||
| val, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64) | ||
| if err != nil { | ||
| return 0, xerrors.Errorf("parse %s: %w", path, err) | ||
| } | ||
| return val, nil | ||
| } | ||
| return 0, xerrors.Errorf("parse %s: did not find line with prefix %s", path, prefix) | ||
| } | ||
61 changes: 61 additions & 0 deletionscli/clistat/container.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| package clistat | ||
| import ( | ||
| "bufio" | ||
| "bytes" | ||
| "os" | ||
| "github.com/spf13/afero" | ||
ammario marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| "golang.org/x/xerrors" | ||
| ) | ||
| const ( | ||
| procMounts = "/proc/mounts" | ||
| procOneCgroup = "/proc/1/cgroup" | ||
| ) | ||
| // IsContainerized returns whether the host is containerized. | ||
| // This is adapted from https://github.com/elastic/go-sysinfo/tree/main/providers/linux/container.go#L31 | ||
| // with modifications to support Sysbox containers. | ||
| // On non-Linux platforms, it always returns false. | ||
| func IsContainerized(fs afero.Fs) (ok bool, err error) { | ||
| cgData, err := afero.ReadFile(fs, procOneCgroup) | ||
| if err != nil { | ||
| if os.IsNotExist(err) { | ||
| return false, nil | ||
| } | ||
| return false, xerrors.Errorf("read file %s: %w", procOneCgroup, err) | ||
| } | ||
| scn := bufio.NewScanner(bytes.NewReader(cgData)) | ||
| for scn.Scan() { | ||
| line := scn.Bytes() | ||
| if bytes.Contains(line, []byte("docker")) || | ||
| bytes.Contains(line, []byte(".slice")) || | ||
| bytes.Contains(line, []byte("lxc")) || | ||
| bytes.Contains(line, []byte("kubepods")) { | ||
| return true, nil | ||
| } | ||
| } | ||
| // Last-ditch effort to detect Sysbox containers. | ||
| // Check if we have anything mounted as type sysboxfs in /proc/mounts | ||
| mountsData, err := afero.ReadFile(fs, procMounts) | ||
| if err != nil { | ||
| if os.IsNotExist(err) { | ||
| return false, nil | ||
| } | ||
| return false, xerrors.Errorf("read file %s: %w", procMounts, err) | ||
| } | ||
| scn = bufio.NewScanner(bytes.NewReader(mountsData)) | ||
| for scn.Scan() { | ||
| line := scn.Bytes() | ||
| if bytes.Contains(line, []byte("sysboxfs")) { | ||
| return true, nil | ||
| } | ||
| } | ||
| // If we get here, we are _probably_ not running in a container. | ||
| return false, nil | ||
| } | ||
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.