- Notifications
You must be signed in to change notification settings - Fork926
chore: turn e2e enterprise tests into e2e premium tests#14979
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Merged
Uh oh!
There was an error while loading.Please reload this page.
Merged
Changes fromall commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
27 changes: 12 additions & 15 deletions.github/workflows/ci.yaml
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -499,19 +499,19 @@ jobs: | ||
| working-directory: site | ||
| test-e2e: | ||
aslilac marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| # test-e2e fails on 2-core 8GB runners, so we use the 4-core 16GB runner | ||
| runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || 'ubuntu-latest' }} | ||
| needs: changes | ||
| if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main' | ||
| timeout-minutes: 20 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| variant: | ||
| -premium: false | ||
| name: test-e2e | ||
| -premium: true | ||
| name: test-e2e-premium | ||
| name: ${{ matrix.variant.name }} | ||
| steps: | ||
| - name: Checkout | ||
| @@ -535,38 +535,35 @@ jobs: | ||
| - run: pnpm playwright:install | ||
| working-directory: site | ||
| # Run tests that don't requirea premium license withouta premium license | ||
| - run: pnpm playwright:test --forbid-only --workers 1 | ||
| if: ${{ !matrix.variant.premium }} | ||
| env: | ||
| DEBUG: pw:api | ||
| working-directory: site | ||
| # Run all of the tests witha premium license | ||
| - run: pnpm playwright:test --forbid-only --workers 1 | ||
| if: ${{ matrix.variant.premium }} | ||
| env: | ||
| DEBUG: pw:api | ||
| CODER_E2E_LICENSE: ${{ secrets.CODER_E2E_LICENSE }} | ||
| CODER_E2E_REQUIRE_PREMIUM_TESTS: "1" | ||
| working-directory: site | ||
| - name: Upload Playwright Failed Tests | ||
| if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork | ||
| uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1 | ||
| with: | ||
aslilac marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| name: failed-test-videos${{ matrix.variant.premium && '-premium' || '' }} | ||
| path: ./site/test-results/**/*.webm | ||
| retention-days: 7 | ||
| - name: Upload pprof dumps | ||
| if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork | ||
| uses: actions/upload-artifact@604373da6381bf24206979c74d06a550515601b9 # v4.4.1 | ||
| with: | ||
| name: debug-pprof-dumps${{ matrix.variant.premium && '-premium' || '' }} | ||
| path: ./site/test-results/**/debug-pprof-*.txt | ||
| retention-days: 7 | ||
21 changes: 18 additions & 3 deletionssite/e2e/constants.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
42 changes: 41 additions & 1 deletionsite/e2e/expectUrl.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
10 changes: 5 additions & 5 deletionssite/e2e/global.setup.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
101 changes: 68 additions & 33 deletionssite/e2e/helpers.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -16,9 +16,10 @@ import { | ||
| agentPProfPort, | ||
| coderMain, | ||
| coderPort, | ||
| defaultOrganizationName, | ||
| license, | ||
| premiumTestsRequired, | ||
| prometheusPort, | ||
| requireTerraformTests, | ||
| } from "./constants"; | ||
| import { expectUrl } from "./expectUrl"; | ||
| @@ -35,22 +36,28 @@ import { | ||
| type RichParameter, | ||
| } from "./provisionerGenerated"; | ||
| /** | ||
| * requiresLicense will skip the test if we're not running with a license added | ||
| */ | ||
| export function requiresLicense() { | ||
| if (premiumTestsRequired) { | ||
| return; | ||
| } | ||
| test.skip(!license); | ||
| } | ||
| /** | ||
| * requireTerraformProvisioner by default is enabled. | ||
| */ | ||
| export function requireTerraformProvisioner() { | ||
| test.skip(!requireTerraformTests); | ||
| } | ||
| /** | ||
| * createWorkspace creates a workspace for a template. It does not wait for it | ||
| * to be running, but it does navigate to the page. | ||
| */ | ||
| export const createWorkspace = async ( | ||
| page: Page, | ||
| templateName: string, | ||
| @@ -90,7 +97,7 @@ export const createWorkspace = async ( | ||
| await expectUrl(page).toHavePathName(`/@admin/${name}`); | ||
| await page.waitForSelector("[data-testid='build-status'] >> text=Running", { | ||
| state: "visible", | ||
| }); | ||
| return name; | ||
| @@ -151,8 +158,10 @@ export const verifyParameters = async ( | ||
| } | ||
| }; | ||
| /** | ||
| * StarterTemplates are ids of starter templates that can be used in place of | ||
| * the responses payload. These starter templates will require real provisioners. | ||
| */ | ||
| export enum StarterTemplates { | ||
| STARTER_DOCKER = "docker", | ||
| } | ||
| @@ -166,11 +175,14 @@ function isStarterTemplate( | ||
| return typeof input === "string"; | ||
| } | ||
| /** | ||
| * createTemplate navigates to the /templates/new page and uploads a template | ||
| * with the resources provided in the responses argument. | ||
| */ | ||
| export const createTemplate = async ( | ||
| page: Page, | ||
| responses?: EchoProvisionerResponses | StarterTemplates, | ||
| orgName = defaultOrganizationName, | ||
| ): Promise<string> => { | ||
| let path = "/templates/new"; | ||
| if (isStarterTemplate(responses)) { | ||
| @@ -191,31 +203,47 @@ export const createTemplate = async ( | ||
| }); | ||
| } | ||
| // If the organization picker is present on the page, select the default | ||
| // organization. | ||
| const orgPicker = page.getByLabel("Belongs to *"); | ||
aslilac marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| const organizationsEnabled = await orgPicker.isVisible(); | ||
| if (organizationsEnabled) { | ||
| await orgPicker.click(); | ||
| await page.getByText(orgName, { exact: true }).click(); | ||
| } | ||
| const name = randomName(); | ||
| await page.getByLabel("Name *").fill(name); | ||
| await page.getByTestId("form-submit").click(); | ||
| await expectUrl(page).toHavePathName( | ||
| organizationsEnabled | ||
| ? `/templates/${orgName}/${name}/files` | ||
| : `/templates/${name}/files`, | ||
| { | ||
| timeout: 30000, | ||
| }, | ||
| ); | ||
| return name; | ||
| }; | ||
| /** | ||
| * createGroup navigates to the /groups/create page and creates a group with a | ||
| * random name. | ||
| */ | ||
| export const createGroup = async (page: Page): Promise<string> => { | ||
| await page.goto("/groups/create", { waitUntil: "domcontentloaded" }); | ||
| await expectUrl(page).toHavePathName("/groups/create"); | ||
| const name = randomName(); | ||
| await page.getByLabel("Name", { exact: true }).fill(name); | ||
| await page.getByTestId("form-submit").click(); | ||
| await expectUrl(page).toHavePathName(`/groups/${name}`); | ||
| return name; | ||
| }; | ||
| /** | ||
| * sshIntoWorkspace spawns a Coder SSH process and a client connected to it. | ||
| */ | ||
| export const sshIntoWorkspace = async ( | ||
| page: Page, | ||
| workspace: string, | ||
| @@ -298,17 +326,21 @@ export const buildWorkspaceWithParameters = async ( | ||
| }); | ||
| }; | ||
| /** | ||
| * startAgent runs the coder agent with the provided token. It waits for the | ||
| * agent to be ready before returning. | ||
| */ | ||
| export const startAgent = async ( | ||
| page: Page, | ||
| token: string, | ||
| ): Promise<ChildProcess> => { | ||
| return startAgentWithCommand(page, token, "go", "run", coderMain); | ||
| }; | ||
| /** | ||
| * downloadCoderVersion downloads the version provided into a temporary dir and | ||
| * caches it so subsequent calls are fast. | ||
| */ | ||
| export const downloadCoderVersion = async ( | ||
| version: string, | ||
| ): Promise<string> => { | ||
| @@ -448,8 +480,10 @@ interface EchoProvisionerResponses { | ||
| apply?: RecursivePartial<Response>[]; | ||
| } | ||
| /** | ||
| * createTemplateVersionTar consumes a series of echo provisioner protobufs and | ||
| * converts it into an uploadable tar file. | ||
| */ | ||
| const createTemplateVersionTar = async ( | ||
| responses?: EchoProvisionerResponses, | ||
| ): Promise<Buffer> => { | ||
| @@ -619,8 +653,10 @@ export const randomName = () => { | ||
| return randomUUID().slice(0, 8); | ||
| }; | ||
| /** | ||
| * Awaiter is a helper that allows you to wait for a callback to be called. It | ||
| * is useful for waiting for events to occur. | ||
| */ | ||
| export class Awaiter { | ||
| private promise: Promise<void>; | ||
| private callback?: () => void; | ||
| @@ -825,7 +861,6 @@ export const updateTemplateSettings = async ( | ||
| await page.goto(`/templates/${templateName}/settings`, { | ||
| waitUntil: "domcontentloaded", | ||
| }); | ||
| for (const [key, value] of Object.entries(templateSettingValues)) { | ||
| // Skip max_port_share_level for now since the frontend is not yet able to handle it | ||
| @@ -839,7 +874,7 @@ export const updateTemplateSettings = async ( | ||
| await page.getByTestId("form-submit").click(); | ||
| const name = templateSettingValues.name ?? templateName; | ||
| await expectUrl(page).toHavePathNameEndingWith(`/${name}`); | ||
| }; | ||
| export const updateWorkspace = async ( | ||
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.