Skip to content

Commit 17c1ede

Browse files
AtofStrykerAtofStryker
authored
chore: run battery of tests against @cypress/grep in CI to make sure we do not introduce regressions (#32522)
1 parent 6eb1997 commit 17c1ede

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+721
-489
lines changed

‎.circleci/src/workflows/@workflows.yml‎

Lines changed: 24 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2368,31 +2368,30 @@ jobs:
23682368
resource_class: small
23692369
steps:
23702370
- restore_cached_workspace
2371-
- run: yarn workspace @cypress/grep cypress:run:tags:before
2372-
- run: yarn workspace @cypress/grep cypress:run:tags:describe
2373-
- run: yarn workspace @cypress/grep cypress:run:tags:config
2374-
- run: yarn workspace @cypress/grep cypress:run:burn
2375-
- run: yarn workspace @cypress/grep cypress:run:tags:inherit
2376-
- run: yarn workspace @cypress/grep cypress:run:multiple-registrations
2377-
- run: yarn workspace @cypress/grep cypress:run:tags:nested-describe
2378-
- run: yarn workspace @cypress/grep cypress:run:skip
2379-
- run: yarn workspace @cypress/grep cypress:run:this
2380-
- run: yarn workspace @cypress/grep cypress:run:tags
2381-
- run: yarn workspace @cypress/grep cypress:run:each
2382-
- run: yarn workspace @cypress/grep cypress:run:grep
2383-
- run: yarn workspace @cypress/grep cypress:run:tags:or
2384-
- run: yarn workspace @cypress/grep cypress:run:tags:and
2385-
- run: yarn workspace @cypress/grep cypress:run:tags:and:not
2386-
- run: yarn workspace @cypress/grep cypress:run:or
2387-
- run: yarn workspace @cypress/grep cypress:run:and
2388-
- run: yarn workspace @cypress/grep cypress:run:and:not
2389-
- run: yarn workspace @cypress/grep cypress:run:filter:specs
2390-
- run: yarn workspace @cypress/grep cypress:run:tags:filter:specs
2391-
- run: yarn workspace @cypress/grep cypress:run:omit:specs
2392-
- run: yarn workspace @cypress/grep cypress:run:tags:omit:specs
2393-
- run: yarn workspace @cypress/grep cypress:run:untagged
2394-
- run: yarn workspace @cypress/grep cypress:run:tag:inverted
2395-
- run: yarn workspace @cypress/grep cypress:run:inverted
2371+
- run: yarn workspace @cypress/grep and
2372+
- run: yarn workspace @cypress/grep and:not
2373+
- run: yarn workspace @cypress/grep burn
2374+
- run: yarn workspace @cypress/grep each
2375+
- run: yarn workspace @cypress/grep filter:specs
2376+
- run: yarn workspace @cypress/grep grep
2377+
- run: yarn workspace @cypress/grep inverted
2378+
- run: yarn workspace @cypress/grep multiple-registrations
2379+
- run: yarn workspace @cypress/grep omit:specs
2380+
- run: yarn workspace @cypress/grep or
2381+
- run: yarn workspace @cypress/grep skip
2382+
- run: yarn workspace @cypress/grep tags:inverted
2383+
- run: yarn workspace @cypress/grep tags:and
2384+
- run: yarn workspace @cypress/grep tags:and:not
2385+
- run: yarn workspace @cypress/grep tags:before
2386+
- run: yarn workspace @cypress/grep tags:config
2387+
- run: yarn workspace @cypress/grep tags:describe
2388+
- run: yarn workspace @cypress/grep tags:filter:specs
2389+
- run: yarn workspace @cypress/grep tags:inherit
2390+
- run: yarn workspace @cypress/grep tags:nested-describe
2391+
- run: yarn workspace @cypress/grep tags:omit:specs
2392+
- run: yarn workspace @cypress/grep tags:or
2393+
- run: yarn workspace @cypress/grep this
2394+
- run: yarn workspace @cypress/grep untagged
23962395
- run:
23972396
name: Run unit tests
23982397
command: yarn workspace @cypress/grep test

‎.circleci/workflows.yml‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2002,31 +2002,31 @@ jobs:
20022002
resource_class: small
20032003
steps:
20042004
- restore_cached_workspace
2005-
- run: yarn workspace @cypress/grep cypress:run:tags:before
2006-
- run: yarn workspace @cypress/grep cypress:run:tags:describe
2007-
- run: yarn workspace @cypress/grep cypress:run:tags:config
2008-
- run: yarn workspace @cypress/grep cypress:run:burn
2009-
- run: yarn workspace @cypress/grep cypress:run:tags:inherit
2010-
- run: yarn workspace @cypress/grep cypress:run:multiple-registrations
2011-
- run: yarn workspace @cypress/grep cypress:run:tags:nested-describe
2012-
- run: yarn workspace @cypress/grep cypress:run:skip
2013-
- run: yarn workspace @cypress/grep cypress:run:this
2014-
- run: yarn workspace @cypress/grep cypress:run:tags
2015-
- run: yarn workspace @cypress/grep cypress:run:each
2016-
- run: yarn workspace @cypress/grep cypress:run:grep
2017-
- run: yarn workspace @cypress/grep cypress:run:tags:or
2018-
- run: yarn workspace @cypress/grep cypress:run:tags:and
2019-
- run: yarn workspace @cypress/grep cypress:run:tags:and:not
2020-
- run: yarn workspace @cypress/grep cypress:run:or
2021-
- run: yarn workspace @cypress/grep cypress:run:and
2022-
- run: yarn workspace @cypress/grep cypress:run:and:not
2023-
- run: yarn workspace @cypress/grep cypress:run:filter:specs
2024-
- run: yarn workspace @cypress/grep cypress:run:tags:filter:specs
2025-
- run: yarn workspace @cypress/grep cypress:run:omit:specs
2026-
- run: yarn workspace @cypress/grep cypress:run:tags:omit:specs
2027-
- run: yarn workspace @cypress/grep cypress:run:untagged
2028-
- run: yarn workspace @cypress/grep cypress:run:tag:inverted
2029-
- run: yarn workspace @cypress/grep cypress:run:inverted
2005+
- run: yarn workspace @cypress/grep and
2006+
- run: yarn workspace @cypress/grep and:not
2007+
- run: yarn workspace @cypress/grep burn
2008+
- run: yarn workspace @cypress/grep each
2009+
- run: yarn workspace @cypress/grep filter:specs
2010+
- run: yarn workspace @cypress/grep grep
2011+
- run: yarn workspace @cypress/grep inverted
2012+
- run: yarn workspace @cypress/grep multiple-registrations
2013+
- run: yarn workspace @cypress/grep omit:specs
2014+
- run: yarn workspace @cypress/grep or
2015+
- run: yarn workspace @cypress/grep skip
2016+
- run: yarn workspace @cypress/grep tags
2017+
- run: yarn workspace @cypress/grep tags:inverted
2018+
- run: yarn workspace @cypress/grep tags:and
2019+
- run: yarn workspace @cypress/grep tags:and:not
2020+
- run: yarn workspace @cypress/grep tags:before
2021+
- run: yarn workspace @cypress/grep tags:config
2022+
- run: yarn workspace @cypress/grep tags:describe
2023+
- run: yarn workspace @cypress/grep tags:filter:specs
2024+
- run: yarn workspace @cypress/grep tags:inherit
2025+
- run: yarn workspace @cypress/grep tags:nested-describe
2026+
- run: yarn workspace @cypress/grep tags:omit:specs
2027+
- run: yarn workspace @cypress/grep tags:or
2028+
- run: yarn workspace @cypress/grep this
2029+
- run: yarn workspace @cypress/grep untagged
20302030
- run:
20312031
command: yarn workspace @cypress/grep test
20322032
name: Run unit tests

‎npm/grep/CONTRIBUTING.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
### E2E `@cypress/grep` verification
3+
4+
The `@cypress/grep` plugin requires the user to pass arguments to the Cypress CLI. Because of this, we have a large list of custom CLI scripts inside this package that run with different arguments. The results of the tests run are recorded to a `.json` output so we can compare the run in the future. This way, if any code changes, we should be able to detect regressions or intentional overwrites. If needing to write a new `.json` file or overwrite the existing file, the `OVERWRITE_EXPECTED` environment variable can be set when running the script. The comparison will no-op if a `PROJECT_NAME` environment variable is NOT present.
5+
6+
### Unit Tests
7+
8+
If making changes that only require a unit test, a `vitest` test can be added in the `test/` directory.

‎npm/grep/cypress.config.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
import { defineConfig } from 'cypress'
22
import { plugin as cypressGrepPlugin } from './src/plugin'
3+
import assert from 'node:assert'
4+
import path from 'path'
5+
import fs from 'fs'
6+
import _ from 'lodash'
7+
import debug from 'debug'
8+
9+
const debugInstance = debug('cypress:grep:compare-results')
310

411
export default defineConfig({
512
e2e: {
@@ -13,6 +20,56 @@ export default defineConfig({
1320
},
1421
})
1522

23+
on('after:run', (results) => {
24+
if (!process.env.PROJECT_NAME) {
25+
debugInstance('PROJECT_NAME is not set, skipping comparison...')
26+
27+
return
28+
}
29+
30+
// compare the file to the expected output
31+
let actualArr = []
32+
33+
// @ts-expect-error - runs is not typed
34+
for (const run of results.runs) {
35+
const specObj: { name: string, tests: { name: string, state: string }[] } = { name: run.spec.name, tests: [] }
36+
37+
// iterate through tests and add to results object
38+
for (const test of run.tests) {
39+
specObj.tests.push({
40+
name: test.title[0],
41+
state: test.state,
42+
})
43+
}
44+
45+
actualArr.push(specObj)
46+
}
47+
48+
const expectedPath = path.join(__dirname, 'expected-json', `${process.env.PROJECT_NAME}.json`)
49+
50+
try {
51+
const expectedArr = JSON.parse(fs.readFileSync(expectedPath, 'utf8'))
52+
53+
debugInstance('expected results are: %o', expectedArr)
54+
55+
debugInstance('actual results are: %o', actualArr)
56+
debugInstance('comparing results')
57+
58+
assert.ok(_.isEqual(actualArr, expectedArr))
59+
} catch (error) {
60+
if (process.env.OVERWRITE_EXPECTED) {
61+
debugInstance('Overwriting expected results')
62+
fs.writeFileSync(expectedPath, JSON.stringify(actualArr, null, 2), {
63+
encoding: 'utf8',
64+
flag: 'w',
65+
})
66+
} else {
67+
debugInstance('Error comparing actual and expected results', error)
68+
assert.fail(error)
69+
}
70+
}
71+
})
72+
1673
return config
1774
},
1875
},

‎npm/grep/cypress/e2e/ts.cy.ts‎

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎npm/grep/expected-json/and-not.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
[
2+
{
3+
"name": "test1.cy.ts",
4+
"tests": [
5+
{
6+
"name": "Test 1",
7+
"state": "pending"
8+
}
9+
]
10+
},
11+
{
12+
"name": "test2.cy.ts",
13+
"tests": [
14+
{
15+
"name": "Test 2",
16+
"state": "pending"
17+
}
18+
]
19+
},
20+
{
21+
"name": "test3.cy.ts",
22+
"tests": [
23+
{
24+
"name": "Test 3",
25+
"state": "pending"
26+
}
27+
]
28+
}
29+
]

‎npm/grep/expected-json/and.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
[
2+
{
3+
"name": "test1.cy.ts",
4+
"tests": [
5+
{
6+
"name": "Test 1",
7+
"state": "pending"
8+
}
9+
]
10+
},
11+
{
12+
"name": "test2.cy.ts",
13+
"tests": [
14+
{
15+
"name": "Test 2",
16+
"state": "passed"
17+
}
18+
]
19+
},
20+
{
21+
"name": "test3.cy.ts",
22+
"tests": [
23+
{
24+
"name": "Test 3",
25+
"state": "pending"
26+
}
27+
]
28+
}
29+
]

‎npm/grep/expected-json/burn.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
[
2+
{
3+
"name": "burn.cy.ts",
4+
"tests": [
5+
{
6+
"name": "burning a test N times",
7+
"state": "passed"
8+
},
9+
{
10+
"name": "burning a test N times",
11+
"state": "passed"
12+
},
13+
{
14+
"name": "burning a test N times",
15+
"state": "passed"
16+
},
17+
{
18+
"name": "burning a test N times",
19+
"state": "passed"
20+
},
21+
{
22+
"name": "burning a test N times",
23+
"state": "passed"
24+
},
25+
{
26+
"name": "burning a test N times",
27+
"state": "passed"
28+
},
29+
{
30+
"name": "burning a test N times",
31+
"state": "passed"
32+
},
33+
{
34+
"name": "burning a test N times",
35+
"state": "passed"
36+
},
37+
{
38+
"name": "burning a test N times",
39+
"state": "passed"
40+
},
41+
{
42+
"name": "burning a test N times",
43+
"state": "passed"
44+
}
45+
]
46+
}
47+
]

‎npm/grep/expected-json/each.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
[
2+
{
3+
"name": "each.cy.ts",
4+
"tests": [
5+
{
6+
"name": "tests that use .each work",
7+
"state": "passed"
8+
},
9+
{
10+
"name": "tests that use .each work",
11+
"state": "passed"
12+
},
13+
{
14+
"name": "tests that use .each work",
15+
"state": "passed"
16+
}
17+
]
18+
}
19+
]

‎npm/grep/expected-json/filter-specs.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[
2+
{
3+
"name": "test2.cy.ts",
4+
"tests": [
5+
{
6+
"name": "Test 2",
7+
"state": "passed"
8+
}
9+
]
10+
}
11+
]

0 commit comments

Comments
 (0)