Skip to content

Commit abb4bd3

Browse files
Merge pull request #56 from testingbot/fix/maestro-assert-screenshot-baselines
fix(maestro): bundle assertScreenshot baselines; support includedPaths in config.yaml
2 parents d9ef2b2 + a64f1d0 commit abb4bd3

4 files changed

Lines changed: 312 additions & 22 deletions

File tree

‎README.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,22 @@ testingbot maestro app.apk ./flows --dry-run
324324
`testingbotctl` also prints a warning if a flow you passed will run more than
325325
once because another top-level flow calls it via `runFlow`.
326326

327+
### Files the flows use
328+
329+
Files referenced by `runFlow`, `runScript`, `addMedia` and `assertScreenshot` are bundled automatically, relative to the flow that references them. For `assertScreenshot`, `.png` is added when the path has no extension, as Maestro does.
330+
331+
A path that contains a variable, such as `screenshots/${DEVICE}/home.png`, can't be resolved before the run. List the files under `includedPaths` in `config.yaml`. The globs are relative to the config file and must stay inside the flows directory:
332+
333+
```yaml
334+
# config.yaml
335+
flows:
336+
- 'flows/*'
337+
includedPaths:
338+
- 'screenshots/**'
339+
```
340+
341+
`--dry-run` lists every bundled file and warns about references that won't be in the upload.
342+
327343
---
328344

329345
### Upload once, run many times

‎src/models/maestro_options.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ export interface MaestroConfig {
55
includeTags?: string[];
66
excludeTags?: string[];
77
executionOrder?: 'continue' | 'stop';
8+
/**
9+
* Extra files to bundle, as globs relative to the config file, for paths
10+
* the CLI cannot discover from the flows (e.g. `assertScreenshot` baselines
11+
* whose path contains a variable).
12+
*/
13+
includedPaths?: string[];
814
}
915

1016
export type Orientation = 'PORTRAIT' | 'LANDSCAPE';

‎src/providers/maestro.ts‎

Lines changed: 170 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,8 @@ export interface MissingFileReference {
138138
flowFile: string;
139139
referencedFile: string;
140140
resolvedPath: string;
141+
/** Extra advice printed after the reference, e.g. for variable paths. */
142+
hint?: string;
141143
}
142144

143145
export interface DuplicateFlowReference {
@@ -1072,6 +1074,18 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
10721074
allFlowFiles.push(...filtered);
10731075
}
10741076

1077+
// includedPaths from config.yaml: files the flows reference in ways the
1078+
// dependency scan cannot follow. Added after tag filtering so they stay.
1079+
if (baseDir) {
1080+
for (const file of await this.collectIncludedPaths(baseDir)) {
1081+
const resolved = path.resolve(file);
1082+
if (!allFilesSet.has(resolved)) {
1083+
allFilesSet.add(resolved);
1084+
allFlowFiles.push(file);
1085+
}
1086+
}
1087+
}
1088+
10751089
if (!this.options.quiet) {
10761090
this.logIncludedFiles(allFlowFiles, topLevelFlowFiles, baseDir);
10771091

@@ -1399,40 +1413,86 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
13991413
}
14001414

14011415
/**
1402-
* Load includeTags / excludeTags declared in the Maestro project's
1403-
* config.yaml (or config.yml). Returns undefined fields when no config
1404-
* exists or the values are not arrays.
1416+
* Files matched by `includedPaths` in the Maestro project's config file.
1417+
* Globs resolve against the config file's directory; matches outside
1418+
* baseDir are skipped, since the bundle is rooted there.
14051419
*/
1406-
private async loadConfigTags(
1420+
private async collectIncludedPaths(baseDir: string): Promise<string[]> {
1421+
const config = await this.readProjectConfig(baseDir);
1422+
const patterns = Array.isArray(config?.parsed?.includedPaths)
1423+
? config.parsed.includedPaths.filter(
1424+
(p): p is string => typeof p === 'string',
1425+
)
1426+
: [];
1427+
if (!config || patterns.length === 0) return [];
1428+
1429+
const configDir = path.dirname(config.path);
1430+
const root = path.resolve(baseDir);
1431+
const files: string[] = [];
1432+
for (const pattern of patterns) {
1433+
const matches = await glob(pattern, {
1434+
cwd: configDir,
1435+
absolute: true,
1436+
nodir: true,
1437+
});
1438+
if (matches.length === 0 && !this.options.quiet) {
1439+
logger.warn(`includedPaths: "${pattern}" matched no files`);
1440+
}
1441+
for (const match of matches.sort()) {
1442+
const relative = path.relative(root, match);
1443+
if (relative.startsWith('..') || path.isAbsolute(relative)) {
1444+
if (!this.options.quiet) {
1445+
logger.warn(
1446+
`includedPaths: skipping ${match}, which is outside ${baseDir}`,
1447+
);
1448+
}
1449+
continue;
1450+
}
1451+
files.push(match);
1452+
}
1453+
}
1454+
return files;
1455+
}
1456+
1457+
/** Path and parsed content of the project's config file, if any. */
1458+
private async readProjectConfig(
14071459
baseDir: string,
1408-
): Promise<{ includeTags?: string[]; excludeTags?: string[] }> {
1460+
): Promise<{ path: string; parsed: MaestroConfig | null } | null> {
14091461
const candidates = this.options.configFile
14101462
? [path.resolve(this.options.configFile)]
14111463
: [path.join(baseDir, 'config.yaml'), path.join(baseDir, 'config.yml')];
14121464

14131465
for (const candidate of candidates) {
14141466
try {
14151467
const content = await fs.promises.readFile(candidate, 'utf-8');
1416-
const parsed = yaml.load(content) as MaestroConfig | null;
1417-
if (parsed && typeof parsed === 'object') {
1418-
const include = Array.isArray(parsed.includeTags)
1419-
? parsed.includeTags.filter(
1420-
(t): t is string => typeof t === 'string',
1421-
)
1422-
: undefined;
1423-
const exclude = Array.isArray(parsed.excludeTags)
1424-
? parsed.excludeTags.filter(
1425-
(t): t is string => typeof t === 'string',
1426-
)
1427-
: undefined;
1428-
return { includeTags: include, excludeTags: exclude };
1429-
}
1430-
return {};
1468+
return {
1469+
path: candidate,
1470+
parsed: yaml.load(content) as MaestroConfig | null,
1471+
};
14311472
} catch {
14321473
// try next candidate
14331474
}
14341475
}
1435-
return {};
1476+
return null;
1477+
}
1478+
1479+
/**
1480+
* Load includeTags / excludeTags declared in the Maestro project's
1481+
* config.yaml (or config.yml). Returns undefined fields when no config
1482+
* exists or the values are not arrays.
1483+
*/
1484+
private async loadConfigTags(
1485+
baseDir: string,
1486+
): Promise<{ includeTags?: string[]; excludeTags?: string[] }> {
1487+
const parsed = (await this.readProjectConfig(baseDir))?.parsed;
1488+
if (!parsed || typeof parsed !== 'object') return {};
1489+
const include = Array.isArray(parsed.includeTags)
1490+
? parsed.includeTags.filter((t): t is string => typeof t === 'string')
1491+
: undefined;
1492+
const exclude = Array.isArray(parsed.excludeTags)
1493+
? parsed.excludeTags.filter((t): t is string => typeof t === 'string')
1494+
: undefined;
1495+
return { includeTags: include, excludeTags: exclude };
14361496
}
14371497

14381498
private async filterFlowsByTags(
@@ -1677,6 +1737,31 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
16771737
}
16781738
}
16791739

1740+
// assertScreenshot: baseline image, string or { path: "..." }. Maestro
1741+
// adds .png when the path does not end in it, so try that name too.
1742+
if ('assertScreenshot' in obj) {
1743+
handledKeys.add('assertScreenshot');
1744+
const assertScreenshot = obj.assertScreenshot;
1745+
const baseline =
1746+
typeof assertScreenshot === 'string'
1747+
? assertScreenshot
1748+
: (assertScreenshot as Record<string, unknown>)?.path;
1749+
if (typeof baseline === 'string') {
1750+
const candidates = baseline.toLowerCase().endsWith('.png')
1751+
? [baseline]
1752+
: [baseline, `${baseline}.png`];
1753+
for (const candidate of candidates) {
1754+
await this.tryAddDependency(
1755+
candidate,
1756+
flowFile,
1757+
baseDir,
1758+
dependencies,
1759+
visited,
1760+
);
1761+
}
1762+
}
1763+
}
1764+
16801765
// onFlowStart: array of commands in frontmatter
16811766
if ('onFlowStart' in obj) {
16821767
handledKeys.add('onFlowStart');
@@ -1746,6 +1831,7 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
17461831
'includeTags',
17471832
'excludeTags',
17481833
'env',
1834+
'includedPaths',
17491835
]);
17501836

17511837
for (const [key, propValue] of Object.entries(obj)) {
@@ -1961,6 +2047,47 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
19612047
/**
19622048
* Recursively find missing file references in a YAML value
19632049
*/
2050+
/**
2051+
* An assertScreenshot baseline counts as bundled when the path, or the path
2052+
* plus .png, is included. A path with a ${...} variable cannot be resolved
2053+
* up front; it counts when an included file fits it with any value in place
2054+
* of each variable (which is what includedPaths in config.yaml is for).
2055+
*/
2056+
private findMissingBaseline(
2057+
baseline: string,
2058+
flowFile: string,
2059+
includedFiles: Set<string>,
2060+
): MissingFileReference | undefined {
2061+
const resolvedPath = path.resolve(path.dirname(flowFile), baseline);
2062+
const candidates = baseline.toLowerCase().endsWith('.png')
2063+
? [resolvedPath]
2064+
: [resolvedPath, `${resolvedPath}.png`];
2065+
2066+
if (!/\$\{[^}]+\}/.test(baseline)) {
2067+
if (candidates.some((c) => includedFiles.has(c))) return undefined;
2068+
return { flowFile, referencedFile: baseline, resolvedPath };
2069+
}
2070+
2071+
const patterns = candidates.map(
2072+
(c) =>
2073+
new RegExp(
2074+
`^${c
2075+
.split(/\$\{[^}]+\}/)
2076+
.map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
2077+
.join('[^/\\\\]+')}$`,
2078+
),
2079+
);
2080+
for (const file of includedFiles) {
2081+
if (patterns.some((re) => re.test(file))) return undefined;
2082+
}
2083+
return {
2084+
flowFile,
2085+
referencedFile: baseline,
2086+
resolvedPath,
2087+
hint: 'the path contains a variable, so list the baselines under includedPaths in config.yaml (e.g. "screenshots/**")',
2088+
};
2089+
}
2090+
19642091
private async findMissingInValue(
19652092
value: unknown,
19662093
flowFile: string,
@@ -2069,6 +2196,24 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
20692196
}
20702197
}
20712198

2199+
// Handle assertScreenshot baselines (string or { path })
2200+
if ('assertScreenshot' in obj) {
2201+
handledKeys.add('assertScreenshot');
2202+
const assertScreenshot = obj.assertScreenshot;
2203+
const baseline =
2204+
typeof assertScreenshot === 'string'
2205+
? assertScreenshot
2206+
: (assertScreenshot as Record<string, unknown>)?.path;
2207+
if (typeof baseline === 'string') {
2208+
const missing = this.findMissingBaseline(
2209+
baseline,
2210+
flowFile,
2211+
includedFiles,
2212+
);
2213+
if (missing) missingReferences.push(missing);
2214+
}
2215+
}
2216+
20722217
// Handle file property
20732218
if ('file' in obj && typeof obj.file === 'string') {
20742219
handledKeys.add('file');
@@ -2108,6 +2253,7 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
21082253
'includeTags',
21092254
'excludeTags',
21102255
'env',
2256+
'includedPaths',
21112257
]);
21122258

21132259
for (const [key, propValue] of Object.entries(obj)) {
@@ -2144,7 +2290,9 @@ export default class Maestro extends BaseProvider<MaestroOptions> {
21442290
const flowRelative = baseDir
21452291
? path.relative(baseDir, ref.flowFile)
21462292
: path.basename(ref.flowFile);
2147-
logger.warn(` In ${flowRelative}: ${ref.referencedFile}`);
2293+
logger.warn(
2294+
` In ${flowRelative}: ${ref.referencedFile}${ref.hint ? ` (${ref.hint})` : ''}`,
2295+
);
21482296
}
21492297

21502298
logger.warn(

0 commit comments

Comments
 (0)