Hojun-Cho
aa0f77b891
Discovers test/wcc/data, hard-pins corpus identity (1,224 fixtures, 314 error / 12 compile / 136 run / 762 run-exit, sha256 name hash), schedules both frontend cells per fixture through os.exec with per-cell capture dirs, and emits the strict wwfix TSV result stream. integration.sh is the black-box owner of the CLI/protocol boundary.
525 lines
15 KiB
Plaintext
525 lines
15 KiB
Plaintext
package wwfixture;
|
|
|
|
import os;
|
|
import strings;
|
|
|
|
let protocolerr: str = "";
|
|
|
|
export fn protocolerror() str = { return protocolerr; };
|
|
|
|
fn protocolfail(message: str) bool = {
|
|
protocolerr = strings.dup(message);
|
|
return false;
|
|
};
|
|
|
|
fn protocolput(b: *[]u8, s: str) void = {
|
|
let i: i32 = 0;
|
|
for (i < s.len) {
|
|
append(*b, s[i]);
|
|
i += 1;
|
|
};
|
|
};
|
|
|
|
fn protocolfield(b: *[]u8, s: str) void = {
|
|
append(*b, '\t': u8);
|
|
protocolput(b, s);
|
|
};
|
|
|
|
fn protocolu64(b: *[]u8, value: u64) void = {
|
|
let digits: [20]u8;
|
|
let n: i32 = 0;
|
|
let v: u64 = value;
|
|
if (v == 0u64) {
|
|
append(*b, '0': u8);
|
|
return;
|
|
};
|
|
for (v != 0u64) {
|
|
digits[n] = '0': u8 + (v % 10u64): u8;
|
|
n += 1;
|
|
v /= 10u64;
|
|
};
|
|
for (n > 0) {
|
|
n -= 1;
|
|
append(*b, digits[n]);
|
|
};
|
|
};
|
|
|
|
fn protocoli64(b: *[]u8, value: i64) void = {
|
|
if (value < 0i64) {
|
|
append(*b, '-': u8);
|
|
protocolu64(b, (-value): u64);
|
|
return;
|
|
};
|
|
protocolu64(b, value: u64);
|
|
};
|
|
|
|
fn protocolintfield(b: *[]u8, value: i64) void = {
|
|
append(*b, '\t': u8);
|
|
protocoli64(b, value);
|
|
};
|
|
|
|
fn protocolnewline(b: *[]u8) void = { append(*b, '\n': u8); };
|
|
|
|
fn protocolwriteall(fd: i32, b: []u8) bool = {
|
|
let off: i32 = 0;
|
|
for (off < b.len) {
|
|
let n: i64 = os.write(fd, b.ptr + (off: u64),
|
|
(b.len - off): u64);
|
|
if (n == -4i64) { continue; };
|
|
if (n <= 0i64) { return protocolfail("cannot write terminal stream"); };
|
|
off += n: i32;
|
|
};
|
|
return true;
|
|
};
|
|
|
|
fn safeprotocolfield(s: str) bool = {
|
|
if (s.len == 0) { return false; };
|
|
let i: i32 = 0;
|
|
for (i < s.len) {
|
|
if (s[i] < 32u8 || s[i] == 127u8) { return false; };
|
|
i += 1;
|
|
};
|
|
return true;
|
|
};
|
|
|
|
fn validrunid(s: str) bool = {
|
|
if (s.len != 16) { return false; };
|
|
let i: i32 = 0;
|
|
for (i < s.len) {
|
|
let c: u8 = s[i];
|
|
if (!((c >= '0': u8 && c <= '9': u8)
|
|
|| (c >= 'a': u8 && c <= 'f': u8))) { return false; };
|
|
i += 1;
|
|
};
|
|
return true;
|
|
};
|
|
|
|
fn validstage(s: stage) bool = { return s == stage.C || s == stage.WW; };
|
|
|
|
fn validphase(p: phase) bool = {
|
|
return p == phase.COMPILE || p == phase.BUILD || p == phase.RUN;
|
|
};
|
|
|
|
fn validverdict(v: verdict) bool = {
|
|
return v == verdict.PASS || v == verdict.FAIL || v == verdict.ERROR;
|
|
};
|
|
|
|
fn validtermination(t: termination) bool = {
|
|
return t == termination.EXIT || t == termination.SIGNAL
|
|
|| t == termination.TIMEOUT || t == termination.SETUP;
|
|
};
|
|
|
|
fn validexpected(c: *corpus, ids: []identity) bool = {
|
|
if (ids.len > nativecount) {
|
|
return protocolfail("selected count exceeds native matrix");
|
|
};
|
|
let i: i32 = 0;
|
|
for (i < ids.len) {
|
|
let id: *identity = &ids[i];
|
|
if (id.ordinal != i + 1) {
|
|
return protocolfail("expected ordinals are not consecutive");
|
|
};
|
|
if (id.fixtureindex < 0 || id.fixtureindex >= c.fixtures.len) {
|
|
return protocolfail("expected fixture index is invalid");
|
|
};
|
|
if (!validstage(id.stage)) {
|
|
return protocolfail("expected stage is invalid");
|
|
};
|
|
let j: i32 = 0;
|
|
for (j < i) {
|
|
if (ids[j].fixtureindex == id.fixtureindex
|
|
&& ids[j].stage == id.stage) {
|
|
return protocolfail("duplicate expected identity");
|
|
};
|
|
j += 1;
|
|
};
|
|
i += 1;
|
|
};
|
|
return true;
|
|
};
|
|
|
|
fn validcapture(r: *cellresult) bool = {
|
|
if (!safeprotocolfield(r.capture)) {
|
|
return protocolfail("unsafe or empty capture field");
|
|
};
|
|
if (r.verdict != verdict.PASS && r.capture == "-") {
|
|
return protocolfail("non-pass result has no retained capture");
|
|
};
|
|
return true;
|
|
};
|
|
|
|
fn validterminationcode(r: *cellresult) bool = {
|
|
if (r.termination == termination.EXIT) {
|
|
if (r.code < 0 || r.code > 255) {
|
|
return protocolfail("exit code is out of range");
|
|
};
|
|
return true;
|
|
};
|
|
if (r.termination == termination.SIGNAL) {
|
|
if (r.code < 1 || r.code > 64) {
|
|
return protocolfail("signal number is out of range");
|
|
};
|
|
return true;
|
|
};
|
|
if (r.termination == termination.TIMEOUT) {
|
|
if (r.code != 0) { return protocolfail("timeout code is not zero"); };
|
|
return true;
|
|
};
|
|
if (r.code >= 0 || r.code < -4095) {
|
|
return protocolfail("setup code is not a negative errno");
|
|
};
|
|
return true;
|
|
};
|
|
|
|
fn validresult(f: *fixture, r: *cellresult) bool = {
|
|
if (!safeprotocolfield(r.id)) {
|
|
return protocolfail("unsafe or empty result identity");
|
|
};
|
|
if (!validstage(r.stage) || !validphase(r.phase)
|
|
|| !validverdict(r.verdict) || !validtermination(r.termination)) {
|
|
return protocolfail("invalid result enum");
|
|
};
|
|
if (r.durationns < 0i64) { return protocolfail("negative duration"); };
|
|
if (!validcapture(r) || !validterminationcode(r)) { return false; };
|
|
if (r.verdict == verdict.PASS && r.termination != termination.EXIT) {
|
|
return protocolfail("pass is not a normal exit");
|
|
};
|
|
if (r.termination == termination.SETUP && r.verdict != verdict.ERROR) {
|
|
return protocolfail("setup termination is not an error");
|
|
};
|
|
if (r.verdict == verdict.FAIL && r.termination == termination.SETUP) {
|
|
return protocolfail("fail result has setup termination");
|
|
};
|
|
if (f.directive == directive.ERROR) {
|
|
if (r.phase != phase.COMPILE) {
|
|
return protocolfail("error fixture has non-compile phase");
|
|
};
|
|
if (r.verdict == verdict.PASS
|
|
&& (r.termination != termination.EXIT || r.code == 0)) {
|
|
return protocolfail("compile-fail pass is not a normal nonzero exit");
|
|
};
|
|
return true;
|
|
};
|
|
if (f.directive == directive.COMPILE) {
|
|
if (r.phase != phase.COMPILE) {
|
|
return protocolfail("compile fixture has non-compile phase");
|
|
};
|
|
if (r.verdict == verdict.PASS
|
|
&& (r.termination != termination.EXIT || r.code != 0)) {
|
|
return protocolfail("compile pass is not a normal exit zero");
|
|
};
|
|
if (r.verdict != verdict.PASS && r.termination == termination.EXIT
|
|
&& r.code == 0) {
|
|
return protocolfail("non-pass compile is a normal exit zero");
|
|
};
|
|
return true;
|
|
};
|
|
if (f.directive != directive.RUN && f.directive != directive.RUNEXIT) {
|
|
return protocolfail("invalid fixture directive");
|
|
};
|
|
if (r.phase == phase.COMPILE) {
|
|
return protocolfail("positive fixture has compile phase");
|
|
};
|
|
if (r.phase == phase.BUILD) {
|
|
if (r.verdict == verdict.PASS) {
|
|
return protocolfail("positive build cannot be a terminal pass");
|
|
};
|
|
if (r.termination == termination.EXIT && r.code == 0) {
|
|
return protocolfail("positive build stopped after normal exit zero");
|
|
};
|
|
return true;
|
|
};
|
|
let expected: i32 = 0;
|
|
if (f.directive == directive.RUNEXIT) { expected = f.exitcode; };
|
|
if (r.verdict == verdict.PASS
|
|
&& (r.termination != termination.EXIT || r.code != expected)) {
|
|
return protocolfail("runtime pass does not match expected exit");
|
|
};
|
|
if (r.verdict == verdict.FAIL && r.termination == termination.EXIT
|
|
&& r.code == expected) {
|
|
return protocolfail("runtime fail matches expected exit");
|
|
};
|
|
return true;
|
|
};
|
|
|
|
fn validrow(c: *corpus, expected: *identity, r: *cellresult) bool = {
|
|
if (r.ordinal != expected.ordinal) {
|
|
return protocolfail("missing, duplicate, or out-of-order ordinal");
|
|
};
|
|
let f: *fixture = &c.fixtures[expected.fixtureindex];
|
|
if (r.id != f.id || r.stage != expected.stage) {
|
|
return protocolfail("unexpected result identity");
|
|
};
|
|
return validresult(f, r);
|
|
};
|
|
|
|
fn putheader(b: *[]u8, runid: str, selected: i32) void = {
|
|
protocolput(b, "wwfix");
|
|
protocolintfield(b, protocolversion: i64);
|
|
protocolfield(b, runid);
|
|
protocolintfield(b, corpuscount: i64);
|
|
protocolintfield(b, nativecount: i64);
|
|
protocolfield(b, corpushash);
|
|
protocolintfield(b, selected: i64);
|
|
protocolnewline(b);
|
|
};
|
|
|
|
fn putrow(b: *[]u8, r: *cellresult) void = {
|
|
protocolput(b, "case");
|
|
protocolintfield(b, r.ordinal: i64);
|
|
protocolfield(b, r.id);
|
|
protocolfield(b, stageword(r.stage));
|
|
protocolfield(b, phaseword(r.phase));
|
|
protocolfield(b, verdictword(r.verdict));
|
|
protocolfield(b, terminationword(r.termination));
|
|
protocolintfield(b, r.code: i64);
|
|
protocolintfield(b, r.durationns);
|
|
protocolfield(b, r.capture);
|
|
protocolnewline(b);
|
|
};
|
|
|
|
fn putfooter(b: *[]u8, selected: i32, failures: i32) void = {
|
|
protocolput(b, "end");
|
|
protocolintfield(b, selected: i64);
|
|
protocolintfield(b, failures: i64);
|
|
protocolnewline(b);
|
|
};
|
|
|
|
fn writestream(fd: i32, runid: str, c: *corpus, ids: []identity,
|
|
results: []cellresult) bool = {
|
|
protocolerr = "";
|
|
if (!validrunid(runid)) { return protocolfail("invalid run ID"); };
|
|
if (!validexpected(c, ids)) { return false; };
|
|
if (results.len != ids.len) {
|
|
return protocolfail("selected result count mismatch");
|
|
};
|
|
let failures: i32 = 0;
|
|
let i: i32 = 0;
|
|
for (i < results.len) {
|
|
if (!validrow(c, &ids[i], &results[i])) { return false; };
|
|
if (results[i].verdict != verdict.PASS) { failures += 1; };
|
|
i += 1;
|
|
};
|
|
let b: []u8;
|
|
putheader(&b, runid, ids.len);
|
|
i = 0;
|
|
for (i < results.len) {
|
|
putrow(&b, &results[i]);
|
|
i += 1;
|
|
};
|
|
putfooter(&b, ids.len, failures);
|
|
return protocolwriteall(fd, b);
|
|
};
|
|
|
|
fn splitfields(line: str, out: *[]str) bool = {
|
|
let fields: []str;
|
|
let start: i32 = 0;
|
|
let i: i32 = 0;
|
|
for (i <= line.len) {
|
|
if (i == line.len || line[i] == '\t': u8) {
|
|
append(fields, line[start:i]);
|
|
start = i + 1;
|
|
};
|
|
i += 1;
|
|
};
|
|
*out = fields;
|
|
return true;
|
|
};
|
|
|
|
fn nextline(data: str, off: *i32, out: *str) bool = {
|
|
if (*off >= data.len) { return false; };
|
|
let end: i32 = *off;
|
|
for (end < data.len && data[end] != '\n': u8) { end += 1; };
|
|
if (end == data.len) { return false; };
|
|
*out = data[*off:end];
|
|
*off = end + 1;
|
|
return true;
|
|
};
|
|
|
|
fn parseu64field(s: str, out: *u64) bool = {
|
|
if (s.len == 0) { return protocolfail("empty integer"); };
|
|
if (s.len > 1 && s[0] == '0': u8) {
|
|
return protocolfail("non-canonical integer");
|
|
};
|
|
let n: u64 = 0u64;
|
|
let i: i32 = 0;
|
|
for (i < s.len) {
|
|
let c: u8 = s[i];
|
|
if (c < '0': u8 || c > '9': u8) {
|
|
return protocolfail("malformed integer");
|
|
};
|
|
let d: u64 = (c - '0': u8): u64;
|
|
if (n > 1844674407370955161u64
|
|
|| (n == 1844674407370955161u64 && d > 5u64)) {
|
|
return protocolfail("integer overflow");
|
|
};
|
|
n = n * 10u64 + d;
|
|
i += 1;
|
|
};
|
|
*out = n;
|
|
return true;
|
|
};
|
|
|
|
fn parsei32nonnegative(s: str, out: *i32) bool = {
|
|
let n: u64 = 0u64;
|
|
if (!parseu64field(s, &n)) { return false; };
|
|
if (n > 2147483647u64) { return protocolfail("i32 overflow"); };
|
|
*out = n: i32;
|
|
return true;
|
|
};
|
|
|
|
fn parsei64nonnegative(s: str, out: *i64) bool = {
|
|
let n: u64 = 0u64;
|
|
if (!parseu64field(s, &n)) { return false; };
|
|
if (n > 9223372036854775807u64) { return protocolfail("i64 overflow"); };
|
|
*out = n: i64;
|
|
return true;
|
|
};
|
|
|
|
fn parsei32field(s: str, out: *i32) bool = {
|
|
if (s.len == 0) { return protocolfail("empty integer"); };
|
|
if (s[0] != '-': u8) { return parsei32nonnegative(s, out); };
|
|
if (s.len == 1 || (s.len > 2 && s[1] == '0': u8)) {
|
|
return protocolfail("malformed negative integer");
|
|
};
|
|
let magnitude: str;
|
|
magnitude.ptr = s.ptr + 1u64;
|
|
magnitude.len = s.len - 1;
|
|
let n: u64 = 0u64;
|
|
if (!parseu64field(magnitude, &n)) { return false; };
|
|
if (n == 0u64 || n > 2147483648u64) {
|
|
return protocolfail("i32 overflow");
|
|
};
|
|
*out = -(n: i32);
|
|
return true;
|
|
};
|
|
|
|
fn parsestagefield(s: str, out: *stage) bool = {
|
|
if (s == "c") { *out = stage.C; return true; };
|
|
if (s == "ww") { *out = stage.WW; return true; };
|
|
return protocolfail("unknown stage");
|
|
};
|
|
|
|
fn parsephasefield(s: str, out: *phase) bool = {
|
|
if (s == "compile") { *out = phase.COMPILE; return true; };
|
|
if (s == "build") { *out = phase.BUILD; return true; };
|
|
if (s == "run") { *out = phase.RUN; return true; };
|
|
return protocolfail("unknown phase");
|
|
};
|
|
|
|
fn parseverdictfield(s: str, out: *verdict) bool = {
|
|
if (s == "pass") { *out = verdict.PASS; return true; };
|
|
if (s == "fail") { *out = verdict.FAIL; return true; };
|
|
if (s == "error") { *out = verdict.ERROR; return true; };
|
|
return protocolfail("unknown verdict");
|
|
};
|
|
|
|
fn parseterminationfield(s: str, out: *termination) bool = {
|
|
if (s == "exit") { *out = termination.EXIT; return true; };
|
|
if (s == "signal") { *out = termination.SIGNAL; return true; };
|
|
if (s == "timeout") { *out = termination.TIMEOUT; return true; };
|
|
if (s == "setup") { *out = termination.SETUP; return true; };
|
|
return protocolfail("unknown termination");
|
|
};
|
|
|
|
fn parseheader(line: str, selected: i32) bool = {
|
|
let fields: []str;
|
|
splitfields(line, &fields);
|
|
if (fields.len != 7) { return protocolfail("malformed header field count"); };
|
|
if (fields[0] != "wwfix") { return protocolfail("missing or malformed header"); };
|
|
let version: i32 = 0;
|
|
let observedcorpus: i32 = 0;
|
|
let observednative: i32 = 0;
|
|
let observedselected: i32 = 0;
|
|
if (!parsei32nonnegative(fields[1], &version)
|
|
|| !parsei32nonnegative(fields[3], &observedcorpus)
|
|
|| !parsei32nonnegative(fields[4], &observednative)
|
|
|| !parsei32nonnegative(fields[6], &observedselected)) { return false; };
|
|
if (version != protocolversion) { return protocolfail("wrong protocol version"); };
|
|
if (!validrunid(fields[2])) { return protocolfail("invalid run ID"); };
|
|
if (observedcorpus != corpuscount || observednative != nativecount
|
|
|| strings.compare(fields[5], corpushash) != 0) {
|
|
return protocolfail("wrong corpus identity");
|
|
};
|
|
if (observedselected != selected) {
|
|
return protocolfail("header selected count mismatch");
|
|
};
|
|
return true;
|
|
};
|
|
|
|
fn parserow(line: str, out: *cellresult) bool = {
|
|
let fields: []str;
|
|
splitfields(line, &fields);
|
|
if (fields.len != 10) { return protocolfail("malformed case field count"); };
|
|
if (fields[0] != "case") { return protocolfail("missing case record"); };
|
|
if (!parsei32nonnegative(fields[1], &out.ordinal)
|
|
|| !parsestagefield(fields[3], &out.stage)
|
|
|| !parsephasefield(fields[4], &out.phase)
|
|
|| !parseverdictfield(fields[5], &out.verdict)
|
|
|| !parseterminationfield(fields[6], &out.termination)
|
|
|| !parsei32field(fields[7], &out.code)
|
|
|| !parsei64nonnegative(fields[8], &out.durationns)) { return false; };
|
|
out.id = fields[2];
|
|
out.capture = fields[9];
|
|
return true;
|
|
};
|
|
|
|
fn parsefooter(line: str, selected: i32, failures: i32) bool = {
|
|
let fields: []str;
|
|
splitfields(line, &fields);
|
|
if (fields.len != 3 || fields[0] != "end") {
|
|
return protocolfail("missing or malformed footer");
|
|
};
|
|
let observedselected: i32 = 0;
|
|
let observedfailures: i32 = 0;
|
|
if (!parsei32nonnegative(fields[1], &observedselected)
|
|
|| !parsei32nonnegative(fields[2], &observedfailures)) { return false; };
|
|
if (observedselected != selected || observedfailures != failures) {
|
|
return protocolfail("inconsistent footer");
|
|
};
|
|
return true;
|
|
};
|
|
|
|
fn validatestream(data: str, c: *corpus, ids: []identity) bool = {
|
|
protocolerr = "";
|
|
if (data.len == 0) { return protocolfail("missing header"); };
|
|
if (data[data.len - 1] != '\n': u8) {
|
|
return protocolfail("incomplete final line");
|
|
};
|
|
let off: i32 = 0;
|
|
let line: str;
|
|
if (!nextline(data, &off, &line)) { return protocolfail("missing header"); };
|
|
if (!parseheader(line, ids.len)) { return false; };
|
|
if (!validexpected(c, ids)) { return false; };
|
|
let failures: i32 = 0;
|
|
let i: i32 = 0;
|
|
for (i < ids.len) {
|
|
if (!nextline(data, &off, &line)) {
|
|
return protocolfail("missing case record");
|
|
};
|
|
let r: cellresult;
|
|
if (!parserow(line, &r) || !validrow(c, &ids[i], &r)) { return false; };
|
|
if (r.verdict != verdict.PASS) { failures += 1; };
|
|
i += 1;
|
|
};
|
|
if (!nextline(data, &off, &line)) { return protocolfail("missing footer"); };
|
|
if (!parsefooter(line, ids.len, failures)) { return false; };
|
|
if (off != data.len) { return protocolfail("trailing records"); };
|
|
return true;
|
|
};
|
|
|
|
export fn validate(data: str, root: str, patterns: []str) bool = {
|
|
protocolerr = "";
|
|
let c: corpus;
|
|
if (!loadcorpus(root, &c)) {
|
|
protocolerr = strings.concat("cannot load corpus: ", error());
|
|
return false;
|
|
};
|
|
let ids: []identity;
|
|
if (!selectidentities(&c, patterns, &ids)) {
|
|
protocolerr = strings.concat("cannot select corpus: ", error());
|
|
return false;
|
|
};
|
|
return validatestream(data, &c, ids);
|
|
};
|