[igt-dev] [PATCH i-g-t v2 2/2] runner/json_tests: Test handling of unprintable output from tests

Petri Latvala petri.latvala at intel.com
Fri Jan 10 12:06:42 UTC 2020 

A simple test output with numbers from 1 to 255, both in plain text
form and as a single byte with that particular value.

Note that the json spec doesn't require \u-encoding for characters
other than '"', '\' and the range U+0000 to U+001F, the raw
non-\u-encoded UTF-8 in the reference.json file for bytes 128 and up
is what libjson-c outputs for those codepoints and is valid.

The validity of the json file can be verified with iconv, i.e.

 $ iconv -f UTF-8 reference.json -o /dev/null && echo it is utf-8

v2: Rebase over dynamic subtest tests, trivial

Signed-off-by: Petri Latvala <petri.latvala at intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler at intel.com> #v1
---
 .../unprintable-characters/0/dmesg.txt        |   5 +
 .../unprintable-characters/0/err.txt          | 258 +++++++++++++++++
 .../unprintable-characters/0/journal.txt      |   2 +
 .../unprintable-characters/0/out.txt          | 259 ++++++++++++++++++
 .../unprintable-characters/README.txt         |   2 +
 .../unprintable-characters/endtime.txt        |   1 +
 .../unprintable-characters/joblist.txt        |   1 +
 .../unprintable-characters/metadata.txt       |  12 +
 .../unprintable-characters/reference.json     |  72 +++++
 .../unprintable-characters/starttime.txt      |   1 +
 .../unprintable-characters/uname.txt          |   1 +
 runner/runner_json_tests.c                    |   1 +
 12 files changed, 615 insertions(+)
 create mode 100644 runner/json_tests_data/unprintable-characters/0/dmesg.txt
 create mode 100644 runner/json_tests_data/unprintable-characters/0/err.txt
 create mode 100644 runner/json_tests_data/unprintable-characters/0/journal.txt
 create mode 100644 runner/json_tests_data/unprintable-characters/0/out.txt
 create mode 100644 runner/json_tests_data/unprintable-characters/README.txt
 create mode 100644 runner/json_tests_data/unprintable-characters/endtime.txt
 create mode 100644 runner/json_tests_data/unprintable-characters/joblist.txt
 create mode 100644 runner/json_tests_data/unprintable-characters/metadata.txt
 create mode 100644 runner/json_tests_data/unprintable-characters/reference.json
 create mode 100644 runner/json_tests_data/unprintable-characters/starttime.txt
 create mode 100644 runner/json_tests_data/unprintable-characters/uname.txt

diff --git a/runner/json_tests_data/unprintable-characters/0/dmesg.txt b/runner/json_tests_data/unprintable-characters/0/dmesg.txt
new file mode 100644
index 00000000..6aac2735
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/0/dmesg.txt
@@ -0,0 +1,5 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+14,955,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/unprintable-characters/0/err.txt b/runner/json_tests_data/unprintable-characters/0/err.txt
new file mode 100644
index 00000000..34d19985
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/0/err.txt
@@ -0,0 +1,258 @@
+Starting subtest: first-subtest
+1:
+2:
+3:
+4:
+5:
+6:
+7:
+8:
+9:	
+10:
+
+11:
+12:
+13:
+14:
+15:
+16:
+17:
+18:
+19:
+20:
+21:
+22:
+23:
+24:
+25:
+26:
+27:
+28:
+29:
+30:
+31:
+32: 
+33:!
+34:"
+35:#
+36:$
+37:%
+38:&
+39:'
+40:(
+41:)
+42:*
+43:+
+44:,
+45:-
+46:.
+47:/
+48:0
+49:1
+50:2
+51:3
+52:4
+53:5
+54:6
+55:7
+56:8
+57:9
+58::
+59:;
+60:<
+61:=
+62:>
+63:?
+64:@
+65:A
+66:B
+67:C
+68:D
+69:E
+70:F
+71:G
+72:H
+73:I
+74:J
+75:K
+76:L
+77:M
+78:N
+79:O
+80:P
+81:Q
+82:R
+83:S
+84:T
+85:U
+86:V
+87:W
+88:X
+89:Y
+90:Z
+91:[
+92:\
+93:]
+94:^
+95:_
+96:`
+97:a
+98:b
+99:c
+100:d
+101:e
+102:f
+103:g
+104:h
+105:i
+106:j
+107:k
+108:l
+109:m
+110:n
+111:o
+112:p
+113:q
+114:r
+115:s
+116:t
+117:u
+118:v
+119:w
+120:x
+121:y
+122:z
+123:{
+124:|
+125:}
+126:~
+127:
+128:€
+129:
+130:‚
+131:ƒ
+132:„
+133:…
+134:†
+135:‡
+136:ˆ
+137:‰
+138:Š
+139:‹
+140:Œ
+141:
+142:Ž
+143:
+144:
+145:‘
+146:’
+147:“
+148:”
+149:•
+150:–
+151:—
+152:˜
+153:™
+154:š
+155:›
+156:œ
+157:
+158:ž
+159:Ÿ
+160: 
+161:¡
+162:¢
+163:£
+164:¤
+165:¥
+166:¦
+167:§
+168:¨
+169:©
+170:ª
+171:«
+172:¬
+173:­
+174:®
+175:¯
+176:°
+177:±
+178:²
+179:³
+180:´
+181:µ
+182:¶
+183:·
+184:¸
+185:¹
+186:º
+187:»
+188:¼
+189:½
+190:¾
+191:¿
+192:À
+193:Á
+194:Â
+195:Ã
+196:Ä
+197:Å
+198:Æ
+199:Ç
+200:È
+201:É
+202:Ê
+203:Ë
+204:Ì
+205:Í
+206:Î
+207:Ï
+208:Ð
+209:Ñ
+210:Ò
+211:Ó
+212:Ô
+213:Õ
+214:Ö
+215:×
+216:Ø
+217:Ù
+218:Ú
+219:Û
+220:Ü
+221:Ý
+222:Þ
+223:ß
+224:à
+225:á
+226:â
+227:ã
+228:ä
+229:å
+230:æ
+231:ç
+232:è
+233:é
+234:ê
+235:ë
+236:ì
+237:í
+238:î
+239:ï
+240:ð
+241:ñ
+242:ò
+243:ó
+244:ô
+245:õ
+246:ö
+247:÷
+248:ø
+249:ù
+250:ú
+251:û
+252:ü
+253:ý
+254:þ
+255:ÿ
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/unprintable-characters/0/journal.txt b/runner/json_tests_data/unprintable-characters/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/unprintable-characters/0/out.txt b/runner/json_tests_data/unprintable-characters/0/out.txt
new file mode 100644
index 00000000..c597674a
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/0/out.txt
@@ -0,0 +1,259 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+1:
+2:
+3:
+4:
+5:
+6:
+7:
+8:
+9:	
+10:
+
+11:
+12:
+13:
+14:
+15:
+16:
+17:
+18:
+19:
+20:
+21:
+22:
+23:
+24:
+25:
+26:
+27:
+28:
+29:
+30:
+31:
+32: 
+33:!
+34:"
+35:#
+36:$
+37:%
+38:&
+39:'
+40:(
+41:)
+42:*
+43:+
+44:,
+45:-
+46:.
+47:/
+48:0
+49:1
+50:2
+51:3
+52:4
+53:5
+54:6
+55:7
+56:8
+57:9
+58::
+59:;
+60:<
+61:=
+62:>
+63:?
+64:@
+65:A
+66:B
+67:C
+68:D
+69:E
+70:F
+71:G
+72:H
+73:I
+74:J
+75:K
+76:L
+77:M
+78:N
+79:O
+80:P
+81:Q
+82:R
+83:S
+84:T
+85:U
+86:V
+87:W
+88:X
+89:Y
+90:Z
+91:[
+92:\
+93:]
+94:^
+95:_
+96:`
+97:a
+98:b
+99:c
+100:d
+101:e
+102:f
+103:g
+104:h
+105:i
+106:j
+107:k
+108:l
+109:m
+110:n
+111:o
+112:p
+113:q
+114:r
+115:s
+116:t
+117:u
+118:v
+119:w
+120:x
+121:y
+122:z
+123:{
+124:|
+125:}
+126:~
+127:
+128:€
+129:
+130:‚
+131:ƒ
+132:„
+133:…
+134:†
+135:‡
+136:ˆ
+137:‰
+138:Š
+139:‹
+140:Œ
+141:
+142:Ž
+143:
+144:
+145:‘
+146:’
+147:“
+148:”
+149:•
+150:–
+151:—
+152:˜
+153:™
+154:š
+155:›
+156:œ
+157:
+158:ž
+159:Ÿ
+160: 
+161:¡
+162:¢
+163:£
+164:¤
+165:¥
+166:¦
+167:§
+168:¨
+169:©
+170:ª
+171:«
+172:¬
+173:­
+174:®
+175:¯
+176:°
+177:±
+178:²
+179:³
+180:´
+181:µ
+182:¶
+183:·
+184:¸
+185:¹
+186:º
+187:»
+188:¼
+189:½
+190:¾
+191:¿
+192:À
+193:Á
+194:Â
+195:Ã
+196:Ä
+197:Å
+198:Æ
+199:Ç
+200:È
+201:É
+202:Ê
+203:Ë
+204:Ì
+205:Í
+206:Î
+207:Ï
+208:Ð
+209:Ñ
+210:Ò
+211:Ó
+212:Ô
+213:Õ
+214:Ö
+215:×
+216:Ø
+217:Ù
+218:Ú
+219:Û
+220:Ü
+221:Ý
+222:Þ
+223:ß
+224:à
+225:á
+226:â
+227:ã
+228:ä
+229:å
+230:æ
+231:ç
+232:è
+233:é
+234:ê
+235:ë
+236:ì
+237:í
+238:î
+239:ï
+240:ð
+241:ñ
+242:ò
+243:ó
+244:ô
+245:õ
+246:ö
+247:÷
+248:ø
+249:ù
+250:ú
+251:û
+252:ü
+253:ý
+254:þ
+255:ÿ
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/unprintable-characters/README.txt b/runner/json_tests_data/unprintable-characters/README.txt
new file mode 100644
index 00000000..046605ea
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/README.txt
@@ -0,0 +1,2 @@
+Any nonprintable output in stdout and stderr should yield a json file
+that is still valid UTF-8.
diff --git a/runner/json_tests_data/unprintable-characters/endtime.txt b/runner/json_tests_data/unprintable-characters/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/unprintable-characters/joblist.txt b/runner/json_tests_data/unprintable-characters/joblist.txt
new file mode 100644
index 00000000..81f914a7
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/joblist.txt
@@ -0,0 +1 @@
+successtest first-subtest
diff --git a/runner/json_tests_data/unprintable-characters/metadata.txt b/runner/json_tests_data/unprintable-characters/metadata.txt
new file mode 100644
index 00000000..c501ae0e
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/metadata.txt
@@ -0,0 +1,12 @@
+abort_mask : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/unprintable-characters/reference.json b/runner/json_tests_data/unprintable-characters/reference.json
new file mode 100644
index 00000000..d3add1eb
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/reference.json
@@ -0,0 +1,72 @@
+{
+  "__type__":"TestrunResult",
+  "results_version":10,
+  "name":"normal-run",
+  "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+  "time_elapsed":{
+    "__type__":"TimeAttribute",
+    "start":1539953735.1110389,
+    "end":1539953735.1723731
+  },
+  "tests":{
+    "igt at successtest@first-subtest":{
+      "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\n1:\u0001\n2:\u0002\n3:\u0003\n4:\u0004\n5:\u0005\n6:\u0006\n7:\u0007\n8:\b\n9:\t\n10:\n\n11:\u000b\n12:\f\n13:\r\n14:\u000e\n15:\u000f\n16:\u0010\n17:\u0011\n18:\u0012\n19:\u0013\n20:\u0014\n21:\u0015\n22:\u0016\n23:\u0017\n24:\u0018\n25:\u0019\n26:\u001a\n27:\u001b\n28:\u001c\n29:\u001d\n30:\u001e\n31:\u001f\n32: \n33:!\n34:\"\n35:#\n36:$\n37:%\n38:&\n39:'\n40:(\n41:)\n42:*\n43:+\n44:,\n45:-\n46:.\n47:\/\n48:0\n49:1\n50:2\n51:3\n52:4\n53:5\n54:6\n55:7\n56:8\n57:9\n58::\n59:;\n60:<\n61:=\n62:>\n63:?\n64:@\n65:A\n66:B\n67:C\n68:D\n69:E\n70:F\n71:G\n72:H\n73:I\n74:J\n75:K\n76:L\n77:M\n78:N\n79:O\n80:P\n81:Q\n82:R\n83:S\n84:T\n85:U\n86:V\n87:W\n88:X\n89:Y\n90:Z\n91:[\n92:\\\n93:]\n94:^\n95:_\n96:`\n97:a\n98:b\n99:c\n100:d\n101:e\n102:f\n103:g\n104:h\n105:i\n106:j\n107:k\n108:l\n109:m\n110:n\n111:o\n112:p\n113:q\n114:r\n115:s\n116:t\n117:u\n118:v\n119:w\n120:x\n121:y\n122:z\n123:{\n124:|\n125:}\n126:~\n127:\n128:€\n129:\n130:‚\n131:ƒ\n132:„\n133:…\n134:†\n135:‡\n136:ˆ\n137:‰\n138:Š\n139:‹\n140:Œ\n141:\n142:Ž\n143:\n144:\n145:‘\n146:’\n147:“\n148:”\n149:•\n150:–\n151:—\n152:˜\n153:™\n154:š\n155:›\n156:œ\n157:\n158:ž\n159:Ÿ\n160: \n161:¡\n162:¢\n163:£\n164:¤\n165:¥\n166:¦\n167:§\n168:¨\n169:©\n170:ª\n171:«\n172:¬\n173:­\n174:®\n175:¯\n176:°\n177:±\n178:²\n179:³\n180:´\n181:µ\n182:¶\n183:·\n184:¸\n185:¹\n186:º\n187:»\n188:¼\n189:½\n190:¾\n191:¿\n192:À\n193:Á\n194:Â\n195:Ã\n196:Ä\n197:Å\n198:Æ\n199:Ç\n200:È\n201:É\n202:Ê\n203:Ë\n204:Ì\n205:Í\n206:Î\n207:Ï\n208:Ð\n209:Ñ\n210:Ò\n211:Ó\n212:Ô\n213:Õ\n214:Ö\n215:×\n216:Ø\n217:Ù\n218:Ú\n219:Û\n220:Ü\n221:Ý\n222:Þ\n223:ß\n224:à\n225:á\n226:â\n227:ã\n228:ä\n229:å\n230:æ\n231:ç\n232:è\n233:é\n234:ê\n235:ë\n236:ì\n237:í\n238:î\n239:ï\n240:ð\n241:ñ\n242:ò\n243:ó\n244:ô\n245:õ\n246:ö\n247:÷\n248:ø\n249:ù\n250:ú\n251:û\n252:ü\n253:ý\n254:þ\n255:ÿ\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"warn",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: first-subtest\n1:\u0001\n2:\u0002\n3:\u0003\n4:\u0004\n5:\u0005\n6:\u0006\n7:\u0007\n8:\b\n9:\t\n10:\n\n11:\u000b\n12:\f\n13:\r\n14:\u000e\n15:\u000f\n16:\u0010\n17:\u0011\n18:\u0012\n19:\u0013\n20:\u0014\n21:\u0015\n22:\u0016\n23:\u0017\n24:\u0018\n25:\u0019\n26:\u001a\n27:\u001b\n28:\u001c\n29:\u001d\n30:\u001e\n31:\u001f\n32: \n33:!\n34:\"\n35:#\n36:$\n37:%\n38:&\n39:'\n40:(\n41:)\n42:*\n43:+\n44:,\n45:-\n46:.\n47:\/\n48:0\n49:1\n50:2\n51:3\n52:4\n53:5\n54:6\n55:7\n56:8\n57:9\n58::\n59:;\n60:<\n61:=\n62:>\n63:?\n64:@\n65:A\n66:B\n67:C\n68:D\n69:E\n70:F\n71:G\n72:H\n73:I\n74:J\n75:K\n76:L\n77:M\n78:N\n79:O\n80:P\n81:Q\n82:R\n83:S\n84:T\n85:U\n86:V\n87:W\n88:X\n89:Y\n90:Z\n91:[\n92:\\\n93:]\n94:^\n95:_\n96:`\n97:a\n98:b\n99:c\n100:d\n101:e\n102:f\n103:g\n104:h\n105:i\n106:j\n107:k\n108:l\n109:m\n110:n\n111:o\n112:p\n113:q\n114:r\n115:s\n116:t\n117:u\n118:v\n119:w\n120:x\n121:y\n122:z\n123:{\n124:|\n125:}\n126:~\n127:\n128:€\n129:\n130:‚\n131:ƒ\n132:„\n133:…\n134:†\n135:‡\n136:ˆ\n137:‰\n138:Š\n139:‹\n140:Œ\n141:\n142:Ž\n143:\n144:\n145:‘\n146:’\n147:“\n148:”\n149:•\n150:–\n151:—\n152:˜\n153:™\n154:š\n155:›\n156:œ\n157:\n158:ž\n159:Ÿ\n160: \n161:¡\n162:¢\n163:£\n164:¤\n165:¥\n166:¦\n167:§\n168:¨\n169:©\n170:ª\n171:«\n172:¬\n173:­\n174:®\n175:¯\n176:°\n177:±\n178:²\n179:³\n180:´\n181:µ\n182:¶\n183:·\n184:¸\n185:¹\n186:º\n187:»\n188:¼\n189:½\n190:¾\n191:¿\n192:À\n193:Á\n194:Â\n195:Ã\n196:Ä\n197:Å\n198:Æ\n199:Ç\n200:È\n201:É\n202:Ê\n203:Ë\n204:Ì\n205:Í\n206:Î\n207:Ï\n208:Ð\n209:Ñ\n210:Ò\n211:Ó\n212:Ô\n213:Õ\n214:Ö\n215:×\n216:Ø\n217:Ù\n218:Ú\n219:Û\n220:Ü\n221:Ý\n222:Þ\n223:ß\n224:à\n225:á\n226:â\n227:ã\n228:ä\n229:å\n230:æ\n231:ç\n232:è\n233:é\n234:ê\n235:ë\n236:ì\n237:í\n238:î\n239:ï\n240:ð\n241:ñ\n242:ò\n243:ó\n244:ô\n245:õ\n246:ö\n247:÷\n248:ø\n249:ù\n250:ú\n251:û\n252:ü\n253:ý\n254:þ\n255:ÿ\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
+    }
+  },
+  "totals":{
+    "":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":1
+    },
+    "root":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":1
+    },
+    "igt at successtest":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":1
+    }
+  },
+  "runtimes":{
+    "igt at successtest":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.014
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/runner/json_tests_data/unprintable-characters/starttime.txt b/runner/json_tests_data/unprintable-characters/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/unprintable-characters/uname.txt b/runner/json_tests_data/unprintable-characters/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/unprintable-characters/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/runner_json_tests.c b/runner/runner_json_tests.c
index 21e87a02..bf4c285b 100644
--- a/runner/runner_json_tests.c
+++ b/runner/runner_json_tests.c
@@ -164,6 +164,7 @@ static const char *dirnames[] = {
 	"dmesg-warn-level-one-piglit-style",
 	"dynamic-subtests",
 	"dynamic-subtest-name-in-multiple-subtests",
+	"unprintable-characters",
 };
 
 igt_main
-- 
2.20.1

More information about the igt-dev mailing list