catch_reporter_json.cpp
1 2 // Copyright Catch2 Authors 3 // Distributed under the Boost Software License, Version 1.0. 4 // (See accompanying file LICENSE.txt or copy at 5 // https://www.boost.org/LICENSE_1_0.txt) 6 7 // SPDX-License-Identifier: BSL-1.0 8 // 9 #include <catch2/catch_test_case_info.hpp> 10 #include <catch2/catch_test_spec.hpp> 11 #include <catch2/catch_version.hpp> 12 #include <catch2/interfaces/catch_interfaces_config.hpp> 13 #include <catch2/internal/catch_list.hpp> 14 #include <catch2/internal/catch_string_manip.hpp> 15 #include <catch2/reporters/catch_reporter_json.hpp> 16 17 namespace Catch { 18 namespace { 19 void writeSourceInfo( JsonObjectWriter& writer, 20 SourceLineInfo const& sourceInfo ) { 21 auto source_location_writer = 22 writer.write( "source-location"_sr ).writeObject(); 23 source_location_writer.write( "filename"_sr ) 24 .write( sourceInfo.file ); 25 source_location_writer.write( "line"_sr ).write( sourceInfo.line ); 26 } 27 28 void writeTags( JsonArrayWriter writer, std::vector<Tag> const& tags ) { 29 for ( auto const& tag : tags ) { 30 writer.write( tag.original ); 31 } 32 } 33 34 void writeProperties( JsonArrayWriter writer, 35 TestCaseInfo const& info ) { 36 if ( info.isHidden() ) { writer.write( "is-hidden"_sr ); } 37 if ( info.okToFail() ) { writer.write( "ok-to-fail"_sr ); } 38 if ( info.expectedToFail() ) { 39 writer.write( "expected-to-fail"_sr ); 40 } 41 if ( info.throws() ) { writer.write( "throws"_sr ); } 42 } 43 44 } // namespace 45 46 JsonReporter::JsonReporter( ReporterConfig&& config ): 47 StreamingReporterBase{ CATCH_MOVE( config ) } { 48 49 m_preferences.shouldRedirectStdOut = true; 50 // TBD: Do we want to report all assertions? XML reporter does 51 // not, but for machine-parseable reporters I think the answer 52 // should be yes. 53 m_preferences.shouldReportAllAssertions = true; 54 55 m_objectWriters.emplace( m_stream ); 56 m_writers.emplace( Writer::Object ); 57 auto& writer = m_objectWriters.top(); 58 59 writer.write( "version"_sr ).write( 1 ); 60 61 { 62 auto metadata_writer = writer.write( "metadata"_sr ).writeObject(); 63 metadata_writer.write( "name"_sr ).write( m_config->name() ); 64 metadata_writer.write( "rng-seed"_sr ).write( m_config->rngSeed() ); 65 metadata_writer.write( "catch2-version"_sr ) 66 .write( libraryVersion() ); 67 if ( m_config->testSpec().hasFilters() ) { 68 metadata_writer.write( "filters"_sr ) 69 .write( m_config->testSpec() ); 70 } 71 } 72 } 73 74 JsonReporter::~JsonReporter() { 75 endListing(); 76 // TODO: Ensure this closes the top level object, add asserts 77 assert( m_writers.size() == 1 && "Only the top level object should be open" ); 78 assert( m_writers.top() == Writer::Object ); 79 endObject(); 80 m_stream << '\n' << std::flush; 81 assert( m_writers.empty() ); 82 } 83 84 JsonArrayWriter& JsonReporter::startArray() { 85 m_arrayWriters.emplace( m_arrayWriters.top().writeArray() ); 86 m_writers.emplace( Writer::Array ); 87 return m_arrayWriters.top(); 88 } 89 JsonArrayWriter& JsonReporter::startArray( StringRef key ) { 90 m_arrayWriters.emplace( 91 m_objectWriters.top().write( key ).writeArray() ); 92 m_writers.emplace( Writer::Array ); 93 return m_arrayWriters.top(); 94 } 95 96 JsonObjectWriter& JsonReporter::startObject() { 97 m_objectWriters.emplace( m_arrayWriters.top().writeObject() ); 98 m_writers.emplace( Writer::Object ); 99 return m_objectWriters.top(); 100 } 101 JsonObjectWriter& JsonReporter::startObject( StringRef key ) { 102 m_objectWriters.emplace( 103 m_objectWriters.top().write( key ).writeObject() ); 104 m_writers.emplace( Writer::Object ); 105 return m_objectWriters.top(); 106 } 107 108 void JsonReporter::endObject() { 109 assert( isInside( Writer::Object ) ); 110 m_objectWriters.pop(); 111 m_writers.pop(); 112 } 113 void JsonReporter::endArray() { 114 assert( isInside( Writer::Array ) ); 115 m_arrayWriters.pop(); 116 m_writers.pop(); 117 } 118 119 bool JsonReporter::isInside( Writer writer ) { 120 return !m_writers.empty() && m_writers.top() == writer; 121 } 122 123 void JsonReporter::startListing() { 124 if ( !m_startedListing ) { startObject( "listings"_sr ); } 125 m_startedListing = true; 126 } 127 void JsonReporter::endListing() { 128 if ( m_startedListing ) { endObject(); } 129 m_startedListing = false; 130 } 131 132 std::string JsonReporter::getDescription() { 133 return "Outputs listings as JSON. Test listing is Work-in-Progress!"; 134 } 135 136 void JsonReporter::testRunStarting( TestRunInfo const& testInfo ) { 137 StreamingReporterBase::testRunStarting( testInfo ); 138 endListing(); 139 140 assert( isInside( Writer::Object ) ); 141 startObject( "test-run"_sr ); 142 startArray( "test-cases"_sr ); 143 } 144 145 static void writeCounts( JsonObjectWriter&& writer, Counts const& counts ) { 146 writer.write( "passed"_sr ).write( counts.passed ); 147 writer.write( "failed"_sr ).write( counts.failed ); 148 writer.write( "fail-but-ok"_sr ).write( counts.failedButOk ); 149 writer.write( "skipped"_sr ).write( counts.skipped ); 150 } 151 152 void JsonReporter::testRunEnded(TestRunStats const& runStats) { 153 assert( isInside( Writer::Array ) ); 154 // End "test-cases" 155 endArray(); 156 157 { 158 auto totals = 159 m_objectWriters.top().write( "totals"_sr ).writeObject(); 160 writeCounts( totals.write( "assertions"_sr ).writeObject(), 161 runStats.totals.assertions ); 162 writeCounts( totals.write( "test-cases"_sr ).writeObject(), 163 runStats.totals.testCases ); 164 } 165 166 // End the "test-run" object 167 endObject(); 168 } 169 170 void JsonReporter::testCaseStarting( TestCaseInfo const& tcInfo ) { 171 StreamingReporterBase::testCaseStarting( tcInfo ); 172 173 assert( isInside( Writer::Array ) && 174 "We should be in the 'test-cases' array" ); 175 startObject(); 176 // "test-info" prelude 177 { 178 auto testInfo = 179 m_objectWriters.top().write( "test-info"_sr ).writeObject(); 180 // TODO: handle testName vs className!! 181 testInfo.write( "name"_sr ).write( tcInfo.name ); 182 writeSourceInfo(testInfo, tcInfo.lineInfo); 183 writeTags( testInfo.write( "tags"_sr ).writeArray(), tcInfo.tags ); 184 writeProperties( testInfo.write( "properties"_sr ).writeArray(), 185 tcInfo ); 186 } 187 188 189 // Start the array for individual test runs (testCasePartial pairs) 190 startArray( "runs"_sr ); 191 } 192 193 void JsonReporter::testCaseEnded( TestCaseStats const& tcStats ) { 194 StreamingReporterBase::testCaseEnded( tcStats ); 195 196 // We need to close the 'runs' array before finishing the test case 197 assert( isInside( Writer::Array ) ); 198 endArray(); 199 200 { 201 auto totals = 202 m_objectWriters.top().write( "totals"_sr ).writeObject(); 203 writeCounts( totals.write( "assertions"_sr ).writeObject(), 204 tcStats.totals.assertions ); 205 // We do not write the test case totals, because there will always be just one test case here. 206 // TODO: overall "result" -> success, skip, fail here? Or in partial result? 207 } 208 // We do not write out stderr/stdout, because we instead wrote those out in partial runs 209 210 // TODO: aborting? 211 212 // And we also close this test case's object 213 assert( isInside( Writer::Object ) ); 214 endObject(); 215 } 216 217 void JsonReporter::testCasePartialStarting( TestCaseInfo const& /*tcInfo*/, 218 uint64_t index ) { 219 startObject(); 220 m_objectWriters.top().write( "run-idx"_sr ).write( index ); 221 startArray( "path"_sr ); 222 // TODO: we want to delay most of the printing to the 'root' section 223 // TODO: childSection key name? 224 } 225 226 void JsonReporter::testCasePartialEnded( TestCaseStats const& tcStats, 227 uint64_t /*index*/ ) { 228 // Fixme: the top level section handles this. 229 //// path object 230 endArray(); 231 if ( !tcStats.stdOut.empty() ) { 232 m_objectWriters.top() 233 .write( "captured-stdout"_sr ) 234 .write( tcStats.stdOut ); 235 } 236 if ( !tcStats.stdErr.empty() ) { 237 m_objectWriters.top() 238 .write( "captured-stderr"_sr ) 239 .write( tcStats.stdErr ); 240 } 241 { 242 auto totals = 243 m_objectWriters.top().write( "totals"_sr ).writeObject(); 244 writeCounts( totals.write( "assertions"_sr ).writeObject(), 245 tcStats.totals.assertions ); 246 // We do not write the test case totals, because there will 247 // always be just one test case here. 248 // TODO: overall "result" -> success, skip, fail here? Or in 249 // partial result? 250 } 251 // TODO: aborting? 252 // run object 253 endObject(); 254 } 255 256 void JsonReporter::sectionStarting( SectionInfo const& sectionInfo ) { 257 assert( isInside( Writer::Array ) && 258 "Section should always start inside an object" ); 259 // We want to nest top level sections, even though it shares name 260 // and source loc with the TEST_CASE 261 auto& sectionObject = startObject(); 262 sectionObject.write( "kind"_sr ).write( "section"_sr ); 263 sectionObject.write( "name"_sr ).write( sectionInfo.name ); 264 writeSourceInfo( m_objectWriters.top(), sectionInfo.lineInfo ); 265 266 267 // TBD: Do we want to create this event lazily? It would become 268 // rather complex, but we could do it, and it would look 269 // better for empty sections. OTOH, empty sections should 270 // be rare. 271 startArray( "path"_sr ); 272 } 273 void JsonReporter::sectionEnded( SectionStats const& /*sectionStats */) { 274 // End the subpath array 275 endArray(); 276 // TODO: metadata 277 // TODO: what info do we have here? 278 279 // End the section object 280 endObject(); 281 } 282 283 void JsonReporter::assertionStarting( AssertionInfo const& /*assertionInfo*/ ) {} 284 void JsonReporter::assertionEnded( AssertionStats const& assertionStats ) { 285 // TODO: There is lot of different things to handle here, but 286 // we can fill it in later, after we show that the basic 287 // outline and streaming reporter impl works well enough. 288 //if ( !m_config->includeSuccessfulResults() 289 // && assertionStats.assertionResult.isOk() ) { 290 // return; 291 //} 292 assert( isInside( Writer::Array ) ); 293 auto assertionObject = m_arrayWriters.top().writeObject(); 294 295 assertionObject.write( "kind"_sr ).write( "assertion"_sr ); 296 writeSourceInfo( assertionObject, 297 assertionStats.assertionResult.getSourceInfo() ); 298 assertionObject.write( "status"_sr ) 299 .write( assertionStats.assertionResult.isOk() ); 300 // TODO: handling of result. 301 // TODO: messages 302 // TODO: totals? 303 } 304 305 306 void JsonReporter::benchmarkPreparing( StringRef name ) { (void)name; } 307 void JsonReporter::benchmarkStarting( BenchmarkInfo const& ) {} 308 void JsonReporter::benchmarkEnded( BenchmarkStats<> const& ) {} 309 void JsonReporter::benchmarkFailed( StringRef error ) { (void)error; } 310 311 void JsonReporter::listReporters( 312 std::vector<ReporterDescription> const& descriptions ) { 313 startListing(); 314 315 auto writer = 316 m_objectWriters.top().write( "reporters"_sr ).writeArray(); 317 for ( auto const& desc : descriptions ) { 318 auto desc_writer = writer.writeObject(); 319 desc_writer.write( "name"_sr ).write( desc.name ); 320 desc_writer.write( "description"_sr ).write( desc.description ); 321 } 322 } 323 void JsonReporter::listListeners( 324 std::vector<ListenerDescription> const& descriptions ) { 325 startListing(); 326 327 auto writer = 328 m_objectWriters.top().write( "listeners"_sr ).writeArray(); 329 330 for ( auto const& desc : descriptions ) { 331 auto desc_writer = writer.writeObject(); 332 desc_writer.write( "name"_sr ).write( desc.name ); 333 desc_writer.write( "description"_sr ).write( desc.description ); 334 } 335 } 336 void JsonReporter::listTests( std::vector<TestCaseHandle> const& tests ) { 337 startListing(); 338 339 auto writer = m_objectWriters.top().write( "tests"_sr ).writeArray(); 340 341 for ( auto const& test : tests ) { 342 auto desc_writer = writer.writeObject(); 343 auto const& info = test.getTestCaseInfo(); 344 345 desc_writer.write( "name"_sr ).write( info.name ); 346 desc_writer.write( "class-name"_sr ).write( info.className ); 347 { 348 auto tag_writer = desc_writer.write( "tags"_sr ).writeArray(); 349 for ( auto const& tag : info.tags ) { 350 tag_writer.write( tag.original ); 351 } 352 } 353 writeSourceInfo( desc_writer, info.lineInfo ); 354 } 355 } 356 void JsonReporter::listTags( std::vector<TagInfo> const& tags ) { 357 startListing(); 358 359 auto writer = m_objectWriters.top().write( "tags"_sr ).writeArray(); 360 for ( auto const& tag : tags ) { 361 auto tag_writer = writer.writeObject(); 362 { 363 auto aliases_writer = 364 tag_writer.write( "aliases"_sr ).writeArray(); 365 for ( auto alias : tag.spellings ) { 366 aliases_writer.write( alias ); 367 } 368 } 369 tag_writer.write( "count"_sr ).write( tag.count ); 370 } 371 } 372 } // namespace Catch