001package org.w3.ldp.testsuite.reporter; 002 003import static org.rendersnake.HtmlAttributesFactory.NO_ESCAPE; 004import static org.rendersnake.HtmlAttributesFactory.class_; 005import static org.rendersnake.HtmlAttributesFactory.colspan; 006import static org.rendersnake.HtmlAttributesFactory.href; 007import static org.rendersnake.HtmlAttributesFactory.id; 008import static org.rendersnake.HtmlAttributesFactory.rowspan; 009import static org.rendersnake.HtmlAttributesFactory.style; 010 011import java.io.File; 012import java.io.FileWriter; 013import java.io.IOException; 014import java.io.StringWriter; 015import java.lang.reflect.Method; 016import java.text.DateFormat; 017import java.util.ArrayList; 018import java.util.Arrays; 019import java.util.Date; 020import java.util.HashMap; 021import java.util.HashSet; 022import java.util.Iterator; 023import java.util.Locale; 024import java.util.Map.Entry; 025import java.util.Set; 026 027import org.rendersnake.HtmlCanvas; 028import org.rendersnake.StringResource; 029import org.rendersnake.tools.PrettyWriter; 030import org.testng.annotations.Test; 031import org.w3.ldp.testsuite.BuildProperties; 032import org.w3.ldp.testsuite.LdpTestSuite; 033import org.w3.ldp.testsuite.annotations.SpecTest; 034import org.w3.ldp.testsuite.annotations.SpecTest.METHOD; 035import org.w3.ldp.testsuite.annotations.SpecTest.STATUS; 036import org.w3.ldp.testsuite.test.BasicContainerTest; 037import org.w3.ldp.testsuite.test.CommonContainerTest; 038import org.w3.ldp.testsuite.test.CommonResourceTest; 039import org.w3.ldp.testsuite.test.DirectContainerTest; 040import org.w3.ldp.testsuite.test.IndirectContainerTest; 041import org.w3.ldp.testsuite.test.NonRDFSourceTest; 042import org.w3.ldp.testsuite.test.RdfSourceTest; 043 044public class LdpTestCaseReporter { 045 046 private static HtmlCanvas html; 047 048 private static StringWriter graphs = new StringWriter(); 049 050 private static Set<String> refURI = new HashSet<String>(); 051 052 private static ArrayList<String> clients = new ArrayList<String>(); 053 private static ArrayList<String> manuals = new ArrayList<String>(); 054 private static HashMap<String, String> readyToBeApproved = new HashMap<String, String>(); // key==method, value==class 055 private static HashMap<String, String> needCode = new HashMap<String, String>(); // key==method, value==class 056 057 private static final HashMap<String, String> implmColor = new HashMap<String, String>(); // key==label value==color 058 private static final HashMap<String, String> statusColor = new HashMap<String, String>(); 059 static{ 060 implmColor.put("automated", "#0099cc"); 061 implmColor.put("unimplemented", "#bf1c56"); 062 implmColor.put("client", "#8d1cbf"); 063 implmColor.put("manual", "#3300cc"); 064 implmColor.put("indirect", "#ffcc66"); 065 statusColor.put("approved", "#a2bf2f"); 066 statusColor.put("pending", "#1cbfbb"); 067 statusColor.put("extends", "#bfa22f"); 068 statusColor.put("deprecated", "#606060"); 069 statusColor.put("clarify", "#1bff95"); 070 } 071 public static final int MUST = 0; 072 public static final int SHOULD = 1; 073 public static final int MAY = 2; 074 public static final int OTHER = 3; 075 076 private static int totalTests = 0; 077 private static int automated = 0; 078 079 private static int mustTotal = 0; 080 private static int shouldTotal = 0; 081 private static int mayTotal = 0; 082 083 private static int[] auto = {0, 0, 0}; 084 private static int[] unimplmnt = {0, 0, 0}; 085 086 private static int pending = 0; 087 private static int approved = 0; 088 private static int extended = 0; 089 private static int deprecated = 0; 090 private static int clarification = 0; 091 092 private static Class<BasicContainerTest> bcTest = BasicContainerTest.class; 093 private static Class<RdfSourceTest> rdfSourceTest = RdfSourceTest.class; 094 private static Class<IndirectContainerTest> indirectContainerTest = IndirectContainerTest.class; 095 private static Class<DirectContainerTest> directContianerTest = DirectContainerTest.class; 096 private static Class<CommonContainerTest> commonContainerTest = CommonContainerTest.class; 097 private static Class<CommonResourceTest> commonResourceTest = CommonResourceTest.class; 098 private static Class<NonRDFSourceTest> nonRdfSourceTest = NonRDFSourceTest.class; 099 100 private static ArrayList<Method> indirectCases = new ArrayList<Method>(); 101 102 @SuppressWarnings("rawtypes") 103 private static Class[] defaultTestClasses = { 104 rdfSourceTest, 105 bcTest, 106 commonContainerTest, 107 commonResourceTest, 108 nonRdfSourceTest, 109 indirectContainerTest, 110 directContianerTest}; 111 112 private static int[] extnd = {0, 0, 0}; 113 private static int[] deprctd = {0, 0, 0}; 114 private static int[] pend = {0, 0, 0}; 115 private static int[] manual = {0, 0, 0}; 116 private static int[] client = {0, 0, 0}; 117 private static int[] approve = {0, 0, 0}; 118 private static int[] clarify = {0, 0, 0}; 119 private static int[] indirect = {0, 0, 0}; 120 121 @SuppressWarnings("rawtypes") 122 protected Class[] testClasses; 123 124 private String title = "LDP"; 125 private String specUri = LdpTestSuite.SPEC_URI; 126 127 public LdpTestCaseReporter() { 128 this.testClasses = defaultTestClasses; 129 } 130 131 @SuppressWarnings("rawtypes") 132 public LdpTestCaseReporter(Class[] inTestClasses, String title, String uri) { 133 this.testClasses = inTestClasses; 134 this.title = title; 135 this.specUri = uri; 136 } 137 138 public static void main(String[] args) throws IOException, SecurityException { 139 LdpTestCaseReporter reporter = new LdpTestCaseReporter(); 140 reporter.generateReport("ldp-testsuite"); 141 } 142 143 public void generateReport(String title) throws IOException, SecurityException { 144 System.out.println("Executing coverage report..."); 145 this.initializeTestClasses(); 146 this.generateHTMLReport(); 147 amendReport(); 148 writeReport(LdpTestSuite.OUTPUT_DIR, title, html.toHtml()); 149 System.out.println("Done!"); 150 } 151 152 @SuppressWarnings("unchecked") 153 protected void generateHTMLReport() throws IOException, SecurityException { 154 html = new HtmlCanvas(); 155 html.html().head(); 156 writeCss(); 157 html.title().content(title + ": Test Cases Coverage Report")._head().body(); 158 159 html.h1().content("W3C Linked Data Platform (" + title + ") Test Suite: Test Cases Coverage Report"); 160 html.p().a(href("http://www.w3.org/2012/ldp/")).content("See also W3C Linked Data Platform WG")._p(); 161 html.p().a(href(specUri)).content("Specification Requirements Page")._p(); 162 163 final String commit = BuildProperties.getRevision(); 164 if (commit != null) { 165 html.div() 166 .write("Test Suite Revision: ") 167 .a(href("https://github.com/w3c/ldp-testsuite/commit/" + commit)).content(commit)._div(); 168 } 169 DateFormat dateFormat = DateFormat.getDateInstance(DateFormat.MEDIUM, Locale.ENGLISH); 170 html.div().content("Updated: " + dateFormat.format(new Date())); 171 172 // Generate the summary diagrams and information 173 createSummaryReport(); 174 toTop(); 175 176 // Go through all the classes and generate details of each test case 177 for (@SuppressWarnings("rawtypes") Class testcaseClass: testClasses) { 178 writeTestCasesForClass(testcaseClass); 179 } 180 } 181 182 private static void amendReport() throws IOException { 183 html.script().content(StringResource.get("/raphael/raphael-min.js"), NO_ESCAPE); 184 html.script().content(StringResource.get("/prototype/prototype.js"), NO_ESCAPE); 185 html.script().content(StringResource.get("/grafico/grafico-min.js"), NO_ESCAPE); 186 html.write(graphs.toString(), NO_ESCAPE); 187 188 html._body()._html(); 189 } 190 191 protected void createSummaryReport() throws IOException { 192 193 html.h2().content("Summary of Test Methods"); 194 html.table(class_("summary")); 195 html.tr().th().content("Totals"); 196 html.th().content("Coverage"); 197 html.th().content("Not Implemented"); 198 html._tr(); 199 200 html.tr(); 201 html.td(rowspan("2")).b().write("" + totalTests)._b().write(" Total Tests"); 202 html.ul(); 203 204 html.li(); 205 writeColorBlock(statusColor, "approved"); 206 html.b().write(approved + " ")._b().write("Approved"); 207 html._li(); 208 209 html.li(); 210 writeColorBlock(statusColor, "pending"); 211 html.b().write(pending + " ")._b().write("Pending"); 212 html._li(); 213 214 html.li(); 215 writeColorBlock(statusColor, "extends"); 216 html.b().write(extended + " ")._b().write("Extension"); 217 html._li(); 218 219 html.li(); 220 writeColorBlock(statusColor, "deprecated"); 221 html.b().write(deprecated + " ")._b().write("No longer valid"); 222 html._li(); 223 224 html.li(); 225 writeColorBlock(statusColor, "clarify"); 226 html.b().write(clarification + " ")._b().write("Needs to be clarified with WG"); 227 html._li(); 228 229 html._ul(); 230 231 html.br().b().a(href("#tobeapproved")).write(readyToBeApproved.size() 232 + " Ready for Approval")._a()._b(); 233 234 html.br(); 235 html.span(class_("chartStart")); 236 // html.label(class_("label")).b().write("Test Case Implementation for Totals")._b()._label(); 237 html.div(class_("barChart").id("overall_statusbar"))._div(); 238 generateStatusGraphJSON("overall", approve, pend, extnd, deprctd, clarify); 239 html._span(); 240 html._td(); 241 242 html.td(); 243 244 html.b().write(automated + " / " + totalTests)._b() 245 .write(" of Total Tests Automated").br(); 246 html.b().write(automated + " / " + (automated + needCode.size()))._b() 247 .write(" of Tests Possible to Automate"); 248 html.ul(); 249 html.li().b().write("" + refURI.size())._b().write(" Requirements Covered") 250 ._li(); 251 html.ul().li().b().write("" + mustTotal)._b().write(" MUST")._li(); 252 html.li().b().write("" + shouldTotal)._b().write(" SHOULD")._li(); 253 html.li().b().write("" + mayTotal)._b().write(" MAY")._li()._ul(); 254 html._ul(); 255 256 html.ul(); 257 int implemented = getTotal(auto); 258 html.li(); 259 writeColorBlock(implmColor, "automated"); 260 html.b().write(implemented + " ")._b().write("Requirements Automated"); 261 html._li(); 262 263 html.ul(); 264 html.li().b().write(auto[MUST] + " / " + mustTotal)._b().write(" MUST")._li(); 265 html.li().b().write(auto[SHOULD] + " / " + shouldTotal)._b().write(" SHOULD") 266 ._li(); 267 html.li().b().write(auto[MAY] + " / " + mayTotal)._b().write(" MAY")._li(); 268 html._ul(); 269 html._ul(); 270 271 writeColorBlock(implmColor, "indirect"); 272 html.b().write(indirectCases.size())._b().write(" Tests ").a(href(("#indirectCases"))).write("Indirectly Covered")._a(); 273 274 html._td(); 275 276 // html.td().content(totalImplemented + " Tests"); 277 html.td(); 278 html.b().write((needCode.size() + clients.size() + manuals.size()) + " ")._b() 279 .write("of the Unimplemented Tests"); 280 html.ul(); 281 282 html.li().b().write(needCode.size() + " ")._b().write("of the Tests ") 283 .a(href("#needCode")).write("Yet to be Coded")._a()._li(); 284 /* 285 * TODO: Determine if disabled is really valuable or not 286 * html.li().b().write(disabled + 287 * " ")._b().write("of the Tests not enabled")._li(); 288 */ 289 html.li(); 290 writeColorBlock(implmColor, "client"); 291 html.b().write(clients.size() + " ")._b().write("of the Total are ") 292 .a(href("#clientTests")).write("Client-Based Tests")._a(); 293 html._li(); 294 295 html.li(); 296 writeColorBlock(implmColor, "manual"); 297 html.b().write(manuals.size() + " ")._b().write("of the Total must be ") 298 .a(href("#manualTests")).write("Tested Manually")._a(); 299 html._li(); 300 301 html._ul(); 302 303 html.write("From the Total, "); 304 305 html.ul(); 306 int unimplemented = getTotal(unimplmnt); 307 308 html.li(); 309 writeColorBlock(implmColor, "client"); 310 html.b().write(clients.size() + " ")._b().write("of the Total are ") 311 .a(href("#clientTests")).write("Client-Based Tests")._a(); 312 html._li(); 313 314 html.li(); 315 writeColorBlock(implmColor, "unimplemented"); 316 html.b().write(unimplemented + " ")._b().write("Requirements not Implemented"); 317 html._li(); 318 319 html.ul(); 320 html.li().b().write(unimplmnt[MUST] + " ")._b().write("MUST")._li(); 321 html.li().b().write(unimplmnt[SHOULD] + " ")._b().write("SHOULD")._li(); 322 html.li().b().write(unimplmnt[MAY] + " ")._b().write("MAY")._li(); 323 html._ul(); 324 html._ul()._td(); 325 html._tr(); 326 327 html.tr().td(colspan("2").style("text-align:center;")); 328 html.span(class_("chartStart")); 329 // html.label(class_("label")).b().write("Test Case Status for Coverage")._b()._label(); 330 html.div(class_("barChart").id("overall_implmtbar"))._div(); 331 writeImplementationGraph("overall", auto, unimplmnt, client, manual, indirect); 332 html._span(); 333 html._td()._tr(); 334 html._table(); 335 336 writeGraphDescription(); 337 writeSummary(); 338 339 } 340 341 private static void writeColorBlock(HashMap<String, String> list, String string) throws IOException { 342 // TODO: Create a class for this style in our stylesheet. 343 html.div(style("background-color:" + list.get(string) + ";").class_("color-block"))._div(); 344 } 345 346 private static int getTotal(int[] array) { 347 int total = 0; 348 for(int i = 0; i < array.length; i++) 349 total += array[i]; 350 return total; 351 } 352 353 @SuppressWarnings({ "rawtypes", "unchecked" }) 354 protected void initializeTestClasses() throws IOException { 355 for (Class testcaseClass: testClasses) { 356 computeTestCasesStats(testcaseClass); 357 } 358 } 359 360 @SuppressWarnings("unchecked") 361 protected void writeSummary() throws IOException { 362 html.h2(id("tobeapproved")).content("Test Cases Ready for Approval"); 363 364 if (readyToBeApproved.size() == 0) { 365 html.b().write("No test cases awaiting approval.")._b().br(); 366 } else { 367 html.p().write("For details of test cases, ") 368 .a(href("https://github.com/w3c/ldp-testsuite")) 369 .write("see source in GitHub")._a()._p(); 370 html.ul(); 371 iterateTableList(readyToBeApproved); 372 html._ul(); 373 } 374 toTop(); 375 376 html.h2(id("needCode")).content("Test Cases Yet to be Implemented"); 377 if (needCode.size() == 0) 378 html.b().write("No test cases that need to be Implemented")._b() 379 .br(); 380 else { 381 html.ul(); 382 iterateTableList(needCode); 383 html._ul(); 384 } 385 toTop(); 386 387 html.h2().a(id("indirectCases")).content("Test Cases Covered Indirectly")._h2(); 388 writeIndirectCases(); 389 390 html.h2().content("Implemented Test Classes"); 391 392 for (@SuppressWarnings("rawtypes") Class testcaseClass: testClasses) { 393 writeTestTables(testcaseClass); 394 } 395 396 toTop(); 397 398 html.h2().a(id("manualTests")) 399 .content("Tests that Must be Tested Manually")._h2(); 400 generateList(manuals); 401 html.h2().a(id("clientTests")).content("Client-Based Test Cases")._h2(); 402 generateList(clients); 403 404 } 405 406 private void writeIndirectCases() throws IOException { 407 if(indirectCases.size() == 0){ 408 html.p().b().write("No Indirect Test Cases.")._b()._p(); 409 } else { 410 html.ul(); 411 for (Method method : indirectCases) { 412 if (method.getAnnotation(SpecTest.class) != null) { 413 String name = method.getDeclaringClass().getCanonicalName(); 414 name = name.substring(name.lastIndexOf(".") + 1); 415 String normalizedName = AbstractEarlReporter.createTestCaseName(name, method.getName()); 416 html.li().a(href("#" + method.getName())).b().write(normalizedName)._b()._a()._li(); 417 } 418 } 419 html._ul(); 420 } 421 422 } 423 424 private static <T> void writeTestTables(Class<T> testClass) throws IOException { 425 html.b().a(href("#" + testClass.getCanonicalName())) 426 .content(testClass.getCanonicalName())._b(); 427 html.br(); 428 writeTestClassTable(testClass); 429 html.br(); 430 } 431 432 private static void iterateTableList(HashMap<String, String> list) throws IOException { 433 String className; 434 String methodName; 435 Iterator<Entry<String, String>> codeIter = list.entrySet().iterator(); 436 while(codeIter.hasNext()){ 437 Entry<String, String> entry = codeIter.next(); 438 className = entry.getValue(); 439 className = className.substring(className.lastIndexOf(".") + 1); 440 methodName = entry.getKey(); 441 String normalizedName = AbstractEarlReporter.createTestCaseName(className, methodName); 442 html.li().a(href("#" + methodName)).b().write(normalizedName)._b()._a()._li(); 443 } 444 } 445 446 private static <T> void writeTestClassTable(Class<T> classType) 447 throws IOException { 448 html.span(class_("chartStart")); 449 html.label(class_("label")).b().write("Test Case Status")._b()._label(); 450 html.div(class_("smallBar").id(classType.getSimpleName() + "_statusbar"))._div(); 451 writeStatusLegend(); 452 453 html.span(class_("chartStart")); 454 html.label(class_("label")).b().write("Test Case Implementation")._b()._label(); 455 html.div(class_("smallBar").id(classType.getSimpleName() + "_implmtbar"))._div(); 456 writeImplmntLegend(); 457 458 acquireTestInfo(classType); 459 460 } 461 462 private static <T> void acquireTestInfo(Class<T> classType) 463 throws IOException { 464 int total = 0, must = 0, should = 0, may = 0; 465 466 int[] autoReq = { 0, 0, 0 }, unimReq = { 0, 0, 0 }, clientReq = { 0, 0, 0 }, manReq = { 0, 0, 0 }, indirect = { 0, 0, 0 }; 467 int[] apprReq = { 0, 0, 0 }, pendReq = { 0, 0, 0 }, extReq = { 0, 0, 0 }, 468 depreReq = {0, 0, 0 }, clariReq = { 0, 0, 0 }; 469 470 Method[] methods = classType.getDeclaredMethods(); 471 472 for (Method method : methods) { 473 if (method.isAnnotationPresent(SpecTest.class) 474 && method.isAnnotationPresent(Test.class)) { 475 total++; 476 SpecTest testLdp = method.getAnnotation(SpecTest.class); 477 Test test = method.getAnnotation(Test.class); 478 String group = Arrays.toString(test.groups()); 479 if (group.contains("MUST")) 480 must++; 481 if (group.contains("SHOULD")) 482 should++; 483 if (group.contains("MAY")) 484 may++; 485 switch (testLdp.testMethod()) { 486 case AUTOMATED: 487 if (group.contains("MUST")) 488 ++autoReq[MUST]; 489 if (group.contains("SHOULD")) 490 ++autoReq[SHOULD]; 491 if (group.contains("MAY")) 492 ++autoReq[MAY]; 493 break; 494 case NOT_IMPLEMENTED: 495 if (group.contains("MUST")) 496 ++unimReq[MUST]; 497 if (group.contains("SHOULD")) 498 ++unimReq[SHOULD]; 499 if (group.contains("MAY")) 500 ++unimReq[MAY]; 501 break; 502 case MANUAL: 503 if (group.contains("MUST")) 504 ++manReq[MUST]; 505 if (group.contains("SHOULD")) 506 ++manReq[SHOULD]; 507 if (group.contains("MAY")) 508 ++manReq[MAY]; 509 break; 510 case CLIENT_ONLY: 511 if (group.contains("MUST")) 512 ++clientReq[MUST]; 513 if (group.contains("SHOULD")) 514 ++clientReq[SHOULD]; 515 if (group.contains("MAY")) 516 ++clientReq[MAY]; 517 break; 518 case INDIRECT: 519 if (group.contains("MUST")) 520 ++indirect[MUST]; 521 if (group.contains("SHOULD")) 522 ++indirect[SHOULD]; 523 if (group.contains("MAY")) 524 ++indirect[MAY]; 525 break; 526 } 527 switch (testLdp.approval()) { 528 case WG_PENDING: 529 if (group.contains("MUST")) 530 ++pendReq[MUST]; 531 if (group.contains("SHOULD")) 532 ++pendReq[SHOULD]; 533 if (group.contains("MAY")) 534 ++pendReq[MAY]; 535 break; 536 case WG_APPROVED: 537 if (group.contains("MUST")) 538 ++apprReq[MUST]; 539 if (group.contains("SHOULD")) 540 ++apprReq[SHOULD]; 541 if (group.contains("MAY")) 542 ++apprReq[MAY]; 543 break; 544 case WG_EXTENSION: 545 if (group.contains("MUST")) 546 ++extReq[MUST]; 547 if (group.contains("SHOULD")) 548 ++extReq[SHOULD]; 549 if (group.contains("MAY")) 550 ++extReq[MAY]; 551 break; 552 case WG_DEPRECATED: 553 if (group.contains("MUST")) 554 ++depreReq[MUST]; 555 if (group.contains("SHOULD")) 556 ++depreReq[SHOULD]; 557 if (group.contains("MAY")) 558 ++depreReq[MAY]; 559 break; 560 case WG_CLARIFICATION: 561 if (group.contains("MUST")) 562 ++clariReq[MUST]; 563 if (group.contains("SHOULD")) 564 ++clariReq[SHOULD]; 565 if (group.contains("MAY")) 566 ++clariReq[MAY]; 567 break; 568 } 569 } 570 } 571 572 // write information into Grafico bar charts 573 generateStatusGraphJSON(classType.getSimpleName(), apprReq, pendReq, extReq, depreReq, clariReq); 574 writeImplementationGraph(classType.getSimpleName(), autoReq, unimReq, clientReq, manReq, indirect); 575 576 html.table(class_("classes")); 577 578 html.tr().th().content("Total Tests"); 579 html.th().content("Coverage"); 580 html.th().content("Status"); 581 html.th().content("Implementation"); 582 html._tr(); 583 584 html.tr().td().content(total + ""); 585 html.td(); 586 if (must > 0) 587 html.b().write("MUST: ")._b().write(must).br(); 588 if (should > 0) 589 html.b().write("SHOULD: ")._b().write(should).br(); 590 if (may > 0) 591 html.b().write("MAY: ")._b().write(may).br(); 592 593 html._td().td(); 594 int approve = getTotal(apprReq); 595 if(approve > 0) 596 html.b().write("Approved: ")._b().write("" + approve).br(); 597 int pend = getTotal(pendReq); 598 if(pend > 0) 599 html.b().write("Pending: ")._b().write("" + pend).br(); 600 int extend = getTotal(extReq); 601 if (extend > 0) 602 html.b().write("Extension: ")._b().write("" + extend).br(); 603 int depre = getTotal(depreReq); 604 if (depre > 0) 605 html.b().write("Deprecated: ")._b().write("" + depre).br(); 606 int clarify = getTotal(clariReq); 607 if(clarify > 0) 608 html.b().write("Clarification: ")._b().write("" + clarify).br(); 609 610 html._td().td(); 611 int auto = getTotal(autoReq); 612 if (auto > 0) 613 html.b().write("Automated: ")._b().write("" + auto).br(); 614 int unimpl = getTotal(unimReq); 615 if (unimpl > 0) 616 html.b().write("Not Implemented: ")._b().write("" + unimpl).br(); 617 int client = getTotal(clientReq); 618 if (client > 0) 619 html.b().write("Client Only: ")._b().write("" + client).br(); 620 int manual = getTotal(manReq); 621 if (manual > 0) 622 html.b().write("Manual: ")._b().write("" + manual).br(); 623 624 html._td(); 625 626 html._tr(); 627 html._table(); 628 629 } 630 631 private static void generateStatusGraphJSON(String type, int[] apprReq, int[] pendReq, 632 int[] extReq, int[] depreReq, int[] clariReq) 633 throws IOException { 634 graphs.write("<script>"); 635 graphs.write("Event.observe(window, 'load', function() {"); 636 637 graphs.write("var " + type + "_statusbar" 638 + " = new Grafico.StackedBarGraph($('" + type 639 + "_statusbar'),{"); 640 graphs.write("approved: [" + apprReq[MUST] + ", " + apprReq[SHOULD] + ", " + apprReq[MAY] + " ],"); 641 graphs.write("pending: [" + pendReq[MUST] + ", " + pendReq[SHOULD] + ", " + pendReq[MAY] + " ],"); 642 graphs.write("extends: [" + extReq[MUST] + ", " + extReq[SHOULD] + ", " + extReq[MAY] + " ],"); 643 graphs.write("deprecated: [" + depreReq[MUST] + ", " + depreReq[SHOULD] + ", " + depreReq[MAY] + " ],"); 644 graphs.write("clarify: [" + clariReq[MUST] + ", " + clariReq[SHOULD] + ", " + clariReq[MAY] + "] "); 645 graphs.write("},"); 646 graphs.write("{ labels: [ \"MUST\", \"SHOULD\", \"MAY\" ],"); 647 graphs.write("colors: { "); 648 writeColors(statusColor); 649 graphs.write(" },"); 650 graphs.write("hover_color: \"#ccccff\","); 651 graphs.write("datalabels: { "); 652 graphs.write("approved: [ \"" + apprReq[MUST] + " Approved\", \"" + apprReq[SHOULD] + " Approved\", \"" + apprReq[MAY] + " Approved\" ],"); 653 graphs.write("pending: [ \"" + pendReq[MUST] + " Pending\", \"" + pendReq[SHOULD] + " Pending\", \"" + pendReq[MAY] + " Pending\" ],"); 654 graphs.write("extends: [ \"" + extReq[MUST] + " Extension\", \"" + extReq[SHOULD] + " Extension\", \"" + extReq[MAY] + " Extension\" ],"); 655 graphs.write("deprecated: [ \"" + depreReq[MUST] + " Deprecated\", \"" + depreReq[SHOULD] + " Deprecated\", \"" + depreReq[MAY] + " Deprecated\" ],"); 656 graphs.write("clarify: [ \"" + clariReq[MUST] + " Clarification\", \"" + clariReq[SHOULD] + " Clarification\", \"" + clariReq[MAY] + " Clarification\" ] }"); 657 658 graphs.write(" }); });"); 659 660 graphs.write("</script>"); 661 662 663 } 664 665 private static void writeImplementationGraph(String classType, 666 int[] autoReq, int[] unimReq, int[] clientReq, int[] manReq, int[] indirect) { 667 graphs.write("<script>"); 668 graphs.write("Event.observe(window, 'load', function() {"); 669 670 graphs.write("var " + classType + "_implmtbar" 671 + " = new Grafico.StackedBarGraph($('" + classType 672 + "_implmtbar'), {"); 673 graphs.write("automated: [" + autoReq[MUST] + ", " + autoReq[SHOULD] + ", " + autoReq[MAY] + " ],"); 674 graphs.write("unimplemented: [" + unimReq[MUST] + ", " + unimReq[SHOULD] + ", " + unimReq[MAY] + " ],"); 675 graphs.write("client: [" + clientReq[MUST] + ", " + clientReq[SHOULD] + ", " + clientReq[MAY] + " ],"); 676 graphs.write("manual: [" + manReq[MUST] + ", " + manReq[SHOULD] + ", " + manReq[MAY] + "],"); 677 graphs.write("indirect: [" + indirect[MUST] + ", " + indirect[SHOULD] + ", " + indirect[MAY] + " ]"); 678 graphs.write("},"); 679 graphs.write("{ labels: [ \"MUST\", \"SHOULD\", \"MAY\" ],"); 680 graphs.write("colors: { "); 681 writeColors(implmColor); 682 graphs.write("},"); 683 graphs.write("hover_color: \"#ccccff\","); 684 graphs.write("datalabels: { "); 685 graphs.write("automated: [ \"" + autoReq[MUST] + " Automated\", \"" + autoReq[SHOULD] + " Automated\", \"" + autoReq[MAY] + " Automated\" ],"); 686 graphs.write("unimplemented: [ \"" + unimReq[MUST] + " Not Implemented\", \"" + unimReq[SHOULD] + " Not Implemented\", \"" + unimReq[MAY] + " Not Implemented\" ],"); 687 graphs.write("client: [ \"" + clientReq[MUST] + " Client Only\", \"" + clientReq[SHOULD] + " Client Only\", \"" + clientReq[MAY] + " Client Only\" ],"); 688 graphs.write("manual: [ \"" + manReq[MUST] + " Manual\", \"" + manReq[SHOULD] + " Manual\", \"" + manReq[MAY] + " Manual\" ], "); 689 graphs.write("indirect: [ \"" + indirect[MUST] + " Indirect\", \"" + indirect[SHOULD] + " Indirect\", \"" + indirect[MAY] + " Indirect\" ]"); 690 graphs.write("},"); 691 692 graphs.write(" }); });"); 693 694 graphs.write("</script>"); 695 } 696 697 private static void writeColors(HashMap<String, String> colorList) { 698 Iterator<Entry<String, String>> codeIter = colorList.entrySet().iterator(); 699 Entry<String, String> value; 700 while(codeIter.hasNext()){ 701 value = codeIter.next(); 702 graphs.write(value.getKey() + ": '" + value.getValue() + "' "); 703 if(codeIter.hasNext()) 704 graphs.write(", "); 705 } 706 } 707 708 private static void generateList(ArrayList<String> list) throws IOException { 709 if(list.size() == 0) { 710 html.p().content("No tests of this type found."); 711 } else { 712 html.ul(); 713 for (int i = 0; i < list.size(); i++) { 714 html.li().a(href("#" + list.get(i))).write(list.get(i))._a()._li(); 715 } 716 html._ul(); 717 } 718 } 719 720 private static <T> void computeTestCasesStats(Class<T> classType) 721 throws IOException { 722 String className = classType.getCanonicalName(); 723 724 for (Method method : classType.getDeclaredMethods()) { 725 if (method.isAnnotationPresent(Test.class)) { 726 generateTestCaseDetails(method, className); 727 } 728 } 729 } 730 731 private <T> void writeTestCasesForClass(Class<T> classType) 732 throws IOException, SecurityException { 733 String className = classType.getCanonicalName(); 734 html.h2().a(id(className)).write("Test Class: " + className)._a()._h2(); 735 736 html.ul(); 737 for (Method method : classType.getDeclaredMethods()) { 738 if (method.isAnnotationPresent(Test.class)) { 739 writeTestCaseDetails(method, className); 740 } 741 } 742 html._ul(); 743 toTop(); 744 } 745 746 private static void generateTestCaseDetails(Method method, String className) 747 throws IOException { 748 SpecTest testLdp = method.getAnnotation(SpecTest.class); 749 Test test = method.getAnnotation(Test.class); 750 if (testLdp == null || test == null) { 751 return; 752 } 753 754 totalTests++; 755 METHOD methodStatus = testLdp.testMethod(); 756 STATUS testApproval = testLdp.approval(); 757 String group = Arrays.toString(test.groups()); 758 if (!refURI.contains(testLdp.specRefUri())) { // for just the requirement testing 759 refURI.add(testLdp.specRefUri()); 760 if (group.contains("MUST")) { 761 mustTotal++; 762 switch (methodStatus) { 763 case AUTOMATED: 764 automated++; 765 ++auto[MUST]; 766 if (testApproval.equals(STATUS.WG_PENDING)) 767 readyToBeApproved.put(method.getName(), method.getDeclaringClass().getCanonicalName()); 768 break; 769 case CLIENT_ONLY: 770 clients.add(method.getName()); 771 ++client[MUST]; 772 break; 773 case MANUAL: 774 manuals.add(method.getName()); 775 ++manual[MUST]; 776 break; 777 case NOT_IMPLEMENTED: 778 ++unimplmnt[MUST]; 779 needCode.put(method.getName(), method.getDeclaringClass().getCanonicalName()); 780 break; 781 case INDIRECT: 782 indirectCases.add(method); 783 ++indirect[MUST]; 784 break; 785 } 786 switch (testApproval) { 787 case WG_APPROVED: 788 ++approve[MUST]; 789 approved++; 790 break; 791 case WG_CLARIFICATION: 792 clarification++; 793 ++clarify[MUST]; 794 break; 795 case WG_DEPRECATED: 796 deprecated++;; 797 ++deprctd[MUST]; 798 break; 799 case WG_EXTENSION: 800 extended++; 801 ++extnd[MUST]; 802 break; 803 case WG_PENDING: 804 ++pend[MUST]; 805 pending++; 806 default: 807 break; 808 } 809 } 810 if (group.contains("SHOULD")) { 811 shouldTotal++; 812 switch (methodStatus) { 813 case AUTOMATED: 814 automated++; 815 ++auto[SHOULD]; 816 if (testApproval.equals(STATUS.WG_PENDING)) 817 readyToBeApproved.put(method.getName(), method.getDeclaringClass().getCanonicalName()); 818 break; 819 case CLIENT_ONLY: 820 clients.add(method.getName()); 821 ++client[SHOULD]; 822 break; 823 case MANUAL: 824 manuals.add(method.getName()); 825 ++manual[SHOULD]; 826 break; 827 case NOT_IMPLEMENTED: 828 ++unimplmnt[SHOULD]; 829 needCode.put(method.getName(), method.getDeclaringClass().getCanonicalName()); 830 break; 831 case INDIRECT: 832 indirectCases.add(method); 833 ++indirect[SHOULD]; 834 break; 835 } 836 switch (testApproval) { 837 case WG_APPROVED: 838 ++approve[SHOULD]; 839 approved++; 840 break; 841 case WG_CLARIFICATION: 842 clarification++; 843 ++clarify[SHOULD]; 844 break; 845 case WG_DEPRECATED: 846 deprecated++; 847 ++deprctd[SHOULD]; 848 break; 849 case WG_EXTENSION: 850 extended++; 851 ++extnd[SHOULD]; 852 break; 853 case WG_PENDING: 854 ++pend[SHOULD]; 855 pending++; 856 default: 857 break; 858 } 859 } 860 if (group.contains("MAY")) { 861 mayTotal++; 862 switch (methodStatus) { 863 case AUTOMATED: 864 automated++; 865 ++auto[MAY]; 866 if (testApproval.equals(STATUS.WG_PENDING)) 867 readyToBeApproved.put(method.getName(), method.getDeclaringClass().getCanonicalName()); 868 break; 869 case CLIENT_ONLY: 870 clients.add(method.getName()); 871 ++client[MAY]; 872 break; 873 case MANUAL: 874 manuals.add(method.getName()); 875 ++manual[MAY]; 876 break; 877 case NOT_IMPLEMENTED: 878 ++unimplmnt[MAY]; 879 needCode.put(method.getName(), method.getDeclaringClass().getCanonicalName()); 880 break; 881 case INDIRECT: 882 indirectCases.add(method); 883 ++indirect[MAY]; 884 break; 885 } 886 switch (testApproval) { 887 case WG_APPROVED: 888 ++approve[MAY]; 889 approved++; 890 break; 891 case WG_CLARIFICATION: 892 clarification++; 893 ++clarify[MAY]; 894 break; 895 case WG_DEPRECATED: 896 deprecated++; 897 ++deprctd[MAY]; 898 break; 899 case WG_EXTENSION: 900 extended++; 901 ++extnd[MAY]; 902 break; 903 case WG_PENDING: 904 ++pend[MAY]; 905 pending++; 906 default: 907 break; 908 } 909 } 910 } else { // for all the other test cases 911 switch (methodStatus) { 912 case AUTOMATED: 913 automated++; 914 if (testApproval.equals(STATUS.WG_APPROVED)){ 915 if (testApproval.equals(STATUS.WG_PENDING)) 916 readyToBeApproved.put(method.getName(), method.getDeclaringClass().getCanonicalName()); 917 } 918 break; 919 case CLIENT_ONLY: 920 clients.add(method.getName()); 921 break; 922 case MANUAL: 923 manuals.add(method.getName()); 924 break; 925 case NOT_IMPLEMENTED: 926 needCode.put(method.getName(), method.getDeclaringClass().getCanonicalName()); 927 break; 928 case INDIRECT: 929 indirectCases.add(method); 930 break; 931 } 932 switch (testApproval) { 933 case WG_APPROVED: 934 approved++; 935 break; 936 case WG_CLARIFICATION: 937 clarification++; 938 break; 939 case WG_DEPRECATED: 940 deprecated++; 941 break; 942 case WG_EXTENSION: 943 extended++; 944 break; 945 case WG_PENDING: 946 pending++; 947 default: 948 break; 949 } 950 } 951 } 952 953 private void writeTestCaseDetails(Method method, String className) 954 throws IOException, SecurityException { 955 SpecTest testLdp = method.getAnnotation(SpecTest.class); 956 Test test = method.getAnnotation(Test.class); 957 if (testLdp == null || test == null) { 958 return; 959 } 960 961 html.li(id(method.getName())); 962 html.b(); 963 html.a(href(ReportUtils.getJavadocLink(method))) 964 .content(method.getName()); 965 html._b(); 966 967 html.div(class_("pad-left")); 968 html.p().write("")._p(); 969 html.b().write("Used by Class: ")._b(); 970 boolean seen=false; 971 for (@SuppressWarnings("rawtypes") Class c: testClasses) { 972 try { 973 @SuppressWarnings("unchecked") 974 Method m = c.getMethod(method.getName(), (Class[])null); 975 if (m != null) { 976 String cName = c.getCanonicalName(); 977 String normalizedName = AbstractEarlReporter.createTestCaseName(cName, method.getName()); 978 String labelName; 979 String shortClassName = cName.substring(cName.lastIndexOf(".") + 1); 980 if (seen) { 981 labelName = ", " + shortClassName; 982 } else { 983 seen = true; 984 labelName = shortClassName; 985 } 986 html.span(id(normalizedName)).write(labelName)._span(); 987 } 988 } catch (NoSuchMethodException e) { 989 // Ignore as not a problem, treat as if getMethod() returned null 990 } 991 } 992 html.br().b().write("Description: ")._b().write(test.description()); 993 html.br().b().write("Specification Section: ")._b() 994 .a(href(testLdp.specRefUri())) 995 .write(testLdp.specRefUri())._a(); 996 html.br().b().write("Groups: ")._b() 997 .write(Arrays.toString(test.groups())); 998 html.br().b().write("Status: ")._b() 999 .write(testLdp.approval().toString()); 1000 html.br().b().write("Test Case Implementation: ")._b() 1001 .write("" + testLdp.testMethod()); 1002 html.br().b().write("Enabled: ")._b() 1003 .write("" + test.enabled()); 1004 if(testLdp.steps().length != 0) 1005 writeSteps(testLdp.steps(), method.getName()); 1006 if(!testLdp.comment().equals("")) 1007 html.p(class_("note")).b().write("NOTE: ")._b() 1008 .write(testLdp.comment())._p(); 1009 if(testLdp.testMethod().equals(METHOD.INDIRECT)){ 1010 html.p().b().write("This test is covered Indirectly by other test cases.")._b()._p(); 1011 html.p().content("Test Cases that cover this test:"); 1012 html.ul(); 1013 for(Class<?> coverTest : testLdp.coveredByTests()) { 1014 Method[] classMethod = coverTest.getDeclaredMethods(); 1015 for(Method m : classMethod) { 1016 if(m.getAnnotation(Test.class) != null) { 1017 String group = Arrays.toString(m.getAnnotation(Test.class).groups()); 1018 for(String groupCover : testLdp.coveredByGroups()) { 1019 if(group.contains(groupCover)) { 1020 String testCaseName = m.getDeclaringClass().getCanonicalName(); 1021 testCaseName = testCaseName.substring(testCaseName.lastIndexOf(".") + 1); 1022 String normalizedName = AbstractEarlReporter.createTestCaseName(testCaseName, m.getName()); 1023 html.li().a(href("#" + m.getName())).b().write(normalizedName)._b()._a()._li(); 1024 } 1025 } 1026 } 1027 } 1028 } 1029 html._ul(); 1030 } 1031 html._div()._li(); 1032 toTestClass(className); 1033 } 1034 1035 private static void writeSteps(String[] steps, String title) throws IOException { 1036 html.p().content("How to Run " + title); 1037 html.ul(); 1038 for(String step : steps) 1039 html.li().content(step); 1040 html._ul(); 1041 } 1042 1043 private static void writeStatusLegend() throws IOException { 1044 html.write("<svg width=\"200\" height=\"200\">", NO_ESCAPE); 1045 html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"0\" style=\"fill:#a2bf2f\"/>", NO_ESCAPE); 1046 html.write("<text x=\"20\" y=\"13\" fill=\"black\">Approved</text>", NO_ESCAPE); 1047 1048 html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"20\" style=\"fill:#1cbfbb\"/>", NO_ESCAPE); 1049 html.write("<text x=\"20\" y=\"33\" fill=\"black\">Pending</text>", NO_ESCAPE); 1050 1051 html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"40\" style=\"fill:#bfa22f\"/>", NO_ESCAPE); 1052 html.write("<text x=\"20\" y=\"53\" fill=\"black\">Extension</text>", NO_ESCAPE); 1053 1054 html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"60\" style=\"fill:#606060 \"/>", NO_ESCAPE); 1055 html.write("<text x=\"20\" y=\"73\" fill=\"black\">Deprecated</text>", NO_ESCAPE); 1056 1057 html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"80\" style=\"fill:#1bff95 \"/>", NO_ESCAPE); 1058 html.write("<text x=\"20\" y=\"93\" fill=\"black\">Clarification</text>", NO_ESCAPE); 1059 1060 html.write("</svg>"); 1061 1062 html._span(); 1063 } 1064 1065 private static void writeImplmntLegend() throws IOException { 1066 html.write("<svg width=\"200\" height=\"200\">", NO_ESCAPE); 1067 html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"0\" style=\"fill:#0099cc\"/>", NO_ESCAPE); 1068 html.write("<text x=\"20\" y=\"13\" fill=\"black\">Automated</text>", NO_ESCAPE); 1069 1070 html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"20\" style=\"fill:#bf1c56\"/>", NO_ESCAPE); 1071 html.write("<text x=\"20\" y=\"33\" fill=\"black\">Not Implemented</text>", NO_ESCAPE); 1072 1073 html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"40\" style=\"fill:#8d1cbf\"/>", NO_ESCAPE); 1074 html.write("<text x=\"20\" y=\"53\" fill=\"black\">Client Only</text>", NO_ESCAPE); 1075 1076 html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"60\" style=\"fill:#3300cc\"/>", NO_ESCAPE); 1077 html.write("<text x=\"20\" y=\"73\" fill=\"black\">Manual</text>", NO_ESCAPE); 1078 1079 html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"80\" style=\"fill:#ffcc66 \"/>", NO_ESCAPE); 1080 html.write("<text x=\"20\" y=\"93\" fill=\"black\">Indirect</text>", NO_ESCAPE); 1081 1082 html.write("</svg>"); 1083 1084 html._span(); 1085 } 1086 1087 private static void writeGraphDescription() throws IOException { 1088 html.h3().content("Description of the Chart Information"); 1089 1090 html.h4().content("Test Status"); 1091 html.ul(); 1092 html.li().b().write("Approved")._b().write(" - the working group has approved this test case")._li(); 1093 html.li().b().write("Pending approval")._b().write(" (default) - no official recommendation from the working group supporting the specification being tested by this test suite")._li(); 1094 html.li().b().write("Extension")._b().write(" - valuable test case but not part of the approved set")._li(); 1095 html.li().b().write("Deprecated")._b().write(" - no longer recommended by the working group")._li(); 1096 html.li().b().write("Clarification")._b().write(" - requires further clarification from the working group")._li(); 1097 html._ul(); 1098 1099 html.h4().content("Test Implementation"); 1100 html.ul(); 1101 html.li().b().write("Automated")._b().write(" - implementation complete")._li(); 1102 html.li().b().write("Not Implemented")._b().write(" (default) - possible to implement, just not done")._li(); 1103 html.li().b().write("Client Only")._b().write(" - test is only client-side, this test suite doesn't test it")._li(); 1104 html.li().b().write("Manual")._b().write(" - server test but not automated")._li(); 1105 html._ul(); 1106 } 1107 1108 private static void toTop() throws IOException { 1109 html.p(class_("totop")).a(href("#top")).content("Back to Top")._p(); 1110 } 1111 1112 private static void toTestClass(String name) throws IOException { 1113 html.p(class_("totest")).a(href("#" + name)) 1114 .content("Back to Main Test Class")._p(); 1115 } 1116 1117 private static void writeCss() throws IOException { 1118 1119 html.style().write(StringResource.get("testCaseStyle.css"), NO_ESCAPE) 1120 ._style(); 1121 } 1122 1123 private static void writeReport(String directory, String title, String output) { 1124 PrettyWriter writer = null; 1125 new File(directory).mkdirs(); 1126 try { 1127 writer = new PrettyWriter(new FileWriter(directory 1128 + "/" + title + "-coverage-report.html")); 1129 writer.write(output); 1130 1131 } catch (IOException e) { 1132 e.printStackTrace(System.err); 1133 } finally { 1134 try { 1135 if (writer != null) 1136 writer.close(); 1137 } catch (IOException e) { 1138 e.printStackTrace(System.err); 1139 } 1140 } 1141 } 1142 1143 public static int getConformanceIndex(String conformance) { 1144 String c = conformance.toUpperCase(); 1145 if (c.equals("MUST")) return MUST; 1146 if (c.equals("SHOULD")) return SHOULD; 1147 if (c.contains("MAY")) return MAY; 1148 return OTHER; 1149 } 1150 1151}