001package org.w3.ldp.testsuite.reporter;
002
003import static org.rendersnake.HtmlAttributesFactory.NO_ESCAPE;
004import static org.rendersnake.HtmlAttributesFactory.class_;
005import static org.rendersnake.HtmlAttributesFactory.colspan;
006import static org.rendersnake.HtmlAttributesFactory.href;
007import static org.rendersnake.HtmlAttributesFactory.id;
008import static org.rendersnake.HtmlAttributesFactory.rowspan;
009import static org.rendersnake.HtmlAttributesFactory.style;
010
011import java.io.File;
012import java.io.FileWriter;
013import java.io.IOException;
014import java.io.StringWriter;
015import java.lang.reflect.Method;
016import java.text.DateFormat;
017import java.util.ArrayList;
018import java.util.Arrays;
019import java.util.Date;
020import java.util.HashMap;
021import java.util.HashSet;
022import java.util.Iterator;
023import java.util.Locale;
024import java.util.Map.Entry;
025import java.util.Set;
026
027import org.rendersnake.HtmlCanvas;
028import org.rendersnake.StringResource;
029import org.rendersnake.tools.PrettyWriter;
030import org.testng.annotations.Test;
031import org.w3.ldp.testsuite.BuildProperties;
032import org.w3.ldp.testsuite.LdpTestSuite;
033import org.w3.ldp.testsuite.annotations.SpecTest;
034import org.w3.ldp.testsuite.annotations.SpecTest.METHOD;
035import org.w3.ldp.testsuite.annotations.SpecTest.STATUS;
036import org.w3.ldp.testsuite.test.BasicContainerTest;
037import org.w3.ldp.testsuite.test.CommonContainerTest;
038import org.w3.ldp.testsuite.test.CommonResourceTest;
039import org.w3.ldp.testsuite.test.DirectContainerTest;
040import org.w3.ldp.testsuite.test.IndirectContainerTest;
041import org.w3.ldp.testsuite.test.NonRDFSourceTest;
042import org.w3.ldp.testsuite.test.RdfSourceTest;
043
044public class LdpTestCaseReporter {
045
046        private static HtmlCanvas html;
047
048        private static StringWriter graphs = new StringWriter();
049
050        private static Set<String> refURI = new HashSet<String>();
051
052        private static ArrayList<String> clients = new ArrayList<String>();
053        private static ArrayList<String> manuals = new ArrayList<String>();
054        private static HashMap<String, String> readyToBeApproved = new HashMap<String, String>(); // key==method, value==class
055        private static HashMap<String, String> needCode = new HashMap<String, String>(); // key==method, value==class
056
057        private static final HashMap<String, String> implmColor = new HashMap<String, String>(); // key==label value==color
058        private static final HashMap<String, String> statusColor = new HashMap<String, String>();
059        static{
060                implmColor.put("automated", "#0099cc");
061                implmColor.put("unimplemented", "#bf1c56");
062                implmColor.put("client", "#8d1cbf");
063                implmColor.put("manual", "#3300cc");
064                implmColor.put("indirect", "#ffcc66");
065                statusColor.put("approved", "#a2bf2f");
066                statusColor.put("pending", "#1cbfbb");
067                statusColor.put("extends", "#bfa22f");
068                statusColor.put("deprecated", "#606060");
069                statusColor.put("clarify", "#1bff95");
070        }
071        public static final int MUST = 0;
072        public static final int SHOULD = 1;
073        public static final int MAY = 2;
074        public static final int OTHER = 3;
075
076        private static int totalTests = 0;
077        private static int automated = 0;
078
079        private static int mustTotal = 0;
080        private static int shouldTotal = 0;
081        private static int mayTotal = 0;
082
083        private static int[] auto  = {0, 0, 0};
084        private static int[] unimplmnt  = {0, 0, 0};
085
086        private static int pending = 0;
087        private static int approved = 0;
088        private static int extended = 0;
089        private static int deprecated = 0;
090        private static int clarification = 0;
091
092        private static Class<BasicContainerTest> bcTest = BasicContainerTest.class;
093        private static Class<RdfSourceTest> rdfSourceTest = RdfSourceTest.class;
094        private static Class<IndirectContainerTest> indirectContainerTest = IndirectContainerTest.class;
095        private static Class<DirectContainerTest> directContianerTest = DirectContainerTest.class;
096        private static Class<CommonContainerTest> commonContainerTest = CommonContainerTest.class;
097        private static Class<CommonResourceTest> commonResourceTest = CommonResourceTest.class;
098        private static Class<NonRDFSourceTest> nonRdfSourceTest = NonRDFSourceTest.class;
099        
100        private static ArrayList<Method> indirectCases = new ArrayList<Method>();
101        
102        @SuppressWarnings("rawtypes")
103        private static Class[] defaultTestClasses = {
104                rdfSourceTest,
105                bcTest,
106                commonContainerTest,
107                commonResourceTest,
108                nonRdfSourceTest,
109                indirectContainerTest,
110                directContianerTest};
111
112        private static int[] extnd = {0, 0, 0};
113        private static int[] deprctd  = {0, 0, 0};
114        private static int[] pend  = {0, 0, 0};
115        private static int[] manual = {0, 0, 0};
116        private static int[] client = {0, 0, 0};
117        private static int[] approve = {0, 0, 0};
118        private static int[] clarify = {0, 0, 0};
119        private static int[] indirect = {0, 0, 0};
120        
121        @SuppressWarnings("rawtypes")
122        protected Class[] testClasses;
123        
124        private String title = "LDP";
125        private String specUri = LdpTestSuite.SPEC_URI;
126        
127        public LdpTestCaseReporter() {
128                this.testClasses = defaultTestClasses;
129        }
130        
131        @SuppressWarnings("rawtypes")
132        public LdpTestCaseReporter(Class[] inTestClasses, String title, String uri) {
133                this.testClasses = inTestClasses;
134                this.title = title;
135                this.specUri = uri;
136        }
137
138        public static void main(String[] args) throws IOException, SecurityException {
139                LdpTestCaseReporter reporter = new LdpTestCaseReporter();
140                reporter.generateReport("ldp-testsuite");
141        }
142        
143        public void generateReport(String title) throws IOException, SecurityException {
144                System.out.println("Executing coverage report...");
145                this.initializeTestClasses();
146                this.generateHTMLReport();
147                amendReport();
148                writeReport(LdpTestSuite.OUTPUT_DIR, title, html.toHtml());
149                System.out.println("Done!");
150        }
151
152        @SuppressWarnings("unchecked")
153        protected void generateHTMLReport() throws IOException, SecurityException {
154                html = new HtmlCanvas();
155                html.html().head();
156                writeCss();
157                html.title().content(title + ": Test Cases Coverage Report")._head().body();
158
159                html.h1().content("W3C Linked Data Platform (" + title + ") Test Suite: Test Cases Coverage Report");
160                html.p().a(href("http://www.w3.org/2012/ldp/")).content("See also W3C Linked Data Platform WG")._p();
161                html.p().a(href(specUri)).content("Specification Requirements Page")._p();
162
163                final String commit = BuildProperties.getRevision();
164                if (commit != null) {
165                        html.div()
166                                .write("Test Suite Revision: ")
167                                .a(href("https://github.com/w3c/ldp-testsuite/commit/" + commit)).content(commit)._div();
168                }
169                DateFormat dateFormat = DateFormat.getDateInstance(DateFormat.MEDIUM, Locale.ENGLISH);
170                html.div().content("Updated: " + dateFormat.format(new Date()));
171
172                // Generate the summary diagrams and information
173                createSummaryReport();
174                toTop();
175                
176                // Go through all the classes and generate details of each test case
177                for (@SuppressWarnings("rawtypes") Class testcaseClass: testClasses) {
178                        writeTestCasesForClass(testcaseClass);
179                }
180        }
181
182        private static void amendReport() throws IOException {
183                html.script().content(StringResource.get("/raphael/raphael-min.js"), NO_ESCAPE);
184                html.script().content(StringResource.get("/prototype/prototype.js"), NO_ESCAPE);
185                html.script().content(StringResource.get("/grafico/grafico-min.js"), NO_ESCAPE);
186                html.write(graphs.toString(), NO_ESCAPE);
187
188                html._body()._html();
189        }
190
191        protected void createSummaryReport() throws IOException {
192
193                html.h2().content("Summary of Test Methods");
194                html.table(class_("summary"));
195                html.tr().th().content("Totals");
196                html.th().content("Coverage");
197                html.th().content("Not Implemented");
198                html._tr();
199
200                html.tr();
201                html.td(rowspan("2")).b().write("" + totalTests)._b().write(" Total Tests");
202                html.ul();
203
204                html.li();
205                writeColorBlock(statusColor, "approved");
206                html.b().write(approved + " ")._b().write("Approved");
207                html._li();
208
209                html.li();
210                writeColorBlock(statusColor, "pending");
211                html.b().write(pending + " ")._b().write("Pending");
212                html._li();
213
214                html.li();
215                writeColorBlock(statusColor, "extends");
216                html.b().write(extended + " ")._b().write("Extension");
217                html._li();
218
219                html.li();
220                writeColorBlock(statusColor, "deprecated");
221                html.b().write(deprecated + " ")._b().write("No longer valid");
222                html._li();
223
224                html.li();
225                writeColorBlock(statusColor, "clarify");
226                html.b().write(clarification + " ")._b().write("Needs to be clarified with WG");
227                html._li();
228
229                html._ul();
230
231                html.br().b().a(href("#tobeapproved")).write(readyToBeApproved.size()
232                                + " Ready for Approval")._a()._b();
233
234                html.br();
235                html.span(class_("chartStart"));
236                // html.label(class_("label")).b().write("Test Case Implementation for Totals")._b()._label();
237                html.div(class_("barChart").id("overall_statusbar"))._div();
238                generateStatusGraphJSON("overall", approve, pend, extnd, deprctd, clarify);
239                html._span();
240                html._td();
241
242                html.td();
243
244                html.b().write(automated + " / " + totalTests)._b()
245                                .write(" of Total Tests Automated").br();
246                html.b().write(automated + " / " + (automated + needCode.size()))._b()
247                                .write(" of Tests Possible to Automate");
248                html.ul();
249                html.li().b().write("" + refURI.size())._b().write(" Requirements Covered")
250                                ._li();
251                html.ul().li().b().write("" + mustTotal)._b().write(" MUST")._li();
252                html.li().b().write("" + shouldTotal)._b().write(" SHOULD")._li();
253                html.li().b().write("" + mayTotal)._b().write(" MAY")._li()._ul();
254                html._ul();
255
256                html.ul();
257                int implemented = getTotal(auto);
258                html.li();
259                writeColorBlock(implmColor, "automated");
260                html.b().write(implemented + " ")._b().write("Requirements Automated");
261                html._li();
262
263                html.ul();
264                html.li().b().write(auto[MUST] + " / " + mustTotal)._b().write(" MUST")._li();
265                html.li().b().write(auto[SHOULD] + " / " + shouldTotal)._b().write(" SHOULD")
266                                ._li();
267                html.li().b().write(auto[MAY] + " / " + mayTotal)._b().write(" MAY")._li();
268                html._ul();
269                html._ul();
270                
271                writeColorBlock(implmColor, "indirect");
272                html.b().write(indirectCases.size())._b().write(" Tests ").a(href(("#indirectCases"))).write("Indirectly Covered")._a();
273                
274                html._td();
275
276                // html.td().content(totalImplemented + " Tests");
277                html.td();
278                html.b().write((needCode.size() + clients.size() + manuals.size()) + " ")._b()
279                                .write("of the Unimplemented Tests");
280                html.ul();
281
282                html.li().b().write(needCode.size() + " ")._b().write("of the Tests ")
283                                .a(href("#needCode")).write("Yet to be Coded")._a()._li();
284                /*
285                 * TODO: Determine if disabled is really valuable or not
286                 * html.li().b().write(disabled +
287                 * " ")._b().write("of the Tests not enabled")._li();
288                 */
289                html.li();
290                writeColorBlock(implmColor, "client");
291                html.b().write(clients.size() + " ")._b().write("of the Total are ")
292                                .a(href("#clientTests")).write("Client-Based Tests")._a();
293                html._li();
294
295                html.li();
296                writeColorBlock(implmColor, "manual");
297                html.b().write(manuals.size() + " ")._b().write("of the Total must be ")
298                                .a(href("#manualTests")).write("Tested Manually")._a();
299                html._li();
300
301                html._ul();
302
303                html.write("From the Total, ");
304
305                html.ul();
306                int unimplemented = getTotal(unimplmnt);
307
308                html.li();
309                writeColorBlock(implmColor, "client");
310                html.b().write(clients.size() + " ")._b().write("of the Total are ")
311                                .a(href("#clientTests")).write("Client-Based Tests")._a();
312                html._li();
313
314                html.li();
315                writeColorBlock(implmColor, "unimplemented");
316                html.b().write(unimplemented + " ")._b().write("Requirements not Implemented");
317                html._li();
318
319                html.ul();
320                html.li().b().write(unimplmnt[MUST] + " ")._b().write("MUST")._li();
321                html.li().b().write(unimplmnt[SHOULD] + " ")._b().write("SHOULD")._li();
322                html.li().b().write(unimplmnt[MAY] + " ")._b().write("MAY")._li();
323                html._ul();
324                html._ul()._td();
325                html._tr();
326
327                html.tr().td(colspan("2").style("text-align:center;"));
328                html.span(class_("chartStart"));
329                // html.label(class_("label")).b().write("Test Case Status for Coverage")._b()._label();
330                html.div(class_("barChart").id("overall_implmtbar"))._div();
331                writeImplementationGraph("overall", auto, unimplmnt, client, manual, indirect);
332                html._span();
333                html._td()._tr();
334                html._table();
335
336                writeGraphDescription();
337                writeSummary();
338
339        }
340
341        private static void writeColorBlock(HashMap<String, String> list, String string) throws IOException {
342                // TODO: Create a class for this style in our stylesheet.
343                html.div(style("background-color:" + list.get(string) + ";").class_("color-block"))._div();
344        }
345
346        private static int getTotal(int[] array) {
347                int total = 0;
348                for(int i = 0; i < array.length; i++)
349                        total += array[i];
350                return total;
351        }
352
353        @SuppressWarnings({ "rawtypes", "unchecked" })
354        protected void initializeTestClasses() throws IOException {
355                for (Class testcaseClass: testClasses) {
356                        computeTestCasesStats(testcaseClass);
357                }
358        }
359
360        @SuppressWarnings("unchecked")
361        protected void writeSummary() throws IOException {
362                html.h2(id("tobeapproved")).content("Test Cases Ready for Approval");
363
364                if (readyToBeApproved.size() == 0) {
365                        html.b().write("No test cases awaiting approval.")._b().br();
366                } else {
367                        html.p().write("For details of test cases, ")
368                                        .a(href("https://github.com/w3c/ldp-testsuite"))
369                                        .write("see source in GitHub")._a()._p();
370                        html.ul();
371                        iterateTableList(readyToBeApproved);
372                        html._ul();
373                }
374                toTop();
375
376                html.h2(id("needCode")).content("Test Cases Yet to be Implemented");
377                if (needCode.size() == 0)
378                        html.b().write("No test cases that need to be Implemented")._b()
379                                        .br();
380                else {
381                        html.ul();
382                        iterateTableList(needCode);
383                        html._ul();
384                }
385                toTop();
386
387                html.h2().a(id("indirectCases")).content("Test Cases Covered Indirectly")._h2();
388                writeIndirectCases();
389                
390                html.h2().content("Implemented Test Classes");
391
392                for (@SuppressWarnings("rawtypes") Class testcaseClass: testClasses) {
393                        writeTestTables(testcaseClass);
394                }
395
396                toTop();
397
398                html.h2().a(id("manualTests"))
399                                .content("Tests that Must be Tested Manually")._h2();
400                generateList(manuals);
401                html.h2().a(id("clientTests")).content("Client-Based Test Cases")._h2();
402                generateList(clients);
403
404        }
405
406        private void writeIndirectCases() throws IOException {
407                if(indirectCases.size() == 0){
408                        html.p().b().write("No Indirect Test Cases.")._b()._p();
409                } else {
410                        html.ul();
411                        for (Method method : indirectCases) {
412                                if (method.getAnnotation(SpecTest.class) != null) {
413                                        String name = method.getDeclaringClass().getCanonicalName();
414                                        name = name.substring(name.lastIndexOf(".") + 1);
415                                        String normalizedName = AbstractEarlReporter.createTestCaseName(name, method.getName());
416                                        html.li().a(href("#" + method.getName())).b().write(normalizedName)._b()._a()._li();
417                                }
418                        }
419                        html._ul();
420                }
421                
422        }
423
424        private static <T> void writeTestTables(Class<T> testClass) throws IOException {
425                html.b().a(href("#" + testClass.getCanonicalName()))
426                                .content(testClass.getCanonicalName())._b();
427                html.br();
428                writeTestClassTable(testClass);
429                html.br();
430        }
431
432        private static void iterateTableList(HashMap<String, String> list) throws IOException {
433                String className;
434                String methodName;
435                Iterator<Entry<String, String>> codeIter = list.entrySet().iterator();
436                while(codeIter.hasNext()){
437                        Entry<String, String> entry = codeIter.next();
438                        className = entry.getValue();
439                        className = className.substring(className.lastIndexOf(".") + 1);
440                        methodName = entry.getKey();
441                        String normalizedName = AbstractEarlReporter.createTestCaseName(className, methodName);
442                        html.li().a(href("#" + methodName)).b().write(normalizedName)._b()._a()._li();
443                }
444        }
445
446        private static <T> void writeTestClassTable(Class<T> classType)
447                        throws IOException {
448                html.span(class_("chartStart"));
449                html.label(class_("label")).b().write("Test Case Status")._b()._label();
450                html.div(class_("smallBar").id(classType.getSimpleName() + "_statusbar"))._div();
451                writeStatusLegend();
452
453                html.span(class_("chartStart"));
454                html.label(class_("label")).b().write("Test Case Implementation")._b()._label();
455                html.div(class_("smallBar").id(classType.getSimpleName() + "_implmtbar"))._div();
456                writeImplmntLegend();
457
458                acquireTestInfo(classType);
459
460        }
461
462        private static <T> void acquireTestInfo(Class<T> classType)
463                        throws IOException {
464                int total = 0, must = 0, should = 0, may = 0;
465
466                int[] autoReq = { 0, 0, 0 }, unimReq = { 0, 0, 0 }, clientReq = { 0, 0, 0 }, manReq = { 0, 0, 0 }, indirect = { 0, 0, 0 };
467                int[] apprReq = { 0, 0, 0 }, pendReq = { 0, 0, 0 }, extReq = { 0, 0, 0 },
468                                depreReq = {0, 0, 0 }, clariReq = { 0, 0, 0 };
469
470                Method[] methods = classType.getDeclaredMethods();
471
472                for (Method method : methods) {
473                        if (method.isAnnotationPresent(SpecTest.class)
474                                        && method.isAnnotationPresent(Test.class)) {
475                                total++;
476                                SpecTest testLdp = method.getAnnotation(SpecTest.class);
477                                Test test = method.getAnnotation(Test.class);
478                                String group = Arrays.toString(test.groups());
479                                if (group.contains("MUST"))
480                                        must++;
481                                if (group.contains("SHOULD"))
482                                        should++;
483                                if (group.contains("MAY"))
484                                        may++;
485                                switch (testLdp.testMethod()) {
486                                case AUTOMATED:
487                                        if (group.contains("MUST"))
488                                                ++autoReq[MUST];
489                                        if (group.contains("SHOULD"))
490                                                ++autoReq[SHOULD];
491                                        if (group.contains("MAY"))
492                                                ++autoReq[MAY];
493                                        break;
494                                case NOT_IMPLEMENTED:
495                                        if (group.contains("MUST"))
496                                                ++unimReq[MUST];
497                                        if (group.contains("SHOULD"))
498                                                ++unimReq[SHOULD];
499                                        if (group.contains("MAY"))
500                                                ++unimReq[MAY];
501                                        break;
502                                case MANUAL:
503                                        if (group.contains("MUST"))
504                                                ++manReq[MUST];
505                                        if (group.contains("SHOULD"))
506                                                ++manReq[SHOULD];
507                                        if (group.contains("MAY"))
508                                                ++manReq[MAY];
509                                        break;
510                                case CLIENT_ONLY:
511                                        if (group.contains("MUST"))
512                                                ++clientReq[MUST];
513                                        if (group.contains("SHOULD"))
514                                                ++clientReq[SHOULD];
515                                        if (group.contains("MAY"))
516                                                ++clientReq[MAY];
517                                        break;
518                                case INDIRECT:
519                                        if (group.contains("MUST"))
520                                                ++indirect[MUST];
521                                        if (group.contains("SHOULD"))
522                                                ++indirect[SHOULD];
523                                        if (group.contains("MAY"))
524                                                ++indirect[MAY];
525                                        break;
526                                }
527                                switch (testLdp.approval()) {
528                                case WG_PENDING:
529                                        if (group.contains("MUST"))
530                                                ++pendReq[MUST];
531                                        if (group.contains("SHOULD"))
532                                                ++pendReq[SHOULD];
533                                        if (group.contains("MAY"))
534                                                ++pendReq[MAY];
535                                        break;
536                                case WG_APPROVED:
537                                        if (group.contains("MUST"))
538                                                ++apprReq[MUST];
539                                        if (group.contains("SHOULD"))
540                                                ++apprReq[SHOULD];
541                                        if (group.contains("MAY"))
542                                                ++apprReq[MAY];
543                                        break;
544                                case WG_EXTENSION:
545                                        if (group.contains("MUST"))
546                                                ++extReq[MUST];
547                                        if (group.contains("SHOULD"))
548                                                ++extReq[SHOULD];
549                                        if (group.contains("MAY"))
550                                                ++extReq[MAY];
551                                        break;
552                                case WG_DEPRECATED:
553                                        if (group.contains("MUST"))
554                                                ++depreReq[MUST];
555                                        if (group.contains("SHOULD"))
556                                                ++depreReq[SHOULD];
557                                        if (group.contains("MAY"))
558                                                ++depreReq[MAY];
559                                        break;
560                                case WG_CLARIFICATION:
561                                        if (group.contains("MUST"))
562                                                ++clariReq[MUST];
563                                        if (group.contains("SHOULD"))
564                                                ++clariReq[SHOULD];
565                                        if (group.contains("MAY"))
566                                                ++clariReq[MAY];
567                                        break;
568                                }
569                        }
570                }
571
572                // write information into Grafico bar charts
573                generateStatusGraphJSON(classType.getSimpleName(), apprReq, pendReq, extReq, depreReq, clariReq);
574                writeImplementationGraph(classType.getSimpleName(), autoReq, unimReq, clientReq, manReq, indirect);
575
576                html.table(class_("classes"));
577
578                html.tr().th().content("Total Tests");
579                html.th().content("Coverage");
580                html.th().content("Status");
581                html.th().content("Implementation");
582                html._tr();
583
584                html.tr().td().content(total + "");
585                html.td();
586                if (must > 0)
587                        html.b().write("MUST: ")._b().write(must).br();
588                if (should > 0)
589                        html.b().write("SHOULD: ")._b().write(should).br();
590                if (may > 0)
591                        html.b().write("MAY: ")._b().write(may).br();
592
593                html._td().td();
594                int approve = getTotal(apprReq);
595                if(approve > 0)
596                        html.b().write("Approved: ")._b().write("" + approve).br();
597                int pend = getTotal(pendReq);
598                if(pend > 0)
599                        html.b().write("Pending: ")._b().write("" + pend).br();
600                int extend = getTotal(extReq);
601                if (extend > 0)
602                        html.b().write("Extension: ")._b().write("" + extend).br();
603                int depre = getTotal(depreReq);
604                if (depre > 0)
605                        html.b().write("Deprecated: ")._b().write("" + depre).br();
606                int clarify = getTotal(clariReq);
607                if(clarify > 0)
608                        html.b().write("Clarification: ")._b().write("" + clarify).br();
609
610                html._td().td();
611                int auto = getTotal(autoReq);
612                if (auto > 0)
613                        html.b().write("Automated: ")._b().write("" + auto).br();
614                int unimpl = getTotal(unimReq);
615                if (unimpl > 0)
616                        html.b().write("Not Implemented: ")._b().write("" + unimpl).br();
617                int client = getTotal(clientReq);
618                if (client > 0)
619                        html.b().write("Client Only: ")._b().write("" + client).br();
620                int manual = getTotal(manReq);
621                if (manual > 0)
622                        html.b().write("Manual: ")._b().write("" + manual).br();
623
624                html._td();
625
626                html._tr();
627                html._table();
628
629        }
630
631        private static void generateStatusGraphJSON(String type, int[] apprReq, int[] pendReq, 
632                        int[] extReq, int[] depreReq, int[] clariReq)
633                        throws IOException {
634                graphs.write("<script>");
635                graphs.write("Event.observe(window, 'load', function() {");
636
637                graphs.write("var " + type + "_statusbar"
638                                + " = new Grafico.StackedBarGraph($('" + type
639                                + "_statusbar'),{");
640                graphs.write("approved: [" + apprReq[MUST] + ", " + apprReq[SHOULD] + ", " + apprReq[MAY] + " ],");
641                graphs.write("pending: [" + pendReq[MUST] + ", " + pendReq[SHOULD] + ", " + pendReq[MAY] + " ],");
642                graphs.write("extends: [" + extReq[MUST] + ", " + extReq[SHOULD] + ", " + extReq[MAY] + " ],");
643                graphs.write("deprecated: [" + depreReq[MUST] + ", " + depreReq[SHOULD] + ", " + depreReq[MAY] + " ],");
644                graphs.write("clarify: [" + clariReq[MUST] + ", " + clariReq[SHOULD] + ", " + clariReq[MAY] + "] ");
645                graphs.write("},");
646                graphs.write("{ labels: [ \"MUST\", \"SHOULD\", \"MAY\" ],");
647                graphs.write("colors: { ");
648                writeColors(statusColor);
649                graphs.write(" },");
650                graphs.write("hover_color: \"#ccccff\",");
651                graphs.write("datalabels: { ");
652                graphs.write("approved: [ \"" + apprReq[MUST] + " Approved\", \"" + apprReq[SHOULD] + " Approved\", \"" + apprReq[MAY] + " Approved\" ],");
653                graphs.write("pending: [ \"" + pendReq[MUST] + " Pending\", \"" + pendReq[SHOULD] + " Pending\", \"" + pendReq[MAY] + " Pending\" ],");
654                graphs.write("extends: [ \"" + extReq[MUST] + " Extension\", \"" + extReq[SHOULD] + " Extension\", \"" + extReq[MAY] + " Extension\" ],");
655                graphs.write("deprecated: [ \"" + depreReq[MUST] + " Deprecated\", \"" + depreReq[SHOULD] + " Deprecated\", \"" + depreReq[MAY] + " Deprecated\" ],");
656                graphs.write("clarify: [ \"" + clariReq[MUST] + " Clarification\", \"" + clariReq[SHOULD] + " Clarification\", \"" + clariReq[MAY] + " Clarification\" ] }");
657
658                graphs.write(" }); });");
659
660                graphs.write("</script>");
661
662
663        }
664
665        private static void writeImplementationGraph(String classType,
666                        int[] autoReq, int[] unimReq, int[] clientReq, int[] manReq, int[] indirect) {
667                graphs.write("<script>");
668                graphs.write("Event.observe(window, 'load', function() {");
669
670                graphs.write("var " + classType + "_implmtbar"
671                                + " = new Grafico.StackedBarGraph($('" + classType
672                                + "_implmtbar'), {");
673                graphs.write("automated: [" + autoReq[MUST] + ", " + autoReq[SHOULD] + ", " + autoReq[MAY] + " ],");
674                graphs.write("unimplemented: [" + unimReq[MUST] + ", " + unimReq[SHOULD] + ", " + unimReq[MAY] + " ],");
675                graphs.write("client: [" + clientReq[MUST] + ", " + clientReq[SHOULD] + ", " + clientReq[MAY] + " ],");
676                graphs.write("manual: [" + manReq[MUST] + ", " + manReq[SHOULD] + ", " + manReq[MAY] + "],");
677                graphs.write("indirect: [" + indirect[MUST] + ", " + indirect[SHOULD] + ", " + indirect[MAY] + " ]");
678                graphs.write("},");
679                graphs.write("{ labels: [ \"MUST\", \"SHOULD\", \"MAY\" ],");
680                graphs.write("colors: { ");
681                writeColors(implmColor);
682                graphs.write("},");
683                graphs.write("hover_color: \"#ccccff\",");
684                graphs.write("datalabels: { ");
685                graphs.write("automated: [ \"" + autoReq[MUST] + " Automated\", \"" + autoReq[SHOULD] + " Automated\", \"" + autoReq[MAY] + " Automated\" ],");
686                graphs.write("unimplemented: [ \"" + unimReq[MUST] + " Not Implemented\", \"" + unimReq[SHOULD] + " Not Implemented\", \"" + unimReq[MAY] + " Not Implemented\" ],");
687                graphs.write("client: [ \"" + clientReq[MUST] + " Client Only\", \"" + clientReq[SHOULD] + " Client Only\", \"" + clientReq[MAY] + " Client Only\" ],");
688                graphs.write("manual: [ \"" + manReq[MUST] + " Manual\", \"" + manReq[SHOULD] + " Manual\", \"" + manReq[MAY] + " Manual\" ], ");
689                graphs.write("indirect: [ \"" + indirect[MUST] + " Indirect\", \"" + indirect[SHOULD] + " Indirect\", \"" + indirect[MAY] + " Indirect\" ]");
690                graphs.write("},");
691
692                graphs.write(" }); });");
693
694                graphs.write("</script>");
695        }
696
697        private static void writeColors(HashMap<String, String> colorList) {
698                Iterator<Entry<String, String>> codeIter = colorList.entrySet().iterator();
699                Entry<String, String> value;
700                while(codeIter.hasNext()){
701                        value = codeIter.next();
702                                graphs.write(value.getKey() + ": '" + value.getValue() + "' ");
703                        if(codeIter.hasNext())
704                                graphs.write(", ");
705                }
706        }
707
708        private static void generateList(ArrayList<String> list) throws IOException {
709                if(list.size() == 0) {
710                        html.p().content("No tests of this type found.");
711                } else {
712                        html.ul();
713                        for (int i = 0; i < list.size(); i++) {
714                                html.li().a(href("#" + list.get(i))).write(list.get(i))._a()._li();
715                        }
716                        html._ul();
717                }
718        }
719
720        private static <T> void computeTestCasesStats(Class<T> classType)
721                        throws IOException {
722                String className = classType.getCanonicalName();
723                
724                for (Method method : classType.getDeclaredMethods()) {
725                        if (method.isAnnotationPresent(Test.class)) {
726                                generateTestCaseDetails(method, className);
727                        }
728                }
729        }
730        
731        private <T> void writeTestCasesForClass(Class<T> classType)
732                        throws IOException, SecurityException {
733                String className = classType.getCanonicalName();
734                html.h2().a(id(className)).write("Test Class: " + className)._a()._h2();
735                
736                html.ul();
737                for (Method method : classType.getDeclaredMethods()) {
738                        if (method.isAnnotationPresent(Test.class)) {
739                                writeTestCaseDetails(method, className);
740                        }
741                }
742                html._ul();
743                toTop();
744        }
745
746        private static void generateTestCaseDetails(Method method, String className)
747                        throws IOException {
748                SpecTest testLdp =  method.getAnnotation(SpecTest.class);
749                Test test = method.getAnnotation(Test.class);
750                if (testLdp == null || test == null) {
751                        return;
752                }
753
754                totalTests++;
755                METHOD methodStatus = testLdp.testMethod();
756                STATUS testApproval = testLdp.approval();
757                String group = Arrays.toString(test.groups());
758                if (!refURI.contains(testLdp.specRefUri())) { // for just the requirement testing
759                        refURI.add(testLdp.specRefUri());
760                        if (group.contains("MUST")) {
761                                mustTotal++;
762                                switch (methodStatus) {
763                                case AUTOMATED:
764                                        automated++;
765                                        ++auto[MUST];
766                                        if (testApproval.equals(STATUS.WG_PENDING))
767                                                readyToBeApproved.put(method.getName(), method.getDeclaringClass().getCanonicalName());
768                                        break;
769                                case CLIENT_ONLY:
770                                        clients.add(method.getName());
771                                        ++client[MUST];
772                                        break;
773                                case MANUAL:
774                                        manuals.add(method.getName());
775                                        ++manual[MUST];
776                                        break;
777                                case NOT_IMPLEMENTED:
778                                        ++unimplmnt[MUST];
779                                        needCode.put(method.getName(), method.getDeclaringClass().getCanonicalName());
780                                        break;
781                                case INDIRECT:
782                                        indirectCases.add(method);
783                                        ++indirect[MUST];
784                                        break;
785                                }
786                                switch (testApproval) {
787                                case WG_APPROVED:
788                                        ++approve[MUST];
789                                        approved++;
790                                        break;
791                                case WG_CLARIFICATION:
792                                        clarification++;
793                                        ++clarify[MUST];
794                                        break;
795                                case WG_DEPRECATED:
796                                        deprecated++;;
797                                        ++deprctd[MUST];
798                                        break;
799                                case WG_EXTENSION:
800                                        extended++;
801                                        ++extnd[MUST];
802                                        break;
803                                case WG_PENDING:
804                                        ++pend[MUST];
805                                        pending++;
806                                default:
807                                        break;
808                                }
809                        }
810                        if (group.contains("SHOULD")) {
811                                shouldTotal++;
812                                switch (methodStatus) {
813                                case AUTOMATED:
814                                        automated++;
815                                        ++auto[SHOULD];
816                                        if (testApproval.equals(STATUS.WG_PENDING))
817                                                readyToBeApproved.put(method.getName(), method.getDeclaringClass().getCanonicalName());
818                                        break;
819                                case CLIENT_ONLY:
820                                        clients.add(method.getName());
821                                        ++client[SHOULD];
822                                        break;
823                                case MANUAL:
824                                        manuals.add(method.getName());
825                                        ++manual[SHOULD];
826                                        break;
827                                case NOT_IMPLEMENTED:
828                                        ++unimplmnt[SHOULD];
829                                        needCode.put(method.getName(), method.getDeclaringClass().getCanonicalName());
830                                        break;
831                                case INDIRECT:
832                                        indirectCases.add(method);
833                                        ++indirect[SHOULD];
834                                        break;
835                                }
836                                switch (testApproval) {
837                                case WG_APPROVED:
838                                        ++approve[SHOULD];
839                                        approved++;
840                                        break;
841                                case WG_CLARIFICATION:
842                                        clarification++;
843                                        ++clarify[SHOULD];
844                                        break;
845                                case WG_DEPRECATED:
846                                        deprecated++;
847                                        ++deprctd[SHOULD];
848                                        break;
849                                case WG_EXTENSION:
850                                        extended++;
851                                        ++extnd[SHOULD];
852                                        break;
853                                case WG_PENDING:
854                                        ++pend[SHOULD];
855                                        pending++;
856                                default:
857                                        break;
858                                }
859                        }
860                        if (group.contains("MAY")) {
861                                mayTotal++;
862                                switch (methodStatus) {
863                                case AUTOMATED:
864                                        automated++;
865                                        ++auto[MAY];
866                                        if (testApproval.equals(STATUS.WG_PENDING))
867                                                readyToBeApproved.put(method.getName(), method.getDeclaringClass().getCanonicalName());
868                                        break;
869                                case CLIENT_ONLY:
870                                        clients.add(method.getName());
871                                        ++client[MAY];
872                                        break;
873                                case MANUAL:
874                                        manuals.add(method.getName());
875                                        ++manual[MAY];
876                                        break;
877                                case NOT_IMPLEMENTED:
878                                        ++unimplmnt[MAY];
879                                        needCode.put(method.getName(), method.getDeclaringClass().getCanonicalName());
880                                        break;
881                                case INDIRECT:
882                                        indirectCases.add(method);
883                                        ++indirect[MAY];
884                                        break;
885                                }
886                                switch (testApproval) {
887                                case WG_APPROVED:
888                                        ++approve[MAY];
889                                        approved++;
890                                        break;
891                                case WG_CLARIFICATION:
892                                        clarification++;
893                                        ++clarify[MAY];
894                                        break;
895                                case WG_DEPRECATED:
896                                        deprecated++;
897                                        ++deprctd[MAY];
898                                        break;
899                                case WG_EXTENSION:
900                                        extended++;
901                                        ++extnd[MAY];
902                                        break;
903                                case WG_PENDING:
904                                        ++pend[MAY];
905                                        pending++;
906                                default:
907                                        break;
908                                }
909                        }
910                } else { // for all the other test cases
911                        switch (methodStatus) {
912                        case AUTOMATED:
913                                automated++;
914                                if (testApproval.equals(STATUS.WG_APPROVED)){
915                                        if (testApproval.equals(STATUS.WG_PENDING))
916                                                readyToBeApproved.put(method.getName(), method.getDeclaringClass().getCanonicalName());
917                                }
918                                break;
919                        case CLIENT_ONLY:
920                                clients.add(method.getName());
921                                break;
922                        case MANUAL:
923                                manuals.add(method.getName());
924                                break;
925                        case NOT_IMPLEMENTED:
926                                needCode.put(method.getName(), method.getDeclaringClass().getCanonicalName());
927                                break;
928                        case INDIRECT:
929                                indirectCases.add(method);
930                                break;
931                        }
932                        switch (testApproval) {
933                        case WG_APPROVED:
934                                approved++;
935                                break;
936                        case WG_CLARIFICATION:
937                                clarification++;
938                                break;
939                        case WG_DEPRECATED:
940                                deprecated++;
941                                break;
942                        case WG_EXTENSION:
943                                extended++;
944                                break;
945                        case WG_PENDING:
946                                pending++;
947                        default:
948                                break;
949                        }
950                }
951        }
952        
953        private void writeTestCaseDetails(Method method, String className)
954                                throws IOException, SecurityException {
955                SpecTest testLdp =  method.getAnnotation(SpecTest.class);
956                Test test = method.getAnnotation(Test.class);
957                if (testLdp == null || test == null) {
958                        return;
959                }
960                
961                html.li(id(method.getName()));
962                html.b();
963                html.a(href(ReportUtils.getJavadocLink(method)))
964                                .content(method.getName());
965                html._b();
966                
967                html.div(class_("pad-left"));
968                html.p().write("")._p();
969                html.b().write("Used by Class: ")._b();
970                boolean seen=false;
971        for (@SuppressWarnings("rawtypes") Class c: testClasses) {
972                try {
973            @SuppressWarnings("unchecked")
974                                Method m = c.getMethod(method.getName(), (Class[])null);
975                        if (m != null) {
976                                String cName = c.getCanonicalName();
977                                        String normalizedName = AbstractEarlReporter.createTestCaseName(cName, method.getName());
978                                        String labelName;
979                                        String shortClassName = cName.substring(cName.lastIndexOf(".") + 1);
980                                        if (seen) {
981                                                labelName = ", " + shortClassName;
982                                        } else {
983                                                seen = true;
984                                                labelName = shortClassName;                                             
985                                        }
986                                html.span(id(normalizedName)).write(labelName)._span();
987                        }
988                } catch (NoSuchMethodException e) { 
989                        // Ignore as not a problem, treat as if getMethod() returned null
990                }
991        }
992                html.br().b().write("Description: ")._b().write(test.description());
993                html.br().b().write("Specification Section: ")._b()
994                                .a(href(testLdp.specRefUri()))
995                                .write(testLdp.specRefUri())._a();
996                html.br().b().write("Groups: ")._b()
997                                .write(Arrays.toString(test.groups()));
998                html.br().b().write("Status: ")._b()
999                                .write(testLdp.approval().toString());
1000                html.br().b().write("Test Case Implementation: ")._b()
1001                                .write("" + testLdp.testMethod());
1002                html.br().b().write("Enabled: ")._b()
1003                                .write("" + test.enabled());
1004                if(testLdp.steps().length != 0)
1005                        writeSteps(testLdp.steps(), method.getName());
1006                if(!testLdp.comment().equals(""))
1007                        html.p(class_("note")).b().write("NOTE: ")._b()
1008                                .write(testLdp.comment())._p();
1009                if(testLdp.testMethod().equals(METHOD.INDIRECT)){
1010                        html.p().b().write("This test is covered Indirectly by other test cases.")._b()._p();
1011                        html.p().content("Test Cases that cover this test:");
1012                        html.ul();
1013                        for(Class<?> coverTest : testLdp.coveredByTests()) {
1014                                Method[] classMethod = coverTest.getDeclaredMethods();
1015                                for(Method m : classMethod) {
1016                                        if(m.getAnnotation(Test.class) != null) {
1017                                                String group = Arrays.toString(m.getAnnotation(Test.class).groups()); 
1018                                                for(String groupCover : testLdp.coveredByGroups()) {
1019                                                        if(group.contains(groupCover)) {
1020                                                                String testCaseName = m.getDeclaringClass().getCanonicalName();
1021                                                                testCaseName = testCaseName.substring(testCaseName.lastIndexOf(".") + 1);
1022                                                                String normalizedName = AbstractEarlReporter.createTestCaseName(testCaseName, m.getName());
1023                                                                html.li().a(href("#" + m.getName())).b().write(normalizedName)._b()._a()._li();
1024                                                        }                                                               
1025                                                }
1026                                        }
1027                                }
1028                        }
1029                        html._ul();
1030                }
1031                html._div()._li();
1032                toTestClass(className);
1033        }
1034
1035        private static void writeSteps(String[] steps, String title) throws IOException {
1036                html.p().content("How to Run " + title);
1037                html.ul();
1038                for(String step : steps)
1039                        html.li().content(step);
1040                html._ul();
1041        }
1042
1043        private static void writeStatusLegend() throws IOException {
1044                html.write("<svg width=\"200\" height=\"200\">", NO_ESCAPE);
1045                html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"0\" style=\"fill:#a2bf2f\"/>", NO_ESCAPE);
1046                html.write("<text x=\"20\" y=\"13\" fill=\"black\">Approved</text>", NO_ESCAPE);
1047
1048                html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"20\" style=\"fill:#1cbfbb\"/>", NO_ESCAPE);
1049                html.write("<text x=\"20\" y=\"33\" fill=\"black\">Pending</text>", NO_ESCAPE);
1050
1051                html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"40\" style=\"fill:#bfa22f\"/>", NO_ESCAPE);
1052                html.write("<text x=\"20\" y=\"53\" fill=\"black\">Extension</text>", NO_ESCAPE);
1053
1054                html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"60\" style=\"fill:#606060 \"/>", NO_ESCAPE);
1055                html.write("<text x=\"20\" y=\"73\" fill=\"black\">Deprecated</text>", NO_ESCAPE);
1056
1057                html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"80\" style=\"fill:#1bff95 \"/>", NO_ESCAPE);
1058                html.write("<text x=\"20\" y=\"93\" fill=\"black\">Clarification</text>", NO_ESCAPE);
1059                
1060                html.write("</svg>");
1061
1062                html._span();
1063        }
1064
1065        private static void writeImplmntLegend() throws IOException {
1066                html.write("<svg width=\"200\" height=\"200\">", NO_ESCAPE);
1067                html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"0\" style=\"fill:#0099cc\"/>", NO_ESCAPE);
1068                html.write("<text x=\"20\" y=\"13\" fill=\"black\">Automated</text>", NO_ESCAPE);
1069
1070                html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"20\" style=\"fill:#bf1c56\"/>", NO_ESCAPE);
1071                html.write("<text x=\"20\" y=\"33\" fill=\"black\">Not Implemented</text>", NO_ESCAPE);
1072
1073                html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"40\" style=\"fill:#8d1cbf\"/>", NO_ESCAPE);
1074                html.write("<text x=\"20\" y=\"53\" fill=\"black\">Client Only</text>", NO_ESCAPE);
1075
1076                html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"60\" style=\"fill:#3300cc\"/>", NO_ESCAPE);
1077                html.write("<text x=\"20\" y=\"73\" fill=\"black\">Manual</text>", NO_ESCAPE);
1078
1079                html.write("<rect width=\"15\" height=\"15\" x=\"0\" y=\"80\" style=\"fill:#ffcc66 \"/>", NO_ESCAPE);
1080                html.write("<text x=\"20\" y=\"93\" fill=\"black\">Indirect</text>", NO_ESCAPE);
1081
1082                html.write("</svg>");
1083
1084                html._span();
1085        }
1086
1087        private static void writeGraphDescription() throws IOException {
1088                html.h3().content("Description of the Chart Information");
1089
1090                html.h4().content("Test Status");
1091                html.ul();
1092                html.li().b().write("Approved")._b().write(" - the working group has approved this test case")._li();
1093                html.li().b().write("Pending approval")._b().write(" (default) - no official recommendation from the working group supporting the specification being tested by this test suite")._li();
1094                html.li().b().write("Extension")._b().write(" - valuable test case but not part of the approved set")._li();
1095                html.li().b().write("Deprecated")._b().write(" - no longer recommended by the working group")._li();
1096                html.li().b().write("Clarification")._b().write(" - requires further clarification from the working group")._li();
1097                html._ul();
1098
1099                html.h4().content("Test Implementation");
1100                html.ul();
1101                html.li().b().write("Automated")._b().write(" - implementation complete")._li();
1102                html.li().b().write("Not Implemented")._b().write(" (default) - possible to implement, just not done")._li();
1103                html.li().b().write("Client Only")._b().write(" - test is only client-side, this test suite doesn't test it")._li();
1104                html.li().b().write("Manual")._b().write(" - server test but not automated")._li();
1105                html._ul();
1106        }
1107
1108        private static void toTop() throws IOException {
1109                html.p(class_("totop")).a(href("#top")).content("Back to Top")._p();
1110        }
1111
1112        private static void toTestClass(String name) throws IOException {
1113                html.p(class_("totest")).a(href("#" + name))
1114                                .content("Back to Main Test Class")._p();
1115        }
1116
1117        private static void writeCss() throws IOException {
1118
1119                html.style().write(StringResource.get("testCaseStyle.css"), NO_ESCAPE)
1120                                ._style();
1121        }
1122
1123        private static void writeReport(String directory, String title, String output) {
1124                PrettyWriter writer = null;
1125                new File(directory).mkdirs();
1126                try {
1127                        writer = new PrettyWriter(new FileWriter(directory
1128                                        + "/" + title + "-coverage-report.html"));
1129                        writer.write(output);
1130
1131                } catch (IOException e) {
1132                        e.printStackTrace(System.err);
1133                } finally {
1134                        try {
1135                                if (writer != null)
1136                                        writer.close();
1137                        } catch (IOException e) {
1138                                e.printStackTrace(System.err);
1139                        }
1140                }
1141        }
1142
1143        public static int getConformanceIndex(String conformance) {
1144                String c = conformance.toUpperCase();
1145                if (c.equals("MUST")) return MUST;
1146                if (c.equals("SHOULD")) return SHOULD;
1147                if (c.contains("MAY")) return MAY;
1148                return OTHER;
1149        }
1150
1151}