Source code

001package org.w3.ldp.testsuite.reporter;
002
003import java.io.IOException;
004import java.lang.reflect.Method;
005import java.util.ArrayList;
006import java.util.Arrays;
007import java.util.GregorianCalendar;
008import java.util.Iterator;
009import java.util.List;
010import java.util.Map;
011
012import org.testng.IReporter;
013import org.testng.IResultMap;
014import org.testng.ISuite;
015import org.testng.ISuiteResult;
016import org.testng.ITestContext;
017import org.testng.ITestNGMethod;
018import org.testng.ITestResult;
019import org.testng.annotations.Test;
020import org.testng.internal.Utils;
021import org.testng.xml.XmlSuite;
022import org.w3.ldp.testsuite.LdpTestSuite;
023import org.w3.ldp.testsuite.annotations.SpecTest;
024import org.w3.ldp.testsuite.annotations.SpecTest.METHOD;
025import org.w3.ldp.testsuite.vocab.Earl;
026import org.w3.ldp.testsuite.vocab.LDP;
027
028import com.hp.hpl.jena.rdf.model.Property;
029import com.hp.hpl.jena.rdf.model.Resource;
030import com.hp.hpl.jena.rdf.model.ResourceFactory;
031import com.hp.hpl.jena.sparql.vocabulary.DOAP;
032import com.hp.hpl.jena.sparql.vocabulary.FOAF;
033import com.hp.hpl.jena.vocabulary.DCTerms;
034import com.hp.hpl.jena.vocabulary.RDF;
035
036/**
037 * Earl Reporter for the LDP Test Suite. Takes in the results of the test suite
038 * and reports the information to a Turtle file and a JSON-LD file, both of
039 * which contains Earl vocabulary.
040 */
041public class LdpEarlReporter extends AbstractEarlReporter implements IReporter {
042
043        private static final String PASS = "TEST PASSED";
044        private static final String FAIL = "TEST FAILED";
045        private static final String SKIP = "TEST SKIPPED";
046
047        // private static final String DIRECT_TEST = "DirectContainerTest";
048        // private static final String MEMBER_TEST = "MemberResourceTest";
049        // private static final String BASIC_TEST = "BasicContainerTest";
050        // private static final String INDIRECT_TEST = "IndirectContainerTest";
051        //
052        // private static String direct;
053        // private static String member;
054        // private static String basic;
055        // private static String indirect;
056
057        private static String software;
058        private static String developer;
059        private static String homepage;
060        private static String assertor;
061        private static String language;
062        private static String mbox;
063        private static String description;
064        private static String shortname;
065        private static ArrayList<String> missingParms = new ArrayList<>();
066
067        private static Property ranAsClass = ResourceFactory
068                        .createProperty(LDP.LDPT_NAMESPACE + "ranAsClass");
069
070        private static String TITLE = "ldp-testsuite";
071
072        private IResultMap passedTests;
073        private IResultMap failedTests;
074        private IResultMap skippedTests;
075
076        private String outputDirectory = LdpTestSuite.OUTPUT_DIR;
077
078        public void setOutputDirectory(String outputDirectory) {
079                this.outputDirectory = outputDirectory;
080        }
081
082        @Override
083        public void generateReport(List<XmlSuite> xmlSuites, List<ISuite> suites, String outputDirectory) {
084                try {
085                        createWriter(this.outputDirectory, "");
086                } catch (IOException e) {
087                        e.printStackTrace(System.err);
088                        System.exit(1);
089                }
090                createModel();
091                createAssertions(suites);
092                write();
093                if (missingParms.size() > 0) {
094                        System.out.print("EARL report missing values for parameters: ");
095                        boolean first=true;
096                        String o="";
097                        for (String p: missingParms) {
098                                if (first) {
099                                        first = false;
100                                        o = p;
101                                } else {
102                                        o = o + ", " + p;
103                                }
104                        }
105                        System.out.println(o);
106                }
107                try {
108                        endWriter();
109                } catch (IOException e) {
110                        e.printStackTrace(System.err);
111                        System.exit(1);
112                }
113        }
114
115        private void createAssertions(List<ISuite> suites) {
116                for (ISuite suite : suites) {
117                        // Acquire parameters
118                        // direct = suite.getParameter("directContainer");
119                        // member = suite.getParameter("memberResource");
120                        // basic = suite.getParameter("basicContainer");
121                        // indirect = suite.getParameter("indirectContainer");
122
123                        homepage = suite.getParameter("homepage");
124                        if (homepage == null) missingParms.add("homepage");
125                        
126                        assertor = suite.getParameter("assertor");
127                        if (assertor == null) missingParms.add("assertor");
128                        
129                        software = suite.getParameter("software");
130                        if (software == null) missingParms.add("software");
131                        
132                        developer = suite.getParameter("developer");
133                        if (developer == null) missingParms.add("developer");
134                        
135                        language = suite.getParameter("language");
136                        if (language == null) missingParms.add("language");
137
138                        mbox = suite.getParameter("mail");
139                        if (mbox == null) missingParms.add("mail");
140
141                        description = suite.getParameter("description");
142                        if (description == null) missingParms.add("description");
143
144                        shortname = suite.getParameter("shortname");
145                        if (shortname == null) missingParms.add("shortname");
146
147                        // Make the Assertor Resource (the thing doing the testing) 
148                        Resource assertorRes = model.createResource(assertor);
149                        assertorRes.addProperty(RDF.type, Earl.Assertor);
150                        
151                        // Create the subject resource (the thing being tested)
152                        Resource subjectResource = model.createResource(homepage,
153                                        Earl.TestSubject);
154
155                        if (description != null)
156                                subjectResource.addProperty(DOAP.description, description);
157
158                        /* Developer Resource (Person) */
159                        Resource personResource = model.createResource(null, FOAF.Person);
160                        if (mbox != null)
161                                personResource.addProperty(FOAF.mbox, mbox);
162                        if(developer != null)
163                                personResource.addProperty(FOAF.name, developer);
164
165                        subjectResource.addProperty(DOAP.developer, personResource);
166
167                        /* Software Resource */
168                        Resource softResource = model
169                                        .createResource(homepage, Earl.Software);
170                        if (software != null)
171                                softResource.addProperty(DCTerms.title, software);
172
173                        if(shortname != null)
174                                softResource.addProperty(DOAP.name, shortname);
175
176                        /* Add properties to the Test Subject Resource */
177                        subjectResource.addProperty(RDF.type, DOAP.Project);
178
179                        if (homepage != null)
180                                subjectResource.addProperty(DOAP.homepage, homepage);
181
182                        if (language != null)
183                                subjectResource
184                                                .addProperty(DOAP.programming_language, language);
185
186                        Map<String, ISuiteResult> tests = suite.getResults();
187
188                        for (ISuiteResult results : tests.values()) {
189                                ITestContext testContext = results.getTestContext();
190                                passedTests = testContext.getPassedTests();
191                                failedTests = testContext.getFailedTests();
192                                skippedTests = testContext.getSkippedTests();
193                                getResultProperties(failedTests, FAIL);
194                                getResultProperties(skippedTests, SKIP);
195                                getResultProperties(passedTests, PASS);
196                        }
197
198                }
199
200        }
201
202        private void getResultProperties(IResultMap tests, String status) {
203                for (ITestResult result : tests.getAllResults()) {
204                        makeResultResource(result, status);
205                }
206        }
207
208        private void makeResultResource(ITestResult result, String status) {
209                String className = result.getTestClass().getName();
210                className = className.substring(className
211                                .lastIndexOf(".") + 1);
212
213                Resource assertionResource = model.createResource(null, Earl.Assertion);
214
215                Resource resultResource = model.createResource(null, Earl.TestResult);
216
217                Resource subjectResource = model.getResource(homepage);
218                Resource assertorResource = model.getResource(assertor);
219
220                assertionResource.addProperty(Earl.testSubject, subjectResource);
221
222                assertionResource.addProperty(
223                                Earl.test,
224                                model.getResource(createTestCaseURL(className, result.getName())));
225
226                /* Test Result Resource */
227                Method method = result.getMethod().getConstructorOrMethod().getMethod();
228                if(method.getAnnotation(SpecTest.class) != null){
229                        SpecTest specTest = method.getAnnotation(SpecTest.class);
230                        if(specTest.coveredByGroups().length > 0 && specTest.coveredByGroups().length > 0){
231                                ArrayList<String> testResults = new ArrayList<String>();
232                                for(Class<?> classVal : specTest.coveredByTests()){
233                                        Method[] classMethod = classVal.getDeclaredMethods();
234                                        for(Method methodName : classMethod) {
235                                                if(methodName.getAnnotation(Test.class) != null) {
236                                                        String group = Arrays.toString(methodName.getAnnotation(Test.class).groups());
237                                                        for(String groupCover : specTest.coveredByGroups()) {
238                                                                if(group.contains(groupCover) && !methodName.getName().contains("Conforms")) {
239                                                                        testResults.add(findTestResult(methodName.getName()));
240                                                                }
241                                                        }
242                                                }
243                                        }
244                                }
245                                // evaluate the testResults for the Indirect Test
246                                // if one of the tests fails, then the entire indirect test fails
247                                // if there are only pass and skipped tests (none failed), then it passes
248                                // if there are only skipped (none passed or failed), then it is skipped
249                                if(testResults.size() > 0){
250                                        if(testResults.contains(FAIL))
251                                                status = FAIL;
252                                        else if(testResults.contains(PASS) && !testResults.contains(FAIL))
253                                                status = PASS;
254                                        else if(testResults.contains(SKIP) && !testResults.contains(FAIL) && !testResults.contains(PASS))
255                                                status = SKIP;
256                                }
257                        }
258                }
259                switch (status) {
260                case FAIL:
261                        resultResource.addProperty(Earl.outcome, Earl.failed);
262                        break;
263                case PASS:
264                        resultResource.addProperty(Earl.outcome, Earl.passed);
265                        break;
266                case SKIP:
267                        resultResource.addProperty(Earl.outcome, Earl.untested);
268                        break;
269                default:
270                        break;
271                }
272
273                if (result.getThrowable() != null) {
274                        createExceptionProperty(result.getThrowable(), resultResource);
275                }
276
277                if (result.getMethod().getConstructorOrMethod().getMethod()
278                                .getAnnotation(SpecTest.class) != null) {
279
280                        SpecTest test = result.getMethod().getConstructorOrMethod()
281                                        .getMethod().getAnnotation(SpecTest.class);
282                        METHOD type = test.testMethod();
283
284                        switch (type) {
285                        case AUTOMATED:
286                                assertionResource.addProperty(Earl.mode, Earl.automatic);
287                                break;
288                        case MANUAL:
289                                assertionResource.addProperty(Earl.mode, Earl.manual);
290                                break;
291                        case INDIRECT:
292                                assertionResource.addProperty(Earl.mode, Earl.automatic);
293                                break;
294                        default:
295                                assertionResource.addProperty(Earl.mode, Earl.notTested);
296                                break;
297                        }
298                }
299
300                assertionResource.addProperty(Earl.assertedBy, assertorResource);
301                assertionResource.addLiteral(ranAsClass, result.getTestClass().getRealClass().getSimpleName());
302
303                resultResource.addProperty(DCTerms.date, model.createTypedLiteral(GregorianCalendar.getInstance()));
304
305                /*
306                 * Add the above resources to the Assertion Resource
307                 */
308                assertionResource.addProperty(Earl.testResult, resultResource);
309
310        }
311
312        private void createExceptionProperty(Throwable thrown, Resource resource) {
313                if (thrown.getClass().getName().contains(SKIP))
314                        resource.addProperty(DCTerms.description, thrown.getMessage());
315                else
316                        resource.addLiteral(DCTerms.description,
317                                        Utils.stackTrace(thrown, false)[0]);
318        }
319
320        private String  findTestResult(String methodName) {
321                Iterator<ITestNGMethod> passed = passedTests.getAllMethods().iterator();
322                while(passed.hasNext()){
323                        ITestNGMethod method = passed.next();
324                        if(method.getMethodName().equals(methodName)){
325                                return PASS;
326                        }
327                }
328
329                Iterator<ITestNGMethod> skipped = skippedTests.getAllMethods().iterator();
330                while(skipped.hasNext()){
331                        ITestNGMethod method = skipped.next();
332                        if(method.getMethodName().equals(methodName)){
333                                return SKIP;
334                        }
335                }
336
337                Iterator<ITestNGMethod> failed = failedTests.getAllMethods().iterator();
338                while(failed.hasNext()){
339                        ITestNGMethod method = failed.next();
340                        if(method.getMethodName().equals(methodName)){
341                                return FAIL;
342                        }
343                }
344
345                return null;
346        }
347
348        @Override
349    protected String getFilename() {
350            return TITLE + "-execution-report-earl";
351    }
352
353        public void setTitle(String title){
354                TITLE = title;
355        }
356}