001package org.w3.ldp.testsuite;
002
003import java.io.File;
004import java.net.URI;
005import java.net.URISyntaxException;
006import java.util.*;
007import java.util.regex.Pattern;
008
009import org.apache.commons.cli.BasicParser;
010import org.apache.commons.cli.CommandLine;
011import org.apache.commons.cli.CommandLineParser;
012import org.apache.commons.cli.HelpFormatter;
013import org.apache.commons.cli.Option;
014import org.apache.commons.cli.OptionBuilder;
015import org.apache.commons.cli.OptionGroup;
016import org.apache.commons.cli.Options;
017import org.apache.commons.cli.ParseException;
018import org.apache.commons.lang3.StringUtils;
019import org.apache.commons.lang3.exception.ExceptionUtils;
020import org.testng.IMethodInstance;
021import org.testng.IMethodInterceptor;
022import org.testng.ITestContext;
023import org.testng.TestNG;
024import org.testng.xml.XmlClass;
025import org.testng.xml.XmlSuite;
026import org.testng.xml.XmlTest;
027import org.w3.ldp.testsuite.reporter.LdpEarlReporter;
028import org.w3.ldp.testsuite.reporter.LdpHtmlReporter;
029import org.w3.ldp.testsuite.reporter.LdpTestListener;
030import org.w3.ldp.testsuite.test.LdpTest;
031import org.w3.ldp.testsuite.transformer.MethodEnabler;
032import org.w3.ldp.testsuite.util.OptionsHandler;
033
034import com.jayway.restassured.RestAssured;
035
036/**
037 * LDP Test Suite Command-Line Interface, a wrapper to {@link org.testng.TestNG}
038 *
039 * @author Sergio Fernández
040 * @author Steve Speicher
041 * @author Samuel Padgett
042 */
043public class LdpTestSuite {
044
045        public static final String NAME = "LDP Test Suite";
046        public static final String SPEC_URI = "http://www.w3.org/TR/ldp";
047        public static final String OUTPUT_DIR = "report";
048
049        static final String[] EARLDEPEDENTARGS = {"software", "developer", "language", "homepage", "assertor", "shortname"};
050
051        private final TestNG testng;
052
053        private final List<XmlClass> classList; // for test types to add in
054
055        private final String reportTitle;
056
057        private String outputDir;
058
059        enum ContainerType {
060                BASIC, DIRECT, INDIRECT
061        }
062
063        /**
064         * Initialize the test suite with options as a map
065         *
066         * @param options map of options
067         */
068        public LdpTestSuite(final Map<String, String> options) {
069                this(new OptionsHandler(options));
070        }
071
072        /**
073         * Initialize the test suite with options as the row command-line input
074         *
075         * @param cmd command-line options
076         */
077        public LdpTestSuite(final CommandLine cmd) {
078                this(new OptionsHandler(cmd));
079        }
080
081        /**
082         * Initialize the test suite with options as the row command-line input
083         *
084         * @param cmd command-line options
085         * @param reportTitle report title
086         */
087        public LdpTestSuite(final CommandLine cmd, String reportTitle) {
088                this(new OptionsHandler(cmd), reportTitle);
089        }
090
091        private LdpTestSuite(OptionsHandler optionsHandler) {
092                this(optionsHandler, null);
093        }
094
095        private LdpTestSuite(OptionsHandler optionsHandler, String reportTitle) {
096                // see: http://testng.org/doc/documentation-main.html#running-testng-programmatically
097                testng = new TestNG();
098                this.reportTitle = reportTitle;
099                this.outputDir = OUTPUT_DIR;
100                this.classList = new ArrayList<>();
101                this.setupSuite(optionsHandler);
102        }
103
104        public void checkUriScheme(String uri) throws URISyntaxException {
105                String scheme = new URI(uri).getScheme();
106                if (!"http".equals(scheme) && !"https".equals(scheme)) {
107                        throw new IllegalArgumentException("non-http uri");
108                }
109        }
110
111        private void setupSuite(OptionsHandler options) {
112
113                testng.setDefaultSuiteName(NAME);
114
115                // create XmlSuite instance
116                XmlSuite testsuite = new XmlSuite();
117                testsuite.setName(NAME);
118
119                // provide included/excluded groups
120                // get groups to include
121                final String[] includedGroups;
122                if(options.hasOption("includedGroups")) {
123                        includedGroups = options.getOptionValues("includedGroups");
124                        for(String group : includedGroups){
125                                testsuite.addIncludedGroup(group);
126                        }
127                } else{
128                        testsuite.addIncludedGroup(LdpTest.MUST);
129                        testsuite.addIncludedGroup(LdpTest.SHOULD);
130                        testsuite.addIncludedGroup(LdpTest.MAY);
131                }
132                // get groups to exclude
133                final String[] excludedGroups;
134                if(options.hasOption("excludedGroups")){
135                        excludedGroups = options.getOptionValues("excludedGroups");
136                        for(String group : excludedGroups){
137                                testsuite.addExcludedGroup(group);
138                        }
139                }
140
141                // create XmlTest instance
142                XmlTest test = new XmlTest(testsuite);
143                test.setName("W3C Linked Data Platform Tests");
144
145                // Add any parameters that you want to set to the Test.
146                // Test suite parameters
147                final Map<String, String> parameters = new HashMap<>();
148
149                if (options.hasOption("output")) {
150                        final String output = options.getOptionValue("output");
151                        outputDir = output + File.separator + LdpTestSuite.OUTPUT_DIR;
152                        testng.setOutputDirectory(output + File.separator + TestNG.DEFAULT_OUTPUTDIR);
153                }
154                parameters.put("output", outputDir);
155
156                final String server;
157                if (options.hasOption("server")) {
158                        server = options.getOptionValue("server");
159                        if (StringUtils.startsWith(server, "https:")) { // allow self-signed certificates for development servers
160                                RestAssured.useRelaxedHTTPSValidation();
161                        }
162                        try {
163                                checkUriScheme(server);
164                        } catch (Exception e) {
165                                throw new IllegalArgumentException("ERROR: invalid server uri, " + e.getLocalizedMessage());
166                        }
167                } else {
168                        throw new IllegalArgumentException("ERROR: missing server uri");
169                }
170
171                // Listener injection from options
172                final String[] listeners;
173                if (options.hasOption("listeners")) {
174                        listeners = options.getOptionValues("listeners");
175                        for (String listener : listeners) {
176                                try {
177                                        Class<?> listenerCl = Class.forName(listener);
178                                        Object instance = listenerCl.newInstance();
179                                        testng.addListener(instance);
180                                } catch (ClassNotFoundException e) {
181                                        throw new IllegalArgumentException("ERROR: invalid listener class name, " + e.getLocalizedMessage());
182                                } catch (InstantiationException | IllegalAccessException e) {
183                                        throw new IllegalArgumentException("ERROR: problem while creating listener, " + e.getLocalizedMessage());
184                                }
185                        }
186                }
187
188                // Add method enabler (Annotation Transformer)
189                testng.addListener(new MethodEnabler());
190
191                testng.addListener(new LdpTestListener());
192                LdpHtmlReporter reporter = new LdpHtmlReporter();
193                if (StringUtils.isNotBlank(reportTitle)) {
194                        reporter.setTitle(reportTitle);
195                }
196                reporter.setOutputDirectory(outputDir);
197                testng.addListener(reporter);
198
199                if (options.hasOption("earl")) {
200                        LdpEarlReporter earlReport = new LdpEarlReporter();
201                        if (StringUtils.isNotBlank(reportTitle)) {
202                                earlReport.setTitle(reportTitle);
203                        }
204                        earlReport.setOutputDirectory(outputDir);
205                        testng.addListener(earlReport);
206
207                        // required --earl args
208                        for (String arg: EARLDEPEDENTARGS) {
209                                if (options.hasOptionWithValue(arg))
210                                        parameters.put(arg, options.getOptionValue(arg));
211                                else
212                                        printEarlUsage(arg);
213                        }
214
215                        // optional --earl args
216                        if (options.hasOptionWithValue("mbox")) {
217                                parameters.put("mbox", options.getOptionValue("mbox"));
218                        }
219
220                }
221
222                if (options.hasOptionWithValue("cont-res")) {
223                        final String containerAsResource = options.getOptionValue("cont-res");
224                        try {
225                                checkUriScheme(containerAsResource);
226                        } catch (Exception e) {
227                                throw new IllegalArgumentException("ERROR: invalid containerAsResource uri, " + e.getLocalizedMessage());
228                        }
229                        parameters.put("containerAsResource", containerAsResource);
230                }
231
232                if (options.hasOption("read-only-prop")) {
233                        parameters.put("readOnlyProp", options.getOptionValue("read-only-prop"));
234                }
235
236                if (options.hasOption("relative-uri")) {
237                        parameters.put("relativeUri", options.getOptionValue("relative-uri"));
238                }
239
240                if (options.hasOptionWithValue("auth")) {
241                        final String auth = options.getOptionValue("auth");
242                        if (auth.contains(":")) {
243                                        String[] split = auth.split(":");
244                                        if (split.length == 2 && StringUtils.isNotBlank(split[0]) && StringUtils.isNotBlank(split[1])) {
245                                                parameters.put("auth", auth);
246                                } else {
247                                        throw new IllegalArgumentException("ERROR: invalid basic authentication credentials");
248                                }
249                        } else {
250                                throw new IllegalArgumentException("ERROR: invalid basic authentication credentials");
251                        }
252                }
253
254                final String postTtl;
255                if (options.hasOption("postTtl")) {
256                        postTtl = options.getOptionValue("postTtl");
257                        parameters.put("postTtl", postTtl);
258                }
259
260                final String memberTtl;
261                if (options.hasOption("memberTtl")) {
262                        memberTtl = options.getOptionValue("memberTtl");
263                        parameters.put("memberTtl", memberTtl);
264                }
265
266                final String memberResource;
267                if (options.hasOption("memberResource")) {
268                        memberResource = options.getOptionValue("memberResource");
269                        parameters.put("memberResource", memberResource);
270                }
271
272                ContainerType type = getSelectedType(options);
273                switch (type) {
274                        case BASIC:
275                                classList.add(new XmlClass("org.w3.ldp.testsuite.test.BasicContainerTest"));
276                                parameters.put("basicContainer", server);
277                                break;
278                        case DIRECT:
279                                classList.add(new XmlClass("org.w3.ldp.testsuite.test.DirectContainerTest"));
280                                parameters.put("directContainer", server);
281                                break;
282                        case INDIRECT:
283                                classList.add(new XmlClass("org.w3.ldp.testsuite.test.IndirectContainerTest"));
284                                parameters.put("indirectContainer", server);
285                                break;
286                }
287
288                classList.add(new XmlClass("org.w3.ldp.testsuite.test.MemberResourceTest"));
289                testsuite.addIncludedGroup("ldpMember");
290
291                if (options.hasOption("non-rdf")) {
292                        classList.add(new XmlClass("org.w3.ldp.testsuite.test.NonRDFSourceTest"));
293                }
294
295                if (options.hasOption("httpLogging")) {
296                        parameters.put("httpLogging", "true");
297                }
298
299                if (options.hasOption("skipLogging")) {
300                        parameters.put("skipLogging", "true");
301                }
302
303                test.setXmlClasses(classList);
304
305                final List<XmlTest> tests = new ArrayList<>();
306                tests.add(test);
307
308                testsuite.setParameters(parameters);
309                testsuite.setTests(tests);
310
311                final List<XmlSuite> suites = new ArrayList<>();
312                suites.add(testsuite);
313                testng.setXmlSuites(suites);
314
315                if (options.hasOption("test")) {
316                        final String[] testNamePatterns = options.getOptionValues("test");
317                        for (int i = 0; i < testNamePatterns.length; i++) {
318                                // We support only * as a wildcard character to keep the command line simple.
319                                // Convert the wildcard pattern into a regex to use internally.
320                                testNamePatterns[i] = wildcardPatternToRegex(testNamePatterns[i]);
321                        }
322
323                        // Add a method intercepter to filter the list for matching tests.
324                        testng.addListener(new IMethodInterceptor() {
325                                @Override
326                                public List<IMethodInstance> intercept(List<IMethodInstance> methods, ITestContext context) {
327                                        ArrayList<IMethodInstance> toRun = new ArrayList<>();
328                                        for (IMethodInstance method : methods) {
329                                                for (String testNamePattern : testNamePatterns) {
330                                                        if (method.getMethod().getMethodName().matches(testNamePattern)) {
331                                                                toRun.add(method);
332                                                        }
333                                                }
334                                        }
335                                        return toRun;
336                                }
337                        });
338                }
339        }
340
341        private ContainerType getSelectedType(OptionsHandler options) {
342                if (options.hasOption("direct")) {
343                        return ContainerType.DIRECT;
344                } else if (options.hasOption("indirect")) {
345                        return ContainerType.INDIRECT;
346                } else {
347                        return ContainerType.BASIC;
348                }
349        }
350
351        public void addTestClass(String klass) {
352                addTestClass(new XmlClass(klass));
353        }
354
355        public void addTestClass(XmlClass klass) {
356                this.classList.add(klass);
357        }
358
359        public void addTestClasses(Collection<XmlClass> classes) {
360                this.classList.addAll(classes);
361        }
362
363        public String wildcardPatternToRegex(String wildcardPattern) {
364                // use lookarounds and zero-width matches to include the * delimeter in the result
365                String[] tokens = wildcardPattern.split("(?<=\\*)|(?=\\*)");
366                StringBuilder builder = new StringBuilder();
367                for (String token : tokens) {
368                        if ("*".equals(token)) {
369                                builder.append(".*");
370                        } else if (token.length() > 0) {
371                                builder.append(Pattern.quote(token));
372                        }
373                }
374
375                return builder.toString();
376        }
377
378        public void run() {
379                testng.run();
380        }
381
382        public int getStatus() {
383                return testng.getStatus();
384        }
385
386        public String getOutputDir() {
387                return outputDir;
388        }
389
390        public static CommandLine getCommandLine(Options options, String[] args){
391                CommandLineParser parser = new BasicParser();
392                CommandLine cmd = null;
393                try {
394                        cmd = parser.parse(options, args);
395                } catch (ParseException e) {
396                        System.err.println("ERROR: " + e.getLocalizedMessage());
397                        printUsage(options);
398                }
399
400                if (cmd.hasOption("help")) {
401                        printUsage(options);
402                }
403                return cmd;
404        }
405
406        public static void executeTestSuite(String[] args, Options options, String reportTitle) {
407                executeTestSuite(args, options, reportTitle, Collections.<XmlClass>emptyList());
408        }
409        public static void executeTestSuite(String[] args, Options options, String reportTitle, List<XmlClass> classes) {
410                // actual test suite execution
411                try {
412                        CommandLine cmd = LdpTestSuite.getCommandLine(options, args);
413                        LdpTestSuite ldpTestSuite = new LdpTestSuite(cmd, reportTitle);
414                        ldpTestSuite.addTestClasses(classes);
415                        ldpTestSuite.run();
416                        System.exit(ldpTestSuite.getStatus());
417                } catch (Exception e) {
418                        e.printStackTrace();
419                        Throwable cause = ExceptionUtils.getRootCause(e);
420                        System.err.println("ERROR: " + (cause != null ? cause.getMessage() : e.getMessage()));
421                        printUsage(options);
422                }
423        }
424
425
426        private static void printUsage(Options options) {
427                HelpFormatter formatter = new HelpFormatter();
428                formatter.setOptionComparator(new Comparator<Option>() {
429                        @Override
430                        public int compare(Option o1, Option o2) {
431                                if ("server".equals(o1.getLongOpt())) {
432                                        return -10000;
433                                } else if ("help".equals(o1.getLongOpt())) {
434                                        return 10000;
435                                } else {
436                                        return o1.getLongOpt().compareTo(o2.getLongOpt());
437                                }
438                        }
439                });
440                System.out.println();
441                formatter.printHelp("java -jar ldp-testsuite.jar", options);
442                System.out.println();
443                System.exit(-1);
444        }
445
446        private static void printEarlUsage(String missingArg) {
447                System.out.println("--earl missing arg: "+missingArg);
448                System.out.println("Required additional args:");
449                for (String arg: EARLDEPEDENTARGS) {
450                        System.out.println("\t--"+arg);
451                }
452                System.exit(1);
453        }
454
455        @SuppressWarnings("static-access")
456        public static OptionGroup addCommonOptions() {
457                OptionGroup common = new OptionGroup();
458                common.addOption(OptionBuilder.withLongOpt("server")
459                                .withDescription("server url to run the test suite").hasArg()
460                                .withArgName("server").isRequired().create());
461
462                common.addOption(OptionBuilder.withLongOpt("auth")
463                                .withDescription("server basic authentication credentials following the syntax username:password").hasArg()
464                                .withArgName("username:password").create());
465
466                common.addOption(OptionBuilder.withLongOpt("earl")
467                                .withDescription("General EARL ttl file").withArgName("earl")
468                                .isRequired(false).create());
469
470                common.addOption(OptionBuilder.withLongOpt("includedGroups")
471                                .withDescription("test groups to run, separated by a space").hasArgs()
472                                .withArgName("includedGroups").isRequired(false)
473                                .create());
474
475                common.addOption(OptionBuilder.withLongOpt("excludedGroups")
476                                .withDescription("test groups to not run, separated by a space").hasArgs()
477                                .withArgName("excludedGroups").isRequired(false)
478                                .create());
479
480                common.addOption(OptionBuilder.withLongOpt("test")
481                                .withDescription("which tests to run (* is a wildcard)")
482                                .hasArgs().withArgName("test names")
483                                .create());
484
485                common.addOption(OptionBuilder.withLongOpt("output")
486                                .withDescription("output directory (current directory by default)")
487                                .hasArgs().withArgName("output")
488                                .isRequired(false).create());
489
490                common.addOption(OptionBuilder.withLongOpt("httpLogging")
491                                .withDescription("log HTTP requests and responses on validation failures")
492                                .isRequired(false).create());
493
494                common.addOption(OptionBuilder.withLongOpt("skipLogging")
495                                .withDescription("log skip test messages")
496                                .isRequired(false).create());
497
498                common.addOption(OptionBuilder.withLongOpt("help")
499                                .withDescription("prints this usage help")
500                                .isRequired(false).create());
501                return common;
502        }
503
504        @SuppressWarnings("static-access")
505        public static OptionGroup addEarlOptions() {
506                OptionGroup earl = new OptionGroup();
507                // --earl dependent values
508                earl.addOption(OptionBuilder
509                                .withLongOpt("software")
510                                .withDescription(
511                                                "title of the software test suite runs on: required with --earl")
512                                .hasArg().withArgName("software").isRequired(false).create());
513
514                earl.addOption(OptionBuilder
515                                .withLongOpt("developer")
516                                .withDescription(
517                                                "the name of the software developer: required with --earl")
518                                .hasArg().withArgName("dev-name").isRequired(false).create());
519
520                earl.addOption(OptionBuilder
521                                .withLongOpt("mbox")
522                                .withDescription(
523                                                "email of the sofware developer: optional with --earl")
524                                .hasArg().withArgName("mbox").isRequired(false).create());
525
526                earl.addOption(OptionBuilder
527                                .withLongOpt("language")
528                                .withDescription(
529                                                "primary programming language of the software: required with --earl")
530                                .hasArg().withArgName("language").isRequired(false).create());
531
532                earl.addOption(OptionBuilder
533                                .withLongOpt("homepage")
534                                .withDescription(
535                                                "the homepage of the suite runs against: required with --earl")
536                                .hasArg().withArgName("homepage").isRequired(false).create());
537
538                earl.addOption(OptionBuilder
539                                .withLongOpt("assertor")
540                                .withDescription(
541                                                "the URL of the person or agent that asserts the results: required with --earl")
542                                .hasArg().withArgName("assertor").isRequired(false).create());
543
544                earl.addOption(OptionBuilder.withLongOpt("shortname")
545                                .withDescription("a simple short name: required with --earl")
546                                .hasArg().withArgName("shortname").isRequired(false).create());
547                // end of --earl dependent values
548                return earl;
549        }
550
551}