runtests.py

#!/usr/bin/env python
"""
Test runner for the JSON-LD test-suite.

This module provides a small command-line test harness used to execute the
JSON-LD test manifests that accompany the project and to produce an EARL
(Evaluation and Report Language) report summarizing results.

Behavior and features
- Loads one or more JSON-LD test manifest files (or directories containing a
    `manifest.jsonld`) and constructs a `unittest.TestSuite` from the entries.
- Supports running tests from the local test directories bundled with the
    repository or from manifests passed on the command line.
- Uses the `pyld` library under test to run each test case (expand, compact,
    frame, normalize, to_rdf, etc.), then compares results against expected
    outputs. Comparison is order-insensitive where appropriate.
- Can write an EARL report using `--earl <file>` for CI / interoperability
    with W3C testing tools.

Usage
- Run the script directly: `python tests/runtests.py [MANIFEST_OR_DIR ...]`
- Common options (see `-h` for full list):
    - `-e, --earl <file>` : write an EARL report to `<file>`
    - `-b, --bail`       : stop at the first failing test
    - `-l, --loader`     : choose the network loader (`requests` or `aiohttp`)
    - `-n, --number`     : focus on tests containing the given test identifier
    - `-v, --verbose`    : print verbose test data

Key classes and functions
- `TestRunner`: command-line entrypoint; builds the root manifest and runs the
    test suite.
- `Manifest`: loads a manifest document and converts its entries into
    `unittest.TestSuite` / `Test` instances.
- `Test` (subclass of `unittest.TestCase`): encapsulates execution and
    verification logic for a single JSON-LD test case.
- Utility helpers: `read_json`, `read_file`, `create_document_loader`, and
    `equalUnordered` (used for order-insensitive comparisons).

.. module:: runtests
    :synopsis: Test harness for pyld

.. moduleauthor:: Dave Longley
.. moduleauthor:: Olaf Conradi <olaf@conradi.org>
"""

# TODO: The code below contains a small CLI test-runner (`TestRunner`), a
# unittest-based `EarlTestResult`, and the `__main__` entrypoint. This file
# is still kept to reuse `Manifest`, `Test` and `EarlReport` from the
# original runner, but when running under `pytest` the separate
# `tests/conftest.py` + `tests/test_manifests.py` integration is used to
# drive tests. The `TestRunner` and `EarlTestResult` blocks can be removed
# (or converted to a separate backward-compatibility script) once the
# pytest migration is complete.
# Also, the module docstring can be updated to reflect the pytest-based
# testing approach once the legacy runner is removed.

import datetime
import json
import os
import re
import sys
import traceback
import unittest

# NOTE: ArgumentParser and TextTestResult were used by the original
# TestRunner / EarlTestResult classes. They are obsolete because
# pytest now provides the test harness; these imports can be removed
# once the legacy CLI runner is deleted.
from argparse import ArgumentParser
from unittest import TextTestResult

from typing_extensions import override

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'lib'))
from pyld import iri_resolver, jsonld

__copyright__ = 'Copyright (c) 2011-2013 Digital Bazaar, Inc.'
__license__ = 'New BSD license'

ROOT_MANIFEST_DIR = None
SKIP_TESTS = []
ONLY_IDENTIFIER = None

# `LOCAL_BASES` lists remote bases used by the official JSON-LD test
# repositories. When a test refers to a URL starting with one of these
# bases the runner attempts to map that URL to a local file in the
# test-suite tree (when possible) so tests can be run offline.


LOCAL_BASES = [
    'https://w3c.github.io/json-ld-api/tests',
    'https://w3c.github.io/json-ld-framing/tests',
    'https://github.com/json-ld/normalization/tests',
    'https://w3c.github.io/rdf-canon/tests/vocab#'
]

SPEC_DIRS = [
    '../specifications/json-ld-api/tests/',
    '../specifications/json-ld-framing/tests/',
    '../specifications/normalization/tests/',
    '../specifications/rdf-canon/tests/'
]

# NOTE: The following TestRunner class can be removed because pytest now
# provides the test harness; this class can be removed once the legacy
# CLI runner is deleted.


class TestRunner(unittest.TextTestRunner):
    """
    Loads test manifests and runs tests.
    """

    def __init__(self, stream=sys.stderr, descriptions=True, verbosity=1):
        unittest.TextTestRunner.__init__(self, stream, descriptions, verbosity)

        # The runner uses an ArgumentParser to accept a list of manifests or
        # test directories and several runner-specific flags (e.g. which
        # document loader to use or whether to bail on failure).
        self.options = {}
        self.parser = ArgumentParser()

    @override
    def _makeResult(self):
        return EarlTestResult(self.stream, self.descriptions, self.verbosity)

    def main(self):
        print('PyLD Tests')
        print('Use -h or --help to view options.\\n')

        # add program options
        self.parser.add_argument(
            'tests', metavar='TEST', nargs='*', help='A manifest or directory to test'
        )
        self.parser.add_argument(
            '-e', '--earl', dest='earl', help='The filename to write an EARL report to'
        )
        self.parser.add_argument(
            '-b',
            '--bail',
            dest='bail',
            action='store_true',
            default=False,
            help='Bail out as soon as any test fails',
        )
        self.parser.add_argument(
            '-l',
            '--loader',
            dest='loader',
            default='requests',
            help='The remote URL document loader: requests, aiohttp '
            '[default: %(default)s]',
        )
        self.parser.add_argument(
            '-n',
            '--number',
            dest='number',
            help='Limit tests to those containing the specified test identifier',
        )
        self.parser.add_argument(
            '-v',
            '--verbose',
            dest='verbose',
            action='store_true',
            default=False,
            help='Print verbose test data',
        )

        # parse command line args
        self.options = self.parser.parse_args()

        # Set a default JSON-LD document loader
        if self.options.loader == 'requests':
            jsonld._default_document_loader = jsonld.requests_document_loader()
        elif self.options.loader == 'aiohttp':
            jsonld._default_document_loader = jsonld.aiohttp_document_loader()

        # The document loader drives how remote HTTP documents are fetched.
        # Tests can choose to run using the 'requests' based loader or the
        # 'aiohttp' async loader; here we select the default based on the
        # CLI option.

        # config runner
        self.failfast = self.options.bail

        # Global for saving test numbers to focus on
        global ONLY_IDENTIFIER
        if self.options.number:
            ONLY_IDENTIFIER = self.options.number

        if len(self.options.tests):
            # tests given on command line
            test_targets = self.options.tests
        else:
            # default to find known test suite directories
            test_targets = []
            for dir in SPEC_DIRS:
                if os.path.exists(dir):
                    print('Test dir found', dir)
                    test_targets.append(dir)
                else:
                    print('Test dir not found', dir)

        # ensure a manifest or a directory was specified
        if len(test_targets) == 0:
            raise Exception('No test manifest or directory specified.')

        # make root manifest with target files and dirs
        root_manifest = {
            '@context': 'https://w3c.github.io/tests/context.jsonld',
            '@id': '',
            '@type': 'mf:Manifest',
            'description': 'Top level PyLD test manifest',
            'name': 'PyLD',
            'sequence': [],
            'filename': '/',
        }
        for test in test_targets:
            if os.path.isfile(test):
                root, ext = os.path.splitext(test)
                if ext in ['.json', '.jsonld']:
                    root_manifest['sequence'].append(os.path.abspath(test))
                    # root_manifest['sequence'].append(test)
                else:
                    raise Exception('Unknown test file ext', root, ext)
            elif os.path.isdir(test):
                filename = os.path.join(test, 'manifest.jsonld')
                if os.path.exists(filename):
                    root_manifest['sequence'].append(os.path.abspath(filename))
                else:
                    raise Exception('Manifest not found', filename)
            else:
                raise Exception('Unknown test target.', test)

        # load root manifest
        global ROOT_MANIFEST_DIR
        # ROOT_MANIFEST_DIR = os.path.dirname(root_manifest['filename'])
        ROOT_MANIFEST_DIR = root_manifest['filename']
        # Build a Manifest object from the root manifest structure. The
        # Manifest will recursively load manifests and produce a
        # `unittest.TestSuite` containing all discovered tests.
        suite = Manifest(root_manifest, root_manifest['filename']).load()

        # run tests
        result = self.run(suite)

        # output earl report if specified
        if self.options.earl:
            filename = os.path.abspath(self.options.earl)
            print(f'Writing EARL report to: {filename}')
            result.writeReport(filename)

        if not result.wasSuccessful():
            exit(1)


class Manifest:
    def __init__(self, data, filename):
        self.data = data
        self.suite = unittest.TestSuite()
        self.filename = filename
        self.dirname = os.path.dirname(self.filename)

    def load(self):
        entries = []
        # get entries and sequence (alias for entries)
        entries.extend(get_jsonld_values(self.data, 'entries'))
        entries.extend(get_jsonld_values(self.data, 'sequence'))

        # add includes to entries as jsonld files
        includes = get_jsonld_values(self.data, 'include')
        for filename in includes:
            entries.append(filename + '.jsonld')

        global ONLY_IDENTIFIER

        for entry in entries:
            if isinstance(entry, str):
                filename = os.path.join(self.dirname, entry)
                entry = read_json(filename)
            else:
                filename = self.filename

            # entry is another manifest
            if is_jsonld_type(entry, 'mf:Manifest'):
                self.suite = unittest.TestSuite(
                    [self.suite, Manifest(entry, filename).load()]
                )
            # If the entry is itself a manifest, recurse into it and
            # append its TestSuite. This mirrors the structure of the
            # W3C test manifests where manifests can include other
            # manifests via 'entries' or 'sequence'.
            # don't add tests that are not focused

            # assume entry is a test
            elif not ONLY_IDENTIFIER or ONLY_IDENTIFIER in entry['@id']:
                self.suite.addTest(Test(self, entry, filename))

            # For simple test entries we construct a `Test` object which
            # wraps the execution and assertion logic for that test case.

        return self.suite


class Test(unittest.TestCase):
    """
    # A Test instance stores the manifest and test description (as
    # loaded from the JSON-LD manifest). The boolean flags below are
    # used to distinguish positive/negative/syntax tests so the
    # runner knows whether an exception is the expected outcome.
    """

    def __init__(self, manifest, data, filename):
        unittest.TestCase.__init__(self)
        # self.maxDiff = None
        self.manifest = manifest
        self.data = data
        self.filename = filename
        self.dirname = os.path.dirname(filename)
        self.is_positive = is_jsonld_type(data, 'jld:PositiveEvaluationTest')
        self.is_negative = is_jsonld_type(data, 'jld:NegativeEvaluationTest')
        self.is_syntax = is_jsonld_type(data, 'jld:PositiveSyntaxTest')
        self.test_type = None
        self.pending = False
        global TEST_TYPES
        for t in TEST_TYPES:
            if is_jsonld_type(data, t):
                self.test_type = t
                break

    def __str__(self):
        manifest = self.manifest.data.get('name', self.manifest.data.get('label'))
        test_id = self.data.get('id', self.data.get('@id'))
        label = self.data.get('purpose', self.data.get('name', self.data.get('label')))

        return f'{manifest}: {test_id}: {label}'

    def _get_expect_property(self):
        '''Find the expected output property or raise error.'''
        if 'expect' in self.data:
            return 'expect'
        elif 'result' in self.data:
            return 'result'
        else:
            raise Exception('No expected output property found')

    def _get_expect_error_code_property(self):
        '''Find the expectErrorCode property.'''
        if 'expectErrorCode' in self.data:
            return 'expectErrorCode'
        else:
            raise Exception('No expectErrorCode property found')

    @override
    def setUp(self):
        data = self.data
        manifest = self.manifest
        # skip unknown and explicitly skipped test types
        global SKIP_TESTS
        types = []
        types.extend(get_jsonld_values(data, '@type'))
        types.extend(get_jsonld_values(data, 'type'))
        if self.test_type is None or self.test_type in SKIP_TESTS:
            self.skipTest(f'Test type of {types}')

        global TEST_TYPES
        test_info = TEST_TYPES[self.test_type]

        # expand @id and input base
        if 'baseIri' in manifest.data:
            data['@id'] = (
                manifest.data['baseIri']
                + os.path.basename(str.replace(manifest.filename, '.jsonld', ''))
                + data['@id']
            )
            self.base = self.manifest.data['baseIri'] + data['input']

        # When manifests define a `baseIri` the runner patches the test
        # `@id` and computes a `base` URL used by the document loader so
        # relative references are resolved consistently during testing.

        # skip based on id regular expression
        skip_id_re = test_info.get('skip', {}).get('idRegex', [])
        for regex in skip_id_re:
            if re.match(regex, data.get('@id', data.get('id', ''))):
                self.skipTest(f'Test with id regex {regex}')

        # mark tests as pending, meaning that they are expected to fail
        pending_id_re = test_info.get('pending', {}).get('idRegex', [])
        for regex in pending_id_re:
            if re.match(regex, data.get('@id', data.get('id', ''))):
                self.pending = f'Test with id regex {regex}'

        # skip based on description regular expression
        skip_description_re = test_info.get('skip', {}).get('descriptionRegex', [])
        for regex in skip_description_re:
            if re.match(regex, data.get('description', '')):
                self.skipTest(f'Test with description regex {regex}')

        # skip based on processingMode
        skip_pm = test_info.get('skip', {}).get('processingMode', [])
        data_pm = data.get('option', {}).get('processingMode', None)
        if data_pm in skip_pm:
            self.skipTest(f'Test with processingMode {data_pm}')

        # skip based on specVersion
        skip_sv = test_info.get('skip', {}).get('specVersion', [])
        data_sv = data.get('option', {}).get('specVersion', None)
        if data_sv in skip_sv:
            self.skipTest(f'Test with specVersion {data_sv}')

        # mark tests to run with local loader
        run_remote_re = test_info.get('runLocal', [])
        for regex in run_remote_re:
            if re.match(regex, data.get('@id', data.get('id', ''))):
                data['runLocal'] = True

        # Tests listed under 'runLocal' are forced to use the local file
        # loader variant rather than fetching remote URLs. This is useful
        # for reproducing the official test-suite behavior without network
        # access.

    @override
    def runTest(self):
        data = self.data
        global TEST_TYPES
        test_info = TEST_TYPES[self.test_type]
        fn = test_info['fn']
        params = test_info['params']
        params = [param(self) for param in params]
        result = None
        if self.is_negative:
            expect = data[self._get_expect_error_code_property()]
        elif self.is_syntax:
            expect = None
        else:
            expect = read_test_property(self._get_expect_property())(self)

        # The following try/except handles three primary scenarios:
        # - Positive tests: compute the result and compare to expected JSON
        #   (order-insensitive where appropriate).
        # - Negative tests: assert that the library raises the expected
        #   JSON-LD error code.
        # - Pending tests: tests expected to fail are marked 'pending' and
        #   their unexpected success is reported specially.

        try:
            result = getattr(jsonld, fn)(*params)
            # Invoke the tested pyld function (e.g. `expand`, `compact`,
            # `normalize`). `fn` is the function name and `params` is a
            # list of callables that are invoked with `self` to produce
            # the actual arguments (this allows lazy file loading).
            if self.is_negative and not self.pending:
                raise AssertionError('Expected an error; one was not raised')
            if self.is_syntax and not self.pending:
                self.assertTrue(True)
            elif self.test_type == 'jld:ToRDFTest':
                # avoid normalization when produceGeneralizedRdf is enabled,
                # since the rdflib parser cannot parse blank-node predicates.
                if data.get('option', {}).get('produceGeneralizedRdf'):
                    result = _normalize_generalized_nquads_text(result)
                    expect = _normalize_generalized_nquads_text(expect)
                else:
                    # Test normalized results
                    result = jsonld.normalize(
                        result,
                        {
                            'algorithm': 'RDFC10',
                            'inputFormat': 'application/n-quads',
                            'format': 'application/n-quads',
                        },
                    )
                    expect = jsonld.normalize(
                        expect,
                        {
                            'algorithm': 'RDFC10',
                            'inputFormat': 'application/n-quads',
                            'format': 'application/n-quads',
                        },
                    )
                if _running_under_pytest():
                    assert result == expect
                else:
                    assert_results_equal(result, expect)
            elif not self.is_negative:
                # If the result is a dict and the expected value is a string,
                # the expected value is probably JSON.
                if isinstance(result, dict) and isinstance(expect, str):
                    expect = json.loads(expect)
                # Perform order-independent equivalence test
                if not equal_unordered(result, expect):
                    if _running_under_pytest():
                        assert result == expect
                    else:
                        assert_results_equal(result, expect, json_output=True)
            else:
                if _running_under_pytest():
                    assert result == expect
                else:
                    assert_results_equal(result, expect)
            if self.pending and not self.is_negative:
                raise AssertionError('pending positive test passed')
        except AssertionError as e:
            if (e.args and e.args[0] == 'pending positive test passed') or (
                not self.is_negative and not self.pending
            ):
                raise e
            elif not self.is_negative or (self.is_negative and self.pending):
                print('pending')
            else:
                raise e
        except Exception as e:
            if not self.is_negative and not self.pending:
                print('\n')
                traceback.print_exc(file=sys.stdout)
                raise e
            result = get_jsonld_error_code(e)
            if self.pending and result == expect:
                print('pending negative test passed')
                raise AssertionError('pending negative test passed') from e
            elif self.pending:
                print('pending')
            else:
                # import pdb; pdb.set_trace()
                if _running_under_pytest():
                    assert result == expect
                else:
                    assert_results_equal(result, expect)


def assert_results_equal(result, expect, json_output=False):
    if json_output:
        expect = json.dumps(expect, indent=2)
        result = json.dumps(result, indent=2)
    print('\nEXPECTED: ', expect)
    print('ACTUAL: ', result)
    raise AssertionError('results differ')


def _running_under_pytest():
    return 'pytest' in sys.modules


def _normalize_generalized_nquads_text(nquads):
    """Normalize generalized N-Quads text for test-suite comparisons."""
    # Remove string datatype
    nquads = nquads.replace(
        '^^<http://www.w3.org/2001/XMLSchema#string>  .',
        '.',
    )

    # Re-label blank nodes
    bnodes = {}

    # Generalized N-Quads expected results only need deterministic comparison,
    # not preservation of input blank node labels.
    nquads = re.sub(r'_:([A-Za-z][A-Za-z0-9]*)', lambda m: bnodes.setdefault(m[0], f'_:b{len(bnodes)}'), nquads)
    lines = (re.sub(r'\s*\.$', '.', line.strip()) for line in nquads.splitlines())

    # Sort nquads
    return '\n'.join(sorted(line for line in lines if line))


# Compare values with order-insensitive array tests
def equal_unordered(result, expect):
    """
    `equalUnordered` implements a simple structural equivalence check that
    ignores ordering in lists. It is used to compare JSON-LD results where
    arrays are considered unordered by the test-suite semantics.
    """
    if isinstance(result, list) and isinstance(expect, list):
        return len(result) == len(expect) and all(
            any(equal_unordered(v1, v2) for v2 in expect) for v1 in result
        )
    elif isinstance(result, dict) and isinstance(expect, dict):
        return len(result) == len(expect) and all(
            k in expect and equal_unordered(v, expect[k]) for k, v in result.items()
        )
    else:
        return result == expect


def is_jsonld_type(node, type_):
    node_types = []
    node_types.extend(get_jsonld_values(node, '@type'))
    node_types.extend(get_jsonld_values(node, 'type'))
    types = type_ if isinstance(type_, list) else [type_]
    return len(set(node_types).intersection(set(types))) > 0


def get_jsonld_values(node, property):
    """
    Safely extract a (possibly multi-valued) property from a JSON-LD node.

    The JSON-LD manifests sometimes use single values or lists for the same
    properties. This helper returns a list in either case so callers can
    uniformly iterate over the returned value.

    Args:
        node: dict-like JSON-LD node.
        property: property name to extract (string).

    Returns:
        A list of values for the property (empty list if property missing).
    """
    rval = []
    if property in node:
        rval = node[property]
        if not isinstance(rval, list):
            rval = [rval]
    return rval


def get_jsonld_error_code(err):
    """
    Walk a JsonLdError chain to extract the most specific error `code`.

    Many pyld error types wrap a cause. This helper attempts to return the
    structured `code` attribute from a `jsonld.JsonLdError` (if present),
    otherwise it falls back to stringifying the exception.
    """
    if isinstance(err, jsonld.JsonLdError):
        if err.code:
            return err.code
        elif err.__cause__:
            return get_jsonld_error_code(err.__cause__)
    return str(err)


def read_json(filename):
    """Read and parse a JSON file from `filename`.

    Returns the parsed Python object.
    """
    with open(filename) as f:
        return json.load(f)


def read_file(filename):
    """
    Read a file and return its contents as text.
    """
    with open(filename) as f:
        return f.read()


def read_test_url(property):
    """
    Return a callable that reads a URL-like property from a test entry.

    Some test entries store input locations as relative paths resolved
    against the manifest's `baseIri`. This factory returns a function that
    accepts a `Test` instance and returns the fully-resolved URL (or
    `None` if the property is missing).
    """

    def read(test):
        if property not in test.data:
            return None
        if 'baseIri' in test.manifest.data:
            return test.manifest.data['baseIri'] + test.data[property]
        else:
            return test.data[property]

    return read


def read_test_property(property):
    """
    Return a callable that reads a test-local property and returns either
    parsed JSON (for `.jsonld`) or raw text.

    The returned function accepts a `Test` instance and resolves the
    filename relative to the test's directory. If the file ends with
    `.jsonld` it is parsed as JSON; otherwise the raw file contents are
    returned.
    """

    def read(test):
        if property not in test.data:
            return None
        filename = os.path.join(test.dirname, test.data[property])
        if filename.endswith('.jsonld'):
            return read_json(filename)
        else:
            return read_file(filename)

    return read


def create_test_options(opts=None):
    """
    Factory returning a function that builds options for a pyld API call.

    The returned callable accepts a `Test` instance and produces the
    options dictionary consumed by functions such as `expand`/`compact`.
    It merges explicit test `option` values with any additional `opts`
    passed to the factory, wires in the test-specific `documentLoader`,
    and resolves `expandContext` files when present.
    """

    def create(test):
        http_options = ['contentType', 'httpLink', 'httpStatus', 'redirectTo']
        test_options = test.data.get('option', {})
        options = {}
        options.update(opts or {})
        # Non-http options from manifest get priority over configured options
        for k, v in test_options.items():
            if k not in http_options:
                options[k] = v
        options['documentLoader'] = create_document_loader(test)
        options['hashAlgorithm'] = test.data.get('hashAlgorithm')
        if 'expandContext' in options:
            filename = os.path.join(test.dirname, options['expandContext'])
            options['expandContext'] = read_json(filename)
        return options

    return create


def create_document_loader(test):
    """
    create_document_loader returns a callable compatible with the JSON-LD
    API's document loader interface. The returned `local_loader` will
    decide whether to load a URL from the local test-tree (mapping
    `LOCAL_BASES` to local files) or delegate to the normal network
    loader. This enables deterministic tests without requiring network
    access for suite-hosted resources.
    """
    loader = jsonld.get_document_loader()

    def is_test_suite_url(url):
        return any(url.startswith(base) for base in LOCAL_BASES)

    def strip_base(url):
        for base in LOCAL_BASES:
            if url.startswith(base):
                return url[len(base) :]
        raise Exception('unkonwn base')

    def strip_fragment(url):
        if '#' in url:
            return url[: url.index('#')]
        else:
            return url

    def load_locally(url):
        options = test.data.get('option', {})
        content_type = options.get('contentType')

        url_no_frag = strip_fragment(url)
        if not content_type and url_no_frag.endswith('.jsonld'):
            content_type = 'application/ld+json'
        if not content_type and url_no_frag.endswith('.json'):
            content_type = 'application/json'
        if not content_type and url_no_frag.endswith('.html'):
            content_type = 'text/html'
        if not content_type:
            content_type = 'application/octet-stream'
        doc = {
            'contentType': content_type,
            'contextUrl': None,
            'documentUrl': url,
            'document': None,
        }
        if options and url == test.base:
            if 'redirectTo' in options and options.get('httpStatus') >= 300:
                doc['documentUrl'] = (
                    test.manifest.data['baseIri'] + options['redirectTo']
                )
            elif 'httpLink' in options:
                link_header = options.get('httpLink', '')
                if isinstance(link_header, list):
                    link_header = ','.join(link_header)
                linked_context = jsonld.parse_link_header(link_header).get(
                    'http://www.w3.org/ns/json-ld#context'
                )
                if linked_context and content_type != 'application/ld+json':
                    if isinstance(linked_context, list):
                        raise Exception('multiple context link headers')
                    doc['contextUrl'] = linked_context['target']
                linked_alternate = jsonld.parse_link_header(link_header).get(
                    'alternate'
                )
                # if not JSON-LD, alternate may point there
                if (
                    linked_alternate
                    and linked_alternate.get('type') == 'application/ld+json'
                    and not re.match(r'^application\/(\w*\+)?json$', content_type)
                ):
                    doc['contentType'] = 'application/ld+json'
                    doc['documentUrl'] = iri_resolver.resolve(
                        linked_alternate['target'], url
                    )
        global ROOT_MANIFEST_DIR
        if doc['documentUrl'].find(':') == -1:
            filename = os.path.join(ROOT_MANIFEST_DIR, doc['documentUrl'])
            doc['documentUrl'] = 'file://' + filename
        else:
            filename = test.dirname + strip_fragment(strip_base(doc['documentUrl']))
        try:
            doc['document'] = read_file(filename)
        except Exception as e:
            raise Exception('loading document failed') from e
        return doc

    def local_loader(url, headers):
        # always load remote-doc tests remotely
        # (some skipped due to lack of reasonable HTTP header support)
        if test.manifest.data.get('name') == 'Remote document' and not test.data.get(
            'runLocal'
        ):
            return loader(url)

        # always load non-base tests remotely
        if not is_test_suite_url(url) and url.find(':') != -1:
            return loader(url)

        # attempt to load locally
        return load_locally(url)

    return local_loader


# NOTE: The EarlTestResult class can be removed because pytest now
# provides the test harness; this class can be removed once the legacy
# CLI runner is deleted.
class EarlTestResult(TextTestResult):
    """
    A `TextTestResult` subclass that records EARL assertions as tests run.

    This result object forwards normal test outcome bookkeeping to the
    base `TextTestResult` and additionally records each assertion in an
    `EarlReport` instance so a machine-readable report can be emitted at
    the end of a test run.
    """

    def __init__(self, stream, descriptions, verbosity):
        TextTestResult.__init__(self, stream, descriptions, verbosity)
        self.report = EarlReport()

    @override
    def addError(self, test, err):
        TextTestResult.addError(self, test, err)
        self.report.add_assertion(test, False)

    @override
    def addFailure(self, test, err):
        TextTestResult.addFailure(self, test, err)
        self.report.add_assertion(test, False)

    @override
    def addSuccess(self, test):
        TextTestResult.addSuccess(self, test)
        self.report.add_assertion(test, True)

    @override
    def writeReport(self, filename):
        self.report.write(filename)

class EarlReport:
    """
    Generates an EARL report.
    """

    def __init__(self):
        # Load package metadata (version) from the library's __about__.py
        about = {}
        with open(
            os.path.join(os.path.dirname(__file__), '..', 'lib', 'pyld', '__about__.py')
        ) as fp:
            exec(fp.read(), about)
        # Timestamp used for test results
        self.now = datetime.datetime.utcnow().replace(microsecond=0)
        # Build the base EARL report structure. The report is a JSON-LD
        # document describing the project and the assertions made about
        # test outcomes.
        self.report = {
            '@context': {
                'doap': 'http://usefulinc.com/ns/doap#',
                'foaf': 'http://xmlns.com/foaf/0.1/',
                'dc': 'http://purl.org/dc/terms/',
                'earl': 'http://www.w3.org/ns/earl#',
                'xsd': 'http://www.w3.org/2001/XMLSchema#',
                'doap:homepage': {'@type': '@id'},
                'doap:license': {'@type': '@id'},
                'dc:creator': {'@type': '@id'},
                'foaf:homepage': {'@type': '@id'},
                'subjectOf': {'@reverse': 'earl:subject'},
                'earl:assertedBy': {'@type': '@id'},
                'earl:mode': {'@type': '@id'},
                'earl:test': {'@type': '@id'},
                'earl:outcome': {'@type': '@id'},
                'dc:date': {'@type': 'xsd:date'},
                'doap:created': {'@type': 'xsd:date'},
            },
            '@id': 'https://github.com/digitalbazaar/pyld',
            '@type': ['doap:Project', 'earl:TestSubject', 'earl:Software'],
            'doap:name': 'PyLD',
            'dc:title': 'PyLD',
            'doap:homepage': 'https://github.com/digitalbazaar/pyld',
            'doap:license': 'https://github.com/digitalbazaar/pyld/blob/master/LICENSE',
            'doap:description': {
                '@value': 'A JSON-LD processor for Python',
                '@language': 'en',
            },
            'doap:programming-language': 'Python',
            'dc:creator': 'https://github.com/dlongley',
            'doap:developer': {
                '@id': 'https://github.com/dlongley',
                '@type': ['foaf:Person', 'earl:Assertor'],
                'foaf:name': 'Dave Longley',
                'foaf:homepage': 'https://github.com/dlongley',
            },
            'doap:release': {
                'doap:name': 'PyLD ' + about['__version__'],
                'doap:revision': about['__version__'],
                'doap:created': self.now.strftime('%Y-%m-%d'),
            },
            'subjectOf': [],
        }

    def add_assertion(self, test, success):
        # Append an EARL assertion describing a single test outcome. The
        # `earl:outcome` is either `earl:passed` or `earl:failed`.
        self.report['subjectOf'].append(
            {
                '@type': 'earl:Assertion',
                'earl:assertedBy': self.report['doap:developer']['@id'],
                'earl:mode': 'earl:automatic',
                'earl:test': test.data.get('id', test.data.get('@id')),
                'earl:result': {
                    '@type': 'earl:TestResult',
                    'dc:date': self.now.isoformat() + 'Z',
                    'earl:outcome': 'earl:passed' if success else 'earl:failed',
                },
            }
        )
        return self

    def write(self, filename):
        # Serialize the EARL report as pretty-printed JSON-LD.
        with open(filename, 'w') as f:
            f.write(json.dumps(self.report, indent=2))
            f.close()


# supported test types
TEST_TYPES = {
    'jld:CompactTest': {
        'pending': {},
        'skip': {
            # skip tests where behavior changed for a 1.1 processor
            # see JSON-LD 1.0 Errata
            'specVersion': ['json-ld-1.0'],
        },
        'fn': 'compact',
        'params': [
            read_test_url('input'),
            read_test_property('context'),
            create_test_options(),
        ],
    },
    'jld:ExpandTest': {
        'pending': {},
        'runLocal': [
            '.*remote-doc-manifest#t0003$',
            '.*remote-doc-manifest#t0004$',
            '.*remote-doc-manifest#t0005$',
            '.*remote-doc-manifest#t0006$',
            '.*remote-doc-manifest#t0007$',
            '.*remote-doc-manifest#t0009$',
            '.*remote-doc-manifest#t0010$',
            '.*remote-doc-manifest#t0011$',
            '.*remote-doc-manifest#t0012$',
            '.*remote-doc-manifest#t0013$',
            '.*remote-doc-manifest#tla01$',
            '.*remote-doc-manifest#tla02$',
            '.*remote-doc-manifest#tla03$',
            '.*remote-doc-manifest#tla04$',
            '.*remote-doc-manifest#tla05$',
        ],
        'skip': {
            # skip tests where behavior changed for a 1.1 processor
            # see JSON-LD 1.0 Errata
            'specVersion': ['json-ld-1.0'],
        },
        'fn': 'expand',
        'params': [read_test_url('input'), create_test_options()],
    },
    'jld:FlattenTest': {
        'pending': {},
        'skip': {
            # skip tests where behavior changed for a 1.1 processor
            # see JSON-LD 1.0 Errata
            'specVersion': ['json-ld-1.0'],
        },
        'fn': 'flatten',
        'params': [
            read_test_url('input'),
            read_test_property('context'),
            create_test_options({"extractAllScripts": False}),
        ],
    },
    'jld:FrameTest': {
        'pending': {},
        'skip': {
            # skip tests where behavior changed for a 1.1 processor
            # see JSON-LD 1.0 Errata
            'specVersion': ['json-ld-1.0'],
        },
        'fn': 'frame',
        'params': [
            read_test_url('input'),
            read_test_property('frame'),
            create_test_options(),
        ],
    },
    'jld:FromRDFTest': {
        'skip': {
            'specVersion': ['json-ld-1.0'],
        },
        'fn': 'from_rdf',
        'params': [
            read_test_property('input'),
            create_test_options({'format': 'application/n-quads'}),
        ],
    },
    'jld:NormalizeTest': {
        'skip': {},
        'fn': 'normalize',
        'params': [
            read_test_property('input'),
            create_test_options({'format': 'application/n-quads'}),
        ],
    },
    'jld:ToRDFTest': {
        'pending': {},
        'skip': {
            # skip tests where behavior changed for a 1.1 processor
            # see JSON-LD 1.0 Errata
            'specVersion': ['json-ld-1.0'],
        },
        'fn': 'to_rdf',
        'params': [
            read_test_url('input'),
            create_test_options({'format': 'application/n-quads'}),
        ],
    },
    'rdfn:Urgna2012EvalTest': {
        'pending': {},
        'skip': {},
        'fn': 'normalize',
        'params': [
            read_test_property('action'),
            create_test_options(
                {
                    'algorithm': 'URGNA2012',
                    'inputFormat': 'application/n-quads',
                    'format': 'application/n-quads',
                }
            ),
        ],
    },
    'rdfn:Urdna2015EvalTest': {
        'pending': {},
        'skip': {},
        'fn': 'normalize',
        'params': [
            read_test_property('action'),
            create_test_options(
                {
                    'algorithm': 'URDNA2015',
                    'inputFormat': 'application/n-quads',
                    'format': 'application/n-quads',
                }
            ),
        ],
    },
    'rdfc:RDFC10EvalTest': {
        'pending': {},
        'skip': {},
        'fn': 'normalize',
        'params': [
            read_test_property('action'),
            create_test_options({
                'algorithm': 'RDFC10',
                'inputFormat': 'application/n-quads',
                'format': 'application/n-quads'
            })
        ]
    },
    'rdfc:RDFC10MapTest': {
        'pending': {},
        'skip': {},
        'fn': 'normalize',
        'params': [
            read_test_property('action'),
            create_test_options({
                'algorithm': 'RDFC10',
                'inputFormat': 'application/n-quads',
                'format': 'application/n-quads',
                'outputMap': True
            })
        ]
    }
}


# NOTE: The legacy command-line entrypoint can be removed because pytest is used to
# run the test-suite. Keep this here for reference; it can be removed once
# the pytest migration is finalized.
if __name__ == '__main__':
    TestRunner(verbosity=2).main()