path: root/ganarchy/data.py

# This file is part of GAnarchy - decentralized project hub
# Copyright (C) 2019, 2020  Soni L.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

"""This module handles GAnarchy's data and config sources.

A data source can be either a config source or a repo list source, but be
careful: they use identical syntax, but have different semantics! Mistaking
a repo list source for a config source is a recipe for security bugs!
"""

import abc
import itertools
import os
import re
import time

import abdl
import abdl.exceptions
import qtoml
import requests

from enum import Enum
from urllib.parse import urlparse

import ganarchy.dirs

# TODO move elsewhere
class URIPredicate(abdl.predicates.Predicate):
    def __init__(self, ports=range(1,65536), schemes=('https',)):
        self.ports = ports
        self.schemes = schemes

    def accept(self, obj):
        try:
            u = urlparse(obj)
            if not u:
                return False
            # also raises for invalid ports, see https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlparse
            # "Reading the port attribute will raise a ValueError if an invalid port is specified in the URL. [...]"
            if u.port is not None and u.port not in self.ports:
                return False
            if u.scheme not in self.schemes:
                return False
        except ValueError:
            return False
        return True

class CommitPredicate(abdl.predicates.Predicate):
    def __init__(self, sha256ready=True):
        if sha256ready:
            self.re = re.compile(r"^[0-9a-fA-F]{40}$|^[0-9a-fA-F]{64}$")
        else:
            self.re = re.compile(r"^[0-9a-fA-F]{40}$")

    def accept(self, obj):
        return self.re.match(obj)

# sanitize = skip invalid entries
# validate = error on invalid entries
# LEGACY. DO NOT USE.
CONFIG_REPOS_SANITIZE = abdl.compile("""->'projects'?:?$dict
                                          ->commit[:?$str:?$commit]:?$dict
                                            ->url[:?$str:?$uri]:?$dict
                                              ->branch:?$dict(->'active'?:?$bool)""",
                                     dict(bool=bool, dict=dict, str=str, uri=URIPredicate(), commit=CommitPredicate()))

CONFIG_TITLE_SANITIZE = abdl.compile("""->title'title'?:?$str""", dict(str=str))
CONFIG_BASE_URL_SANITIZE = abdl.compile("""->base_url'base_url'?:?$str:?$uri""", dict(str=str, uri=URIPredicate()))

# modern matchers, raise ValidationError if the data doesn't exist.
# they still skip "bad" entries, just like the old matchers.

_MATCHER_REPOS = abdl.compile("""->'projects':$dict
                                   ->commit[:?$str:?$commit]:?$dict
                                     ->url[:?$str:?$uri]:?$dict
                                       ->branch:?$dict(->'active'?:?$bool)""",
                              dict(bool=bool, dict=dict, str=str, uri=URIPredicate(), commit=CommitPredicate()))
_MATCHER_REPO_LIST_SRCS = abdl.compile("""->'repo_list_srcs':$dict
                                            ->src[:?$str:?$uri]:?$dict
                                              (->'active'?:?$bool)""",
                                       dict(bool=bool, list=list, dict=dict, str=str, uri=URIPredicate(schemes=('https','file',))))
# TODO
#_MATCHER_ALIASES = abdl.compile("""->'project_settings':$dict
#                                     ->commit/[0-9a-fA-F]{40}|[0-9a-fA-F]{64}/?:?$dict
#                                       """, {'dict': dict}) # FIXME check for aliases, might require changes to abdl

# TODO
#_MATCHER_URI_FILTERS = abdl.compile("""->'uri_filters':$dict
#                                         ->filter[:?$str]:?$dict
#                                           (->'active'?:?$bool)""",
#                                    dict(dict=dict, str=str, bool=bool))

_MATCHER_TITLE = abdl.compile("""->title'title':$str""", dict(str=str))
_MATCHER_BASE_URL = abdl.compile("""->base_url'base_url':$str:$uri""", dict(str=str, uri=URIPredicate()))

class OverridableProperty(abc.ABC):
    """An overridable property, with options.

    Attributes:
        options (dict): Options.
    """

    @abc.abstractmethod
    def as_key(self):
        """Returns an opaque representation of this OverridablePRoperty
        suitable for use as a dict key.

        The returned object is not suitable for other purposes.
        """
        return ()

    @property
    def active(self):
        """Whether this property is active.
        """
        return self.options.get('active', False)

class PCTP(OverridableProperty):
    """A Project Commit-Tree Path.

    Attributes:
        project_commit (str): The project commit.
        uri (str): The URI of a fork of the project.
        branch (str): The branch name, or "HEAD" for the default branch.
        options (dict): A dict of fork-specific options.
    """

    def __init__(self, project_commit, uri, branch, options):
        self.project_commit = project_commit
        self.uri = uri
        self.branch = branch
        self.options = options

    def as_key(self):
        return (self.project_commit, self.uri, self.branch, )

class RepoListSource(OverridableProperty):
    """A source for a repo list.

    Attributes:
        uri (str): The URI of the repo list.
        options (dict): A dict of repo list-specific options.
    """

    def __init__(self, uri, options):
        self.uri = uri
        self.options = options

    def as_key(self):
        return (self.uri, )

class DataProperty(Enum):
    """Represents values that can be returned by a data source.

    See documentation for DataSource get_property_value and
    DataSource get_property_values for more details.
    """
    INSTANCE_TITLE = (1, str)
    INSTANCE_BASE_URL = (2, str)
    VCS_REPOS = (3, PCTP)
    REPO_LIST_SOURCES = (4, RepoListSource)

    def get_type(self):
        """Returns the expected type for values from this DataProperty.
        """
        return self.value[1]

class PropertyError(LookupError):
    """Raised to indicate improper use of a DataProperty.
    """
    pass

class DataSource(abc.ABC):
    @abc.abstractmethod
    def update(self):
        """Refreshes the data associated with this source, if necessary.
        """
        pass

    @abc.abstractmethod
    def exists(self):
        """Returns whether this source has usable data.
        """
        pass

    @abc.abstractmethod
    def get_supported_properties(self):
        """Returns an iterable of properties supported by this data source.

        Returns:
            Iterable of DataProperty: Supported properties.

        """
        return ()

    def get_property_value(self, prop):
        """Returns the value associated with the given property.

        If duplicated, an earlier value should override a later value.

        Args:
            prop (DataProperty): The property.

        Returns:
            The value associated with the given property.

        Raises:
            PropertyError: If the property is not supported by this data
            source.
            LookupError: If the property is supported, but isn't available.
            ValueError: If the property doesn't have exactly one value.
        """
        iterator = self.get_property_values(prop)
        try:
            # note: unpacking
            ret, = iterator
        except LookupError as exc: raise RuntimeError from exc  # don't accidentally swallow bugs in the iterator
        return ret

    @abc.abstractmethod
    def get_property_values(self, prop):
        """Yields the values associated with the given property.

        If duplicated, earlier values should override later values.

        Args:
            prop (DataProperty): The property.

        Yields:
            The values associated with the given property.

        Raises:
            PropertyError: If the property is not supported by this data
            source.
            LookupError: If the property is supported, but isn't available.

        """
        raise PropertyError

class DummyDataSource(DataSource):
    """A DataSource that provides nothing.
    """

class ObjectDataSource(DataSource):
    """A DataSource backed by a Python object.

    Updates to the backing object will be immediately reflected in this
    DataSource.
    """
    _SUPPORTED_PROPERTIES = {
                                DataProperty.INSTANCE_TITLE: lambda obj: (d['title'][1] for d in _MATCHER_TITLE.match(obj)),
                                DataProperty.INSTANCE_BASE_URL: lambda obj: (d['base_url'][1] for d in _MATCHER_BASE_URL.match(obj)),
                                DataProperty.VCS_REPOS: lambda obj: (PCTP(r['commit'][0], r['url'][0], r['branch'][0], r['branch'][1]) for r in _MATCHER_REPOS.match(obj)),
                                DataProperty.REPO_LIST_SOURCES: lambda obj: (RepoListSource(d['src'][0], d['src'][1]) for d in _MATCHER_REPO_LIST_SRCS.match(obj)),
                            }

    def __init__(self, obj):
        self._obj = obj

    def update(self):
        pass

    def exists(self):
        return True

    def get_property_values(self, prop):
        try:
            factory = self.get_supported_properties()[prop]
        except KeyError as exc: raise PropertyError from exc
        iterator = factory(self._obj)
        try:
            first = next(iterator)
        except StopIteration: return (x for x in ())
        except abdl.exceptions.ValidationError as exc: raise LookupError from exc
        except LookupError as exc: raise RuntimeError from exc  # don't accidentally swallow bugs in the iterator
        return itertools.chain([first], iterator)

    @classmethod
    def get_supported_properties(cls):
        return cls._SUPPORTED_PROPERTIES

class LocalDataSource(ObjectDataSource):
    def __init__(self, filename):
        super().__init__({})
        self.file_exists = False
        self.last_updated = None
        self.filename = filename

    def update(self):
        try:
            updtime = self.last_updated
            self.last_updated = os.stat(self.filename).st_mtime
            if not self.file_exists or updtime != self.last_updated:
                with open(self.filename, 'r', encoding='utf-8', newline='') as f:
                    self._obj = qtoml.load(f)
            self.file_exists = True
        except (OSError, UnicodeDecodeError, qtoml.decoder.TOMLDecodeError) as e:
            self.file_exists = False
            self.last_updated = None
            self._obj = {}
            return e

    def exists(self):
        return self.file_exists

    def __repr__(self):
        return "LocalDataSource({!r})".format(self.filename)

class RemoteDataSource(ObjectDataSource):
    def __init__(self, uri):
        super().__init__({})
        self.uri = uri
        self.remote_exists = False
        self.next_update = 0

    def update(self):
        if self.next_update > time.time():
            return
        # I long for the day when toml has a registered media type
        response = requests.get(self.uri, headers={'user-agent': 'ganarchy/0.0.0', 'accept': '*/*'})
        self.remote_exists = response.status_code == 200
        seconds = 3600
        if (refresh := response.headers.get('Refresh', None)) is not None:
            try:
                seconds = int(refresh)
            except ValueError:
                refresh = refresh.split(';', 1)
                try:
                    seconds = int(refresh[0])
                except ValueError:
                    pass
        self.next_update = time.time() + seconds
        if self.remote_exists:
            response.encoding = 'utf-8'
            try:
                self._obj = qtoml.loads(response.text)
            except (UnicodeDecodeError, qtoml.decoder.TOMLDecodeError) as e:
                self._obj = {}
                return e
        else:
            return response

    def exists(self):
        return self.remote_exists

    def __repr__(self):
        return "RemoteDataSource({!r})".format(self.uri)

class DefaultsDataSource(ObjectDataSource):
    """Provides a way for contributors to define/encourage some default
    settings.

    In particular, enables contributors to have a say in default domain
    blocks.
    """
    DEFAULTS = {}

    def __init__(self):
        super().__init__(self.DEFAULTS)

    def exists(self):
        return True

    def update(self):
        return

    def __repr__(self):
        return "DefaultsDataSource()"


class ConfigManager(DataSource):
    """A ConfigManager takes care of managing config sources and
    collecting their details.

    Args:
        sources (list of DataSource): The config sources to be managed.
    """
    def __init__(self, sources):
        self.sources = sources

    @classmethod
    def new_default(cls):
        srcs = [LocalDataSource(d + "/config.toml") for d in [ganarchy.dirs.CONFIG_HOME] + ganarchy.dirs.CONFIG_DIRS]
        return cls(srcs + [DefaultsDataSource()])

    def exists(self):
        return True

    def update(self):
        excs = []
        for source in self.sources:
            excs.append(source.update())
        return excs

    def get_supported_properties(self):
        return DataProperty

    def get_property_values(self, prop):
        if prop not in self.get_supported_properties():
            raise PropertyError
        elif prop == DataProperty.VCS_REPOS:
            return self._get_vcs_repos()
        elif prop == DataProperty.REPO_LIST_SOURCES:
            return self._get_repo_list_sources()
        else:
            # short-circuiting, as these are only supposed to return a single value
            for source in self.sources:
                try:
                    return source.get_property_values(prop)
                except PropertyError:
                    pass
                except LookupError:
                    pass
            raise LookupError

    def _get_vcs_repos(self):
        for source in self.sources:
            if DataProperty.VCS_REPOS in source.get_supported_properties():
                try:
                    iterator = source.get_property_values(DataProperty.VCS_REPOS)
                except LookupError:
                    pass
                else:
                    yield from iterator

    def _get_repo_list_sources(self):
        for source in self.sources:
            if DataProperty.REPO_LIST_SOURCES in source.get_supported_properties():
                try:
                    iterator = source.get_property_values(DataProperty.REPO_LIST_SOURCES)
                except LookupError:
                    pass
                else:
                    yield from iterator

class RepoListManager(DataSource):
    """A RepoListManager takes care of managing repo lists.

    Args:
        config_manager (DataSource): The config manager from which the repo
            lists come.
    """
    def __init__(self, config_manager):
        self.config_manager = EffectiveSource(config_manager)
        self.sources = [self.config_manager]

    def exists(self):
        return True

    def update(self):
        excs = [self.config_manager.update()]
        if DataProperty.REPO_LIST_SOURCES in self.config_manager.get_supported_properties():
            self.sources = [self.config_manager]
            try:
                it = self.config_manager.get_property_values(DataProperty.REPO_LIST_SOURCES)
            except LookupError:
                pass
            else:
                self.sources.extend(RemoteDataSource(rls.uri) for rls in it if rls.active)
        for source in self.sources[1:]:
            excs.append(source.update())
        return excs

    def get_supported_properties(self):
        return {DataProperty.VCS_REPOS}

    def get_property_values(self, prop):
        if prop not in self.get_supported_properties():
            raise PropertyError
        assert prop == DataProperty.VCS_REPOS
        return self._get_vcs_repos()

    def _get_vcs_repos(self):
        assert self.config_manager == self.sources[0]
        try:
            # config manager may override repo lists
            iterator = self.config_manager.get_property_values(DataProperty.VCS_REPOS)
        except (PropertyError, LookupError):
            pass
        else:
            yield from iterator
        for source in self.sources:
            if DataProperty.VCS_REPOS in source.get_supported_properties():
                try:
                    iterator = source.get_property_values(DataProperty.VCS_REPOS)
                except LookupError:
                    pass
                else:
                    for pctp in iterator:
                        # but repo lists aren't allowed to override anything
                        if pctp.active:
                            yield pctp

class EffectiveSource(DataSource):
    """Wraps another ``DataSource`` and yields "unique" results suitable
    for general use.

    Methods on this class, in particular ``get_property_values``, handle
    ``OverridableProperty`` overrides both to avoid code duplication and
    so the user doesn't have to.

    Args:
        raw_source (DataSource): The raw backing source.
    """
    def __init__(self, raw_source):
        self.raw_source = raw_source

    def exists(self):
        return self.raw_source.exists()

    def update(self):
        return self.raw_source.update()

    def get_property_value(self, prop):
        return self.raw_source.get_property_value(prop)

    def get_supported_properties(self):
        return self.raw_source.get_supported_properties()

    def get_property_values(self, prop):
        # must raise exceptions *now*
        # not when the generator runs
        return self._wrap_values(prop, self.raw_source.get_property_values(prop))

    def _wrap_values(self, prop, it):
        if issubclass(prop.get_type(), OverridableProperty):
            seen = {}
            for v in it:
                k = v.as_key()
                if k in seen:
                    continue
                seen[k] = v
                yield v
        else:
            yield from it

    def __repr__(self):
        return "EffectiveSource({!r})".format(self.raw_source)

# class Config:
#     def __init__(self, toml_file, base=None, remove=True):
#         self.projects = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(dict))))
#         config_data = qtoml.load(toml_file)
#         self.remote_configs = config_data.get('config_srcs', [])
#         self.title = config_data.get('title', '')
#         self.base_url = config_data.get('base_url', '')
#         # TODO blocked domains (but only read them from config_data if remove is True)
#         self.blocked_domains = []
#         self.blocked_domain_suffixes = []
#         self.blocked_domains.sort()
#         self.blocked_domain_suffixes.sort(key=lambda x: x[::-1])
#         # FIXME remove duplicates and process invalid entries
#         self.blocked_domains = tuple(self.blocked_domains)
#         self.blocked_domain_suffixes = tuple(self.blocked_domain_suffixes) # MUST be tuple
#         # TODO re.compile("(^" + "|^".join(map(re.escape, domains)) + "|" + "|".join(map(re.escape, suffixes) + ")$")
#         if base:
#             # FIXME is remove=remove the right thing to do?
#             self._update_projects(base.projects, remove=remove, sanitize=False) # already sanitized
#         projects = config_data.get('projects', {})
#         self._update_projects(projects, remove=remove)
# 
#     def _update_projects(self, projects, remove, sanitize=True):
#         m = (m_ganarchy_config.CONFIG_PATTERN_SANITIZE if sanitize else m_ganarchy_config.CONFIG_PATTERN).match(projects)
#         for v in m:
#             commit, repo_url, branchname, options = v['commit'][0], v['url'][0], v['branch'][0], v['branch'][1]
#             try:
#                 u = urlparse(repo_url)
#                 if not u:
#                     raise ValueError
#                 # also raises for invalid ports, see https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlparse
#                 # "Reading the port attribute will raise a ValueError if an invalid port is specified in the URL. [...]"
#                 if u.port == 0:
#                     raise ValueError
#                 if u.scheme not in ('http', 'https'):
#                     raise ValueError
#                 if (u.hostname in self.blocked_domains) or (u.hostname.endswith(self.blocked_domain_suffixes)):
#                     raise ValueError
#             except ValueError:
#                 continue
#             if branchname == "HEAD":
#                 branchname = None
#             active = options.get('active', None)
#             if active not in (True, False):
#                 continue
#             branch = self.projects[commit][repo_url][branchname]
#             branch['active'] = active or (branch.get('active', False) and not remove)