diff options
Diffstat (limited to 'ganarchy')
| -rw-r--r-- | ganarchy/__init__.py | 16 | ||||
| -rw-r--r-- | ganarchy/__main__.py | 29 | ||||
| -rw-r--r-- | ganarchy/cli/__init__.py | 27 | ||||
| -rw-r--r-- | ganarchy/cli/db.py | 71 | ||||
| -rw-r--r-- | ganarchy/cli/debug.py | 112 | ||||
| -rw-r--r-- | ganarchy/cli/merge_configs.py | 25 | ||||
| -rw-r--r-- | ganarchy/cli/run_targets.py | 231 | ||||
| -rw-r--r-- | ganarchy/core.py | 286 | ||||
| -rw-r--r-- | ganarchy/data.py | 565 | ||||
| -rw-r--r-- | ganarchy/db.py | 369 | ||||
| -rw-r--r-- | ganarchy/dirs.py | 53 | ||||
| -rw-r--r-- | ganarchy/git.py | 173 | ||||
| -rw-r--r-- | ganarchy/templating/__init__.py | 21 | ||||
| -rw-r--r-- | ganarchy/templating/environment.py | 31 | ||||
| -rw-r--r-- | ganarchy/templating/templates.py | 128 | ||||
| -rw-r--r-- | ganarchy/templating/toml.py | 33 |
16 files changed, 0 insertions, 2170 deletions
diff --git a/ganarchy/__init__.py b/ganarchy/__init__.py deleted file mode 100644 index fdb6788..0000000 --- a/ganarchy/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# GAnarchy - decentralized project hub -# Copyright (C) 2019, 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - diff --git a/ganarchy/__main__.py b/ganarchy/__main__.py deleted file mode 100644 index 63e55ed..0000000 --- a/ganarchy/__main__.py +++ /dev/null @@ -1,29 +0,0 @@ -# GAnarchy - decentralized project hub -# Copyright (C) 2019, 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -# The base CLI -import ganarchy.cli - -# FIXME this shouldn't be here -import ganarchy - -# Additional CLI commands -import ganarchy.cli.db -import ganarchy.cli.debug -import ganarchy.cli.merge_configs -import ganarchy.cli.run_targets - -ganarchy.cli.main(prog_name='ganarchy') diff --git a/ganarchy/cli/__init__.py b/ganarchy/cli/__init__.py deleted file mode 100644 index 727c48d..0000000 --- a/ganarchy/cli/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2019 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -"""The GAnarchy CLI. - -This module just defines the main command group. Submodules define -actual commands. -""" - -import click - -@click.group() -def main(): - pass diff --git a/ganarchy/cli/db.py b/ganarchy/cli/db.py deleted file mode 100644 index e754c34..0000000 --- a/ganarchy/cli/db.py +++ /dev/null @@ -1,71 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -"""Database-related CLI commands. - -""" - -import os - -import click - -import ganarchy.cli -import ganarchy.data -import ganarchy.db -import ganarchy.dirs - -@ganarchy.cli.main.command() -def initdb(): - """Initializes the ganarchy database.""" - # TODO: makedirs in a separate command? - os.makedirs(ganarchy.dirs.DATA_HOME, exist_ok=True) - db = ganarchy.db.connect_database(ganarchy.data.ConfigManager.new_default()) - db.initialize() - db.close() - -@ganarchy.cli.main.group() -def migrations(): - """Modifies the DB to work with a newer/older version. - - WARNING: THIS COMMAND CAN BE EXTREMELY DESTRUCTIVE!""" - -@migrations.command() -@click.argument('migration') -def apply(migration): - """Applies the migration with the given name.""" - db = ganarchy.db.connect_database(ganarchy.data.ConfigManager.new_default()) - click.echo(ganarchy.db.MIGRATIONS[migration][0]) - db.apply_migration(migration) - db.close() - -@click.argument('migration') -@migrations.command() -def revert(migration): - """Reverts the migration with the given name.""" - db = ganarchy.db.connect_database(ganarchy.data.ConfigManager.new_default()) - click.echo(ganarchy.db.MIGRATIONS[migration][1]) - db.revert_migration(migration) - db.close() - -@click.argument('migration', required=False) -@migrations.command() -def info(migration): - """Shows information about the migration with the given name.""" - if not migration: - # TODO could be improved - click.echo(ganarchy.db.MIGRATIONS.keys()) - else: - click.echo(ganarchy.db.MIGRATIONS[migration][2]) diff --git a/ganarchy/cli/debug.py b/ganarchy/cli/debug.py deleted file mode 100644 index 95ad045..0000000 --- a/ganarchy/cli/debug.py +++ /dev/null @@ -1,112 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2019 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -import click -import qtoml - -import ganarchy -import ganarchy.cli -import ganarchy.data - -@ganarchy.cli.main.group() -def debug(): - pass - -@debug.command() -def paths(): - click.echo('Config home: {}'.format(ganarchy.config_home)) - click.echo('Additional config search path: {}'.format(ganarchy.config_dirs)) - click.echo('Cache home: {}'.format(ganarchy.cache_home)) - click.echo('Data home: {}'.format(ganarchy.data_home)) - -def print_data_source(data_source): - if ganarchy.data.DataProperty.INSTANCE_TITLE in data_source.get_supported_properties(): - try: - title = data_source.get_property_value(ganarchy.data.DataProperty.INSTANCE_TITLE) - except LookupError: - title = None - click.echo("\tTitle: {}".format(title)) - - if ganarchy.data.DataProperty.INSTANCE_BASE_URL in data_source.get_supported_properties(): - try: - base_url = data_source.get_property_value(ganarchy.data.DataProperty.INSTANCE_BASE_URL) - except LookupError: - base_url = None - click.echo("\tBase URL: {}".format(base_url)) - - if ganarchy.data.DataProperty.REPO_LIST_SOURCES in data_source.get_supported_properties(): - click.echo("\tRepo list sources:") - try: - iterator = data_source.get_property_values(ganarchy.data.DataProperty.REPO_LIST_SOURCES) - except LookupError: - click.echo("\t\tNone") - else: - for i, rls in enumerate(iterator, 1): - click.echo("\t\t{}.".format(i)) - click.echo("\t\t\tURI: {}".format(rls.uri)) - click.echo("\t\t\tOptions: {}".format(rls.options)) - click.echo("\t\t\tActive: {}".format(rls.active)) - - if ganarchy.data.DataProperty.VCS_REPOS in data_source.get_supported_properties(): - click.echo("\tRepos:") - try: - iterator = data_source.get_property_values(ganarchy.data.DataProperty.VCS_REPOS) - except LookupError: - click.echo("\t\tNone") - else: - for i, pctp in enumerate(iterator, 1): - click.echo("\t\t{}.".format(i)) - click.echo("\t\t\tProject: {}".format(pctp.project_commit)) - click.echo("\t\t\tURI: {}".format(pctp.uri)) - click.echo("\t\t\tBranch: {}".format(pctp.branch)) - click.echo("\t\t\tOptions: {}".format(pctp.options)) - click.echo("\t\t\tActive: {}".format(pctp.active)) - -@debug.command() -def configs(): - confs = ganarchy.data.ConfigManager.new_default() - click.echo("Configs (raw): {}".format(confs.sources)) - click.echo("Breaking down the configs.") - update_excs = confs.update() - for conf, exc in zip(reversed(confs.sources), reversed(update_excs)): - click.echo("Config: {}".format(conf)) - if exc is not None: - click.echo("\tError(s): {}".format(exc)) - if conf.exists(): - print_data_source(conf) - click.echo("ConfigManager (raw):") - print_data_source(confs) - click.echo("ConfigManager (effective):") - print_data_source(ganarchy.data.EffectiveSource(confs)) - -@debug.command() -def repo_lists(): - confs = ganarchy.data.ConfigManager.new_default() - repo_lists = ganarchy.data.RepoListManager(confs) - update_excs = repo_lists.update() - click.echo("Repo lists (raw): {}".format(repo_lists.sources)) - click.echo("Breaking down the repo lists.") - for repo_list, exc in zip(reversed(repo_lists.sources), reversed(update_excs)): - click.echo("Repo list: {}".format(repo_list)) - if exc is not None: - click.echo("\tError(s): {}".format(exc)) - if repo_list.exists(): - print_data_source(repo_list) - click.echo("RepoListManager (raw):") - print_data_source(repo_lists) - click.echo("RepoListManager (effective):") - print_data_source(ganarchy.data.EffectiveSource(repo_lists)) - diff --git a/ganarchy/cli/merge_configs.py b/ganarchy/cli/merge_configs.py deleted file mode 100644 index d8e12e6..0000000 --- a/ganarchy/cli/merge_configs.py +++ /dev/null @@ -1,25 +0,0 @@ -import pathlib - -import click - -import ganarchy -import ganarchy.cli -import ganarchy.data - -@ganarchy.cli.main.command() -@click.option('--skip-errors/--no-skip-errors', default=False) -@click.argument('files', type=click.Path(exists=True, dir_okay=False, resolve_path=True), nargs=-1) -def merge_configs(skip_errors, files): - """Merges config files.""" - configs = [ganarchy.data.LocalDataSource(filename) for filename in files] - rlm = ganarchy.data.RepoListManager(ganarchy.data.ObjectDataSource({})) - rlm.sources += configs - res = [] - for src in rlm.sources: - res.append(src.update()) - effective = ganarchy.data.EffectiveSource(rlm) - if any(x is None for x in res): - click.echo("# This is DEPRECATED and will be REMOVED at some point!") - for pctp in effective.get_property_values(ganarchy.data.DataProperty.VCS_REPOS): - if pctp.active: - click.echo(f"""projects."{ganarchy.tomlescape(pctp.project_commit)}"."{ganarchy.tomlescape(pctp.uri)}"."{ganarchy.tomlescape(pctp.branch)}" = {{ active=true }}""") diff --git a/ganarchy/cli/run_targets.py b/ganarchy/cli/run_targets.py deleted file mode 100644 index 24497b9..0000000 --- a/ganarchy/cli/run_targets.py +++ /dev/null @@ -1,231 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -"""This module contains the CLI Run Targets. -""" - -import os -import shutil - -import click - -from ganarchy import cli -from ganarchy import core -from ganarchy import data -from ganarchy import db -from ganarchy import dirs -from ganarchy.templating import environment - -@cli.main.command() -@click.option('--keep-stale-projects/--no-keep-stale-projects', default=True) -@click.argument('out', required=True, type=click.Path(file_okay=False, resolve_path=True)) -def run_once(out, keep_stale_projects): - """Runs GAnarchy once. - - Processes any necessary updates and updates the output directory to match. - """ -# """Runs ganarchy standalone. -# -# This will run ganarchy so it regularly updates the output directory given -# by OUT. Additionally, it'll also search for the following hooks in its -# config dirs: -# -# - post_object_update_hook - executed after an object is updated. -# -# - post_update_cycle_hook - executed after all objects in an update -# cycle are updated. -# """ - # create config objects - conf = data.ConfigManager.new_default() - effective_conf = data.EffectiveSource(conf) - repos = data.RepoListManager(effective_conf) - effective_repos = data.EffectiveSource(repos) - - # create dir if it doesn't exist - os.makedirs(out, exist_ok=True) - - # load template environment - env = environment.get_env() - - # make sure the cache dir exists - os.makedirs(dirs.CACHE_HOME, exist_ok=True) - - # make sure it is a git repo - core.GIT.create() - - if True: - # reload config and repo data - effective_repos.update() - database = db.connect_database(effective_conf) - database.load_repos(effective_repos) - - instance = core.GAnarchy(database, effective_conf) - - if not instance.base_url: - click.echo("No base URL specified", err=True) - return - - instance.load_projects() - - # update and render projects - if not keep_stale_projects: - shutil.rmtree(out + "/project") - - os.makedirs(out + "/project", exist_ok=True) - - template_project = env.get_template('project.html') - for p in instance.projects: - p.load_repos() - - generate_html = [] - results = p.update() - #if not p.exists: - # ... - for (repo, count) in results: - if count is not None: - generate_html.append( - (repo.url, repo.message, count, repo.branch) - ) - else: - click.echo(repo.errormsg, err=True) - html_entries = [] - for (url, msg, count, branch) in generate_html: - history = database.list_repobranch_activity(p.commit, url, branch) - # TODO process history into SVG - # TODO move this into a separate system - # (e.g. ``if project.startswith("svg-"):``) - html_entries.append((url, msg, "", branch)) - - os.makedirs(out + "/project/" + p.commit, exist_ok=True) - - with open(out + "/project/" + p.commit + "/index.html", "w") as f: - template_project.stream( - project_title = p.title, - project_desc = p.description, - project_body = p.commit_body, - project_commit = p.commit, - repos = html_entries, - base_url = instance.base_url, - # I don't think this thing supports deprecating the above? - project = p, - ganarchy = instance - ).dump(f) - - # render the config - template = env.get_template('index.toml') - with open(out + "/index.toml", "w") as f: - template.stream(database=database).dump(f) - - # render the index - # but reload projects first to pick up sorting order - # (new projects don't get sorted until their repos get fetched for the - # first time, because that's where the metadata is stored) - # FIXME .sort_projects()? - instance.load_projects() - template = env.get_template('index.html') - with open(out + "/index.html", "w") as f: - template.stream(ganarchy=instance).dump(f) - - -@cli.main.command() -@click.option('--dry-run/--no-dry-run', '--no-update/--update', default=False) -@click.argument('project', required=False) -def cron_target(dry_run, project): - """Runs ganarchy as a cron target. - - "Deprecated". Useful if you want full control over how GAnarchy - generates the pages. - """ - # create config objects - conf = data.ConfigManager.new_default() - effective_conf = data.EffectiveSource(conf) - repos = data.RepoListManager(effective_conf) - effective_repos = data.EffectiveSource(repos) - - # load config and repo data - effective_repos.update() - database = db.connect_database(effective_conf) - database.load_repos(effective_repos) - - # load template environment - env = environment.get_env() - - # handle config and project list - if project == "config": - # render the config - template = env.get_template('index.toml') - click.echo(template.render(database=database), nl=False) - return - if project == "project-list": - # could be done with a template but eh w/e, this is probably better - for project in database.list_projects(): - click.echo(project) - return - - # make sure the cache dir exists - os.makedirs(dirs.CACHE_HOME, exist_ok=True) - - # make sure it is a git repo - core.GIT.create() - - instance = core.GAnarchy(database, effective_conf) - - if not instance.base_url or not project: - click.echo("No base URL or project commit specified", err=True) - return - - if project == "index": - instance.load_projects() - # render the index - template = env.get_template('index.html') - click.echo(template.render(ganarchy=instance), nl=False) - return - - p = core.Project(database, project) - p.load_repos() - - generate_html = [] - results = p.update(dry_run=dry_run) - #if not p.exists: - # ... - for (repo, count) in results: - if count is not None: - generate_html.append((repo.url, repo.message, count, repo.branch)) - else: - click.echo(repo.errormsg, err=True) - html_entries = [] - for (url, msg, count, branch) in generate_html: - history = database.list_repobranch_activity(project, url, branch) - # TODO process history into SVG - # TODO move this into a separate system - # (e.g. ``if project.startswith("svg-"):``) - html_entries.append((url, msg, "", branch)) - - template = env.get_template('project.html') - click.echo( - template.render( - project_title = p.title, - project_desc = p.description, - project_body = p.commit_body, - project_commit = p.commit, - repos = html_entries, - base_url = instance.base_url, - # I don't think this thing supports deprecating the above? - project = p, - ganarchy = instance - ), - nl=False - ) diff --git a/ganarchy/core.py b/ganarchy/core.py deleted file mode 100644 index b1025d1..0000000 --- a/ganarchy/core.py +++ /dev/null @@ -1,286 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -"""Core logic of GAnarchy. -""" - -import hashlib -import hmac -import re -from urllib import parse - -import ganarchy.git -import ganarchy.dirs -import ganarchy.data - -# Currently we only use one git repo, at CACHE_HOME -# TODO optimize -GIT = ganarchy.git.Git(ganarchy.dirs.CACHE_HOME) - -class Repo: - """A GAnarchy repo. - - Args: - dbconn (ganarchy.db.Database): The database connection. - project_commit (str): The project commit. - url (str): The git URL. - branch (str): The branch. - head_commit (str): The last known head commit. - - Attributes: - branch (str or None): The remote git branch. - branchname (str): The local git branch. - """ - # TODO fill in Attributes. - - def __init__(self, dbconn, project_commit, url, branch, head_commit): - self.url = url - self.branch = branch - self.project_commit = project_commit - self.errormsg = None - self.erroring = False - self.message = None - self.hash = None - self.branchname = None - self.head = None - - if not self._check_branch(): - return - - if not branch: - self.branchname = "gan" + hashlib.sha256(url.encode("utf-8")).hexdigest() - self.head = "HEAD" - else: - self.branchname = "gan" + hmac.new(branch.encode("utf-8"), url.encode("utf-8"), "sha256").hexdigest() - self.head = "refs/heads/" + branch - - if head_commit: - self.hash = head_commit - else: - try: # FIXME should we even do this? - self.hash = GIT.get_hash(self.branchname) - except ganarchy.git.GitError: - self.erroring = True - - self.refresh_metadata() - - def _check_branch(self): - """Checks if ``self.branch`` is a valid git branch name, or None. Sets - ``self.errormsg`` and ``self.erroring`` accordingly. - - Returns: - bool: True if valid, False otherwise. - """ - if not self.branch: - return True - try: - GIT.check_branchname(self.branch) - return True - except ganarchy.git.GitError as e: - self.erroring = True - self.errormsg = e - return False - - def refresh_metadata(self): - """Refreshes repo metadata. - """ - if not self._check_branch(): - return - try: - self.message = GIT.get_commit_message(self.branchname) - except ganarchy.git.GitError as e: - self.erroring = True - self.errormsg = e - - # FIXME maybe this shouldn't be "public"? - # reasoning: this update() isn't reflected in the db. - # but this might be handy for dry runs. - # alternatively: change the return to be the new head commit, - # and update things accordingly. - def update(self, *, dry_run=False): - """Updates the git repo, returning a commit count. - - Args: - dry_run (bool): To simulate an update without doing anything. - In particular, without fetching commits. - """ - if not self._check_branch(): - return None - if not dry_run: - try: - GIT.force_fetch(self.url, self.head, self.branchname) - except ganarchy.git.GitError as e: - # This may error for various reasons, but some - # are important: dead links, etc - self.erroring = True - self.errormsg = e - return None - pre_hash = self.hash - try: - post_hash = GIT.get_hash(self.branchname) - except ganarchy.git.GitError as e: - # This should never happen, but maybe there's some edge cases? - # TODO check - self.erroring = True - self.errormsg = e - return None - self.hash = post_hash - if not pre_hash: - pre_hash = post_hash - count = GIT.get_count(pre_hash, post_hash) - try: - GIT.check_history(self.branchname, self.project_commit) - self.refresh_metadata() - return count - except ganarchy.git.GitError as e: - self.erroring = True - self.errormsg = e - return None - -class Project: - """A GAnarchy project. - - Args: - dbconn (ganarchy.db.Database): The database connection. - project_commit (str): The project commit. - - Attributes: - commit (str): The project commit. - repos (list, optional): Repos associated with this project. - title (str, optional): Title of the project. - description (str, optional): Description of the project. - commit_body (str, optional): Raw commit message for title and - description. - exists (bool): Whether the project exists in our git cache. - """ - - def __init__(self, dbconn, project_commit): - self.commit = project_commit - self.refresh_metadata() - self.repos = None - self._dbconn = dbconn - - def load_repos(self): - """Loads the repos into this project. - - If repos have already been loaded, re-loads them. - """ - repos = [] - for url, branch, head_commit in self._dbconn.list_repobranches(self.commit): - repos.append( - Repo(self._dbconn, self.commit, url, branch, head_commit) - ) - self.repos = repos - - def refresh_metadata(self): - """Refreshes project metadata. - """ - try: - project = GIT.get_commit_message(self.commit) - project_title, project_desc = (lambda x: x.groups() if x is not None else ('', None))(re.fullmatch('^\\[Project\\]\s+(.+?)(?:\n\n(.+))?$', project, flags=re.ASCII|re.DOTALL|re.IGNORECASE)) - if not project_title.strip(): # FIXME - project_title, project_desc = ("Error parsing project commit",)*2 - # if project_desc: # FIXME - # project_desc = project_desc.strip() - self.exists = True - self.commit_body = project - self.title = project_title - self.description = project_desc - except ganarchy.git.GitError: - self.exists = False - self.commit_body = None - self.title = None - self.description = None - - def update(self, *, dry_run=False): - """Updates the project and its repos. - """ - # TODO? check if working correctly - results = [] - if self.repos is not None: - for repo in self.repos: - results.append((repo, repo.update(dry_run=dry_run))) - self.refresh_metadata() - if self.repos is not None: - results.sort(key=lambda x: x[1] or -1, reverse=True) - if not dry_run: - entries = [] - for (repo, count) in results: - if count is not None: - entries.append(( - self.commit, - repo.url, - repo.branch, - repo.hash, - count - )) - self._dbconn.insert_activities(entries) - return results - -class GAnarchy: - """A GAnarchy instance. - - Args: - dbconn (ganarchy.db.Database): The database connection. - config (ganarchy.data.DataSource): The (effective) config. - - Attributes: - base_url (str): Instance base URL. - title (str): Instance title. - projects (list, optional): Projects associated with this instance. - """ - - def __init__(self, dbconn, config): - self.title = None - self.base_url = None - self.projects = None - self._dbconn = dbconn - self._config = config - self.load_metadata() - - def load_metadata(self): - """Loads instance metadata from config. - - If instance metadata has already been loaded, re-loads it. - """ - try: - base_url = self._config.get_property_value( - ganarchy.data.DataProperty.INSTANCE_BASE_URL - ) - except LookupError: - # FIXME use a more appropriate error type - raise ValueError - - try: - title = self._config.get_property_value( - ganarchy.data.DataProperty.INSTANCE_TITLE - ) - except LookupError: - title = "GAnarchy on " + parse.urlparse(base_url).hostname - - self.title = title - self.base_url = base_url - - def load_projects(self): - """Loads the projects into this GAnarchy instance. - - If projects have already been loaded, re-loads them. - """ - projects = [] - for project in self._dbconn.list_projects(): - projects.append(Project(self._dbconn, project)) - projects.sort(key=lambda p: p.title or "") # sort projects by title - self.projects = projects diff --git a/ganarchy/data.py b/ganarchy/data.py deleted file mode 100644 index 36c32d9..0000000 --- a/ganarchy/data.py +++ /dev/null @@ -1,565 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2019, 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -"""This module handles GAnarchy's data and config sources. - -A data source can be either a config source or a repo list source, but be -careful: they use identical syntax, but have different semantics! Mistaking -a repo list source for a config source is a recipe for security bugs! -""" - -import abc -import itertools -import os -import re -import time - -import abdl -import abdl.exceptions -import qtoml -import requests - -from enum import Enum -from urllib.parse import urlparse - -import ganarchy.dirs - -# TODO move elsewhere -class URIPredicate(abdl.predicates.Predicate): - def __init__(self, ports=range(1,65536), schemes=('https',)): - self.ports = ports - self.schemes = schemes - - def accept(self, obj): - try: - u = urlparse(obj) - if not u: - return False - # also raises for invalid ports, see https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlparse - # "Reading the port attribute will raise a ValueError if an invalid port is specified in the URL. [...]" - if u.port is not None and u.port not in self.ports: - return False - if u.scheme not in self.schemes: - return False - except ValueError: - return False - return True - -class CommitPredicate(abdl.predicates.Predicate): - def __init__(self, sha256ready=True): - if sha256ready: - self.re = re.compile(r"^[0-9a-fA-F]{40}$|^[0-9a-fA-F]{64}$") - else: - self.re = re.compile(r"^[0-9a-fA-F]{40}$") - - def accept(self, obj): - return self.re.match(obj) - -# sanitize = skip invalid entries -# validate = error on invalid entries -# LEGACY. DO NOT USE. -# TODO remove -CONFIG_REPOS_SANITIZE = abdl.compile("""->'projects'?:?$dict - ->commit[:?$str:?$commit]:?$dict - ->url[:?$str:?$uri]:?$dict - ->branch:?$dict(->'active'?:?$bool)""", - dict(bool=bool, dict=dict, str=str, uri=URIPredicate(), commit=CommitPredicate())) - -CONFIG_TITLE_SANITIZE = abdl.compile("""->title'title'?:?$str""", dict(str=str)) -CONFIG_BASE_URL_SANITIZE = abdl.compile("""->base_url'base_url'?:?$str:?$uri""", dict(str=str, uri=URIPredicate())) - -# modern matchers, raise ValidationError if the data doesn't exist. -# they still skip "bad" entries, just like the old matchers. - -_MATCHER_REPOS = abdl.compile("""->'projects':$dict - ->commit[:?$str:?$commit]:?$dict - ->url[:?$str:?$uri]:?$dict - ->branch:?$dict - (->active'active'?:?$bool) - (->federate'federate'?:?$bool)?""", - dict(bool=bool, dict=dict, str=str, uri=URIPredicate(), commit=CommitPredicate())) -_MATCHER_REPO_LIST_SRCS = abdl.compile("""->'repo_list_srcs':$dict - ->src[:?$str:?$uri]:?$dict - (->'active'?:?$bool)""", - dict(bool=bool, list=list, dict=dict, str=str, uri=URIPredicate(schemes=('https','file',)))) -# TODO -#_MATCHER_ALIASES = abdl.compile("""->'project_settings':$dict -# ->commit/[0-9a-fA-F]{40}|[0-9a-fA-F]{64}/?:?$dict -# """, {'dict': dict}) # FIXME check for aliases, might require changes to abdl - -# TODO -#_MATCHER_URI_FILTERS = abdl.compile("""->'uri_filters':$dict -# ->filter[:?$str]:?$dict -# (->'active'?:?$bool)""", -# dict(dict=dict, str=str, bool=bool)) - -_MATCHER_TITLE = abdl.compile("""->title'title':$str""", dict(str=str)) -_MATCHER_BASE_URL = abdl.compile("""->base_url'base_url':$str:$uri""", dict(str=str, uri=URIPredicate())) - -class OverridableProperty(abc.ABC): - """An overridable property, with options. - - Attributes: - options (dict): Options. - """ - - @abc.abstractmethod - def as_key(self): - """Returns an opaque representation of this OverridablePRoperty - suitable for use as a dict key. - - The returned object is not suitable for other purposes. - """ - return () - - @property - def active(self): - """Whether this property is active. - """ - return self.options.get('active', False) - -class PCTP(OverridableProperty): - """A Project Commit-Tree Path. - - Attributes: - project_commit (str): The project commit. - uri (str): The URI of a fork of the project. - branch (str): The branch name, or None for the default branch. - options (dict): A dict of fork-specific options. - """ - - def __init__(self, project_commit, uri, branch, options): - self.project_commit = project_commit - self.uri = uri - if branch == "HEAD": - self.branch = None - else: - self.branch = branch or None - self.options = options - - def as_key(self): - return (self.project_commit, self.uri, self.branch, ) - - @property - def federate(self): - return self.options.get('federate', True) - -class RepoListSource(OverridableProperty): - """A source for a repo list. - - Attributes: - uri (str): The URI of the repo list. - options (dict): A dict of repo list-specific options. - """ - - def __init__(self, uri, options): - self.uri = uri - self.options = options - - def as_key(self): - return (self.uri, ) - -class DataProperty(Enum): - """Represents values that can be returned by a data source. - - See documentation for DataSource get_property_value and - DataSource get_property_values for more details. - """ - INSTANCE_TITLE = (1, str) - INSTANCE_BASE_URL = (2, str) - VCS_REPOS = (3, PCTP) - REPO_LIST_SOURCES = (4, RepoListSource) - - def get_type(self): - """Returns the expected type for values from this DataProperty. - """ - return self.value[1] - -class PropertyError(LookupError): - """Raised to indicate improper use of a DataProperty. - """ - pass - -class DataSource(abc.ABC): - @abc.abstractmethod - def update(self): - """Refreshes the data associated with this source, if necessary. - """ - pass - - @abc.abstractmethod - def exists(self): - """Returns whether this source has usable data. - """ - pass - - @abc.abstractmethod - def get_supported_properties(self): - """Returns an iterable of properties supported by this data source. - - Returns: - Iterable of DataProperty: Supported properties. - - """ - return () - - def get_property_value(self, prop): - """Returns the value associated with the given property. - - If duplicated, an earlier value should override a later value. - - Args: - prop (DataProperty): The property. - - Returns: - The value associated with the given property. - - Raises: - PropertyError: If the property is not supported by this data - source. - LookupError: If the property is supported, but isn't available. - ValueError: If the property doesn't have exactly one value. - """ - iterator = self.get_property_values(prop) - try: - # note: unpacking - ret, = iterator - except LookupError as exc: raise RuntimeError from exc # don't accidentally swallow bugs in the iterator - return ret - - @abc.abstractmethod - def get_property_values(self, prop): - """Yields the values associated with the given property. - - If duplicated, earlier values should override later values. - - Args: - prop (DataProperty): The property. - - Yields: - The values associated with the given property. - - Raises: - PropertyError: If the property is not supported by this data - source. - LookupError: If the property is supported, but isn't available. - - """ - raise PropertyError - -class DummyDataSource(DataSource): - """A DataSource that provides nothing. - """ - -class ObjectDataSource(DataSource): - """A DataSource backed by a Python object. - - Updates to the backing object will be immediately reflected in this - DataSource. - """ - _SUPPORTED_PROPERTIES = { - DataProperty.INSTANCE_TITLE: lambda obj: (d['title'][1] for d in _MATCHER_TITLE.match(obj)), - DataProperty.INSTANCE_BASE_URL: lambda obj: (d['base_url'][1] for d in _MATCHER_BASE_URL.match(obj)), - DataProperty.VCS_REPOS: lambda obj: (PCTP(r['commit'][0], r['url'][0], r['branch'][0], {k: v[1] for k, v in r.items() if k in {'active', 'federate'}}) for r in _MATCHER_REPOS.match(obj)), - DataProperty.REPO_LIST_SOURCES: lambda obj: (RepoListSource(d['src'][0], d['src'][1]) for d in _MATCHER_REPO_LIST_SRCS.match(obj)), - } - - def __init__(self, obj): - self._obj = obj - - def update(self): - pass - - def exists(self): - return True - - def get_property_values(self, prop): - try: - factory = self.get_supported_properties()[prop] - except KeyError as exc: raise PropertyError from exc - iterator = factory(self._obj) - try: - first = next(iterator) - except StopIteration: return (x for x in ()) - except abdl.exceptions.ValidationError as exc: raise LookupError from exc - except LookupError as exc: raise RuntimeError from exc # don't accidentally swallow bugs in the iterator - return itertools.chain([first], iterator) - - @classmethod - def get_supported_properties(cls): - return cls._SUPPORTED_PROPERTIES - -class LocalDataSource(ObjectDataSource): - def __init__(self, filename): - super().__init__({}) - self.file_exists = False - self.last_updated = None - self.filename = filename - - def update(self): - try: - updtime = self.last_updated - self.last_updated = os.stat(self.filename).st_mtime - if not self.file_exists or updtime != self.last_updated: - with open(self.filename, 'r', encoding='utf-8', newline='') as f: - self._obj = qtoml.load(f) - self.file_exists = True - except (OSError, UnicodeDecodeError, qtoml.decoder.TOMLDecodeError) as e: - self.file_exists = False - self.last_updated = None - self._obj = {} - return e - - def exists(self): - return self.file_exists - - def __repr__(self): - return "LocalDataSource({!r})".format(self.filename) - -class RemoteDataSource(ObjectDataSource): - def __init__(self, uri): - super().__init__({}) - self.uri = uri - self.remote_exists = False - self.next_update = 0 - - def update(self): - if self.next_update > time.time(): - return - # I long for the day when toml has a registered media type - response = requests.get(self.uri, headers={'user-agent': 'ganarchy/0.0.0', 'accept': '*/*'}) - self.remote_exists = response.status_code == 200 - seconds = 3600 - if (refresh := response.headers.get('Refresh', None)) is not None: - try: - seconds = int(refresh) - except ValueError: - refresh = refresh.split(';', 1) - try: - seconds = int(refresh[0]) - except ValueError: - pass - self.next_update = time.time() + seconds - if self.remote_exists: - response.encoding = 'utf-8' - try: - self._obj = qtoml.loads(response.text) - except (UnicodeDecodeError, qtoml.decoder.TOMLDecodeError) as e: - self._obj = {} - return e - else: - return response - - def exists(self): - return self.remote_exists - - def __repr__(self): - return "RemoteDataSource({!r})".format(self.uri) - -class DefaultsDataSource(ObjectDataSource): - """Provides a way for contributors to define/encourage some default - settings. - - In particular, enables contributors to have a say in default domain - blocks. - """ - DEFAULTS = {} - - def __init__(self): - super().__init__(self.DEFAULTS) - - def exists(self): - return True - - def update(self): - return - - def __repr__(self): - return "DefaultsDataSource()" - - -class ConfigManager(DataSource): - """A ConfigManager takes care of managing config sources and - collecting their details. - - Args: - sources (list of DataSource): The config sources to be managed. - """ - def __init__(self, sources): - self.sources = sources - - @classmethod - def new_default(cls): - srcs = [LocalDataSource(d + "/config.toml") for d in [ganarchy.dirs.CONFIG_HOME] + ganarchy.dirs.CONFIG_DIRS] - return cls(srcs + [DefaultsDataSource()]) - - def exists(self): - return True - - def update(self): - excs = [] - for source in self.sources: - excs.append(source.update()) - return excs - - def get_supported_properties(self): - return DataProperty - - def get_property_values(self, prop): - if prop not in self.get_supported_properties(): - raise PropertyError - elif prop == DataProperty.VCS_REPOS: - return self._get_vcs_repos() - elif prop == DataProperty.REPO_LIST_SOURCES: - return self._get_repo_list_sources() - else: - # short-circuiting, as these are only supposed to return a single value - for source in self.sources: - try: - return source.get_property_values(prop) - except PropertyError: - pass - except LookupError: - pass - raise LookupError - - def _get_vcs_repos(self): - for source in self.sources: - if DataProperty.VCS_REPOS in source.get_supported_properties(): - try: - iterator = source.get_property_values(DataProperty.VCS_REPOS) - except LookupError: - pass - else: - yield from iterator - - def _get_repo_list_sources(self): - for source in self.sources: - if DataProperty.REPO_LIST_SOURCES in source.get_supported_properties(): - try: - iterator = source.get_property_values(DataProperty.REPO_LIST_SOURCES) - except LookupError: - pass - else: - yield from iterator - -class RepoListManager(DataSource): - """A RepoListManager takes care of managing repo lists. - - Args: - config_manager (DataSource): The config manager from which the repo - lists come. - """ - def __init__(self, config_manager): - self.config_manager = EffectiveSource(config_manager) - self.sources = [self.config_manager] - - def exists(self): - return True - - def update(self): - excs = [self.config_manager.update()] - if DataProperty.REPO_LIST_SOURCES in self.config_manager.get_supported_properties(): - self.sources = [self.config_manager] - try: - it = self.config_manager.get_property_values(DataProperty.REPO_LIST_SOURCES) - except LookupError: - pass - else: - self.sources.extend(RemoteDataSource(rls.uri) for rls in it if rls.active) - for source in self.sources[1:]: - excs.append(source.update()) - return excs - - def get_supported_properties(self): - return {DataProperty.VCS_REPOS} - - def get_property_values(self, prop): - if prop not in self.get_supported_properties(): - raise PropertyError - assert prop == DataProperty.VCS_REPOS - return self._get_vcs_repos() - - def _get_vcs_repos(self): - assert self.config_manager == self.sources[0] - try: - # config manager may override repo lists - iterator = self.config_manager.get_property_values(DataProperty.VCS_REPOS) - except (PropertyError, LookupError): - pass - else: - yield from iterator - for source in self.sources: - if DataProperty.VCS_REPOS in source.get_supported_properties(): - try: - iterator = source.get_property_values(DataProperty.VCS_REPOS) - except LookupError: - pass - else: - for pctp in iterator: - # but repo lists aren't allowed to override anything - try: - del pctp.options['federate'] - except KeyError: - pass - if pctp.active: - yield pctp - -class EffectiveSource(DataSource): - """Wraps another ``DataSource`` and yields "unique" results suitable - for general use. - - Methods on this class, in particular ``get_property_values``, handle - ``OverridableProperty`` overrides both to avoid code duplication and - so the user doesn't have to. - - Args: - raw_source (DataSource): The raw backing source. - """ - def __init__(self, raw_source): - self.raw_source = raw_source - - def exists(self): - return self.raw_source.exists() - - def update(self): - return self.raw_source.update() - - def get_property_value(self, prop): - return self.raw_source.get_property_value(prop) - - def get_supported_properties(self): - return self.raw_source.get_supported_properties() - - def get_property_values(self, prop): - # must raise exceptions *now* - # not when the generator runs - return self._wrap_values(prop, self.raw_source.get_property_values(prop)) - - def _wrap_values(self, prop, it): - if issubclass(prop.get_type(), OverridableProperty): - seen = {} - for v in it: - k = v.as_key() - if k in seen: - continue - seen[k] = v - yield v - else: - yield from it - - def __repr__(self): - return "EffectiveSource({!r})".format(self.raw_source) diff --git a/ganarchy/db.py b/ganarchy/db.py deleted file mode 100644 index b7aa29b..0000000 --- a/ganarchy/db.py +++ /dev/null @@ -1,369 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -"""This module handles GAnarchy's database. - -Attributes: - MIGRATIONS: Migrations. -""" - -import sqlite3 - -import ganarchy.dirs -import ganarchy.data - -# FIXME this should not be used directly because it's a pain. -MIGRATIONS = { - "toml-config": ( - ( - '''UPDATE "repo_history" - SET "project" = (SELECT "git_commit" FROM "config") - WHERE "project" IS NULL''', - - '''ALTER TABLE "repos" - RENAME TO "repos_old"''', - ), - ( - '''UPDATE "repo_history" - SET "project" = NULL - WHERE "project" = (SELECT "git_commit" FROM "config")''', - - '''ALTER TABLE "repos_old" - RENAME TO "repos"''', - ), - "switches to toml config format. the old 'repos' " #cont - "table is preserved as 'repos_old'" - ), - "better-project-management": ( - ( - '''ALTER TABLE "repos" - ADD COLUMN "branch" TEXT''', - - '''ALTER TABLE "repos" - ADD COLUMN "project" TEXT''', - - '''CREATE UNIQUE INDEX "repos_url_branch_project" - ON "repos" ("url", "branch", "project")''', - - '''CREATE INDEX "repos_project" - ON "repos" ("project")''', - - '''ALTER TABLE "repo_history" - ADD COLUMN "branch" TEXT''', - - '''ALTER TABLE "repo_history" - ADD COLUMN "project" TEXT''', - - '''CREATE INDEX "repo_history_url_branch_project" - ON "repo_history" ("url", "branch", "project")''', - ), - ( - '''DELETE FROM "repos" - WHERE "branch" IS NOT NULL OR "project" IS NOT NULL''', - '''DELETE FROM "repo_history" - WHERE "branch" IS NOT NULL OR "project" IS NOT NULL''', - ), - "supports multiple projects, and allows choosing " #cont - "non-default branches" - ), - "test": ( - ( - '''-- apply''', - ), - ( - '''-- revert''', - ), - "does nothing" - ) - } - -class Database: - """A database connection/session, returned by ``connect_database``. - - Some methods may require repos to be loaded. - """ - - def __init__(self, conn): - self.conn = conn - - def initialize(self): - """Initializes the database tables as expected by GAnarchy. - """ - c = self.conn.cursor() - c.execute(''' - CREATE TABLE "repo_history" ( - "entry" INTEGER PRIMARY KEY ASC AUTOINCREMENT, - "url" TEXT, - "count" INTEGER, - "head_commit" TEXT, - "branch" TEXT, - "project" TEXT - ) - ''') - c.execute(''' - CREATE INDEX "repo_history_url_branch_project" - ON "repo_history" ("url", "branch", "project") - ''') - self.conn.commit() - c.close() - - def apply_migration(self, migration): - """Applies a migration, by name. - - WARNING: Destructive operation. - - Args: - migration (str): The name of the migration. - """ - c = self.conn.cursor() - for migration in MIGRATIONS[migration][0]: - c.execute(migration) - self.conn.commit() - c.close() - - def revert_migration(self, migration): - """Reverts a previously-applied migration, by name. - - WARNING: Destructive operation. - - Args: - migration (str): The name of the migration. - """ - c = self.conn.cursor() - for migration in MIGRATIONS[migration][1]: - c.execute(migration) - self.conn.commit() - c.close() - - def load_repos(self, effective_repo_list): - """Loads repos from repo list. - - Must be done once for each instance of Database. - - Args: - effective_repo_list (ganarchy.data.DataSource): The data - source for the repo list. - """ - c = self.conn.cursor() - c.execute(''' - CREATE TEMPORARY TABLE "repos" ( - "url" TEXT, - "active" INT, - "branch" TEXT, - "project" TEXT, - "federate" INT - ) - ''') - c.execute(''' - CREATE UNIQUE INDEX "temp"."repos_url_branch_project" - ON "repos" ("url", "branch", "project") - ''') - c.execute(''' - CREATE INDEX "temp"."repos_project" - ON "repos" ("project") - ''') - c.execute(''' - CREATE INDEX "temp"."repos_active" - ON "repos" ("active") - ''') - for repo in effective_repo_list.get_property_values( - ganarchy.data.DataProperty.VCS_REPOS - ): - if repo.active: - c.execute( - '''INSERT INTO "repos" VALUES (?, ?, ?, ?, ?)''', - (repo.uri, 1, repo.branch, repo.project_commit, int(repo.federate)) - ) - self.conn.commit() - c.close() - - def insert_activity(self, project_commit, uri, branch, head, count): - """Inserts activity of a repo-branch. - - Args: - project_commit: The project commit. - uri: The repo uri. - branch: The branch. - head: The latest known head commit. - count: The number of new commits. - """ - self.insert_activities([(project_commit, uri, branch, head, count)]) - - def insert_activities(self, activities): - """Inserts activities of repo-branches. - - Args: - activities: List of tuple. The tuple must match up with the - argument order specified by ``insert_activity``. - """ - c = self.conn.cursor() - c.executemany( - ''' - INSERT INTO "repo_history" ( - "project", - "url", - "branch", - "head_commit", - "count" - ) - VALUES (?, ?, ?, ?, ?) - ''', - activities - ) - self.conn.commit() - c.close() - - def list_projects(self): - """Lists loaded projects. - - Repos must be loaded first. - - Yields: - str: Project commit of each project. - """ - c = self.conn.cursor() - try: - for (project,) in c.execute( - '''SELECT DISTINCT "project" FROM "repos" ''' - ): - yield project - finally: - c.close() - - def list_repobranches(self, project_commit): - """Lists repo-branches of a project. - - Repos must be loaded first. - - Results are sorted by recent activity. - - Args: - project_commit: The project commit. - - Yields: - A 3-tuple holding the URI, branch name, and last known head - commit. - """ - c = self.conn.cursor() - try: - for (e, url, branch, head_commit) in c.execute( - ''' - SELECT "max"("e"), "url", "branch", "head_commit" - FROM ( - SELECT - "max"("T1"."entry") "e", - "T1"."url", - "T1"."branch", - "T1"."head_commit" - FROM "repo_history" "T1" - WHERE ( - SELECT "active" - FROM "repos" "T2" - WHERE - "url" = "T1"."url" - AND "branch" IS "T1"."branch" - AND "project" IS ?1 - ) - GROUP BY "T1"."url", "T1"."branch" - UNION - SELECT null, "T3"."url", "T3"."branch", null - FROM "repos" "T3" - WHERE "active" AND "project" IS ?1 - ) - GROUP BY "url", "branch" - ORDER BY "e" - ''', - (project_commit,) - ): - yield url, branch, head_commit - finally: - c.close() - - def list_repobranch_activity(self, project_commit, uri, branch): - """Lists activity of a repo-branch. - - Args: - project_commit: The project commit. - uri: The repo uri. - branch: The branch. - - Returns: - list of int: Number of commits between updates. - """ - c = self.conn.cursor() - history = c.execute( - ''' - SELECT "count" - FROM "repo_history" - WHERE - "url" = ? - AND "branch" IS ? - AND "project" IS ? - ORDER BY "entry" ASC - ''', - (uri, branch, project_commit) - ).fetchall() - history = [x for [x] in history] - c.close() - return history - - def should_repo_federate(self, project_commit, uri, branch): - """Returns whether a repo should federate. - - Args: - project_commit: The project commit. - uri: The repo uri. - branch: The branch. - - Returns: - bool, optional: Whether the repo should federate, or None if it - doesn't exist. - """ - c = self.conn.cursor() - federate = c.execute( - ''' - SELECT "federate" - FROM "repos" - WHERE - "url" = ? - AND "branch" IS ? - AND "project" IS ? - ''', - (uri, branch, project_commit) - ).fetchall() - try: - ((federate,),) = federate - federate = bool(federate) - except ValueError: - federate = None - c.close() - return federate - - def close(self): - """Closes the database. - """ - self.conn.close() - -def connect_database(effective_config): - """Opens the database specified by the given config. - - Args: - effective_config (ganarchy.data.DataSource): The data source - for the config. - """ - del effective_config # currently unused, intended for the future - conn = sqlite3.connect(ganarchy.dirs.DATA_HOME + "/ganarchy.db") - return Database(conn) diff --git a/ganarchy/dirs.py b/ganarchy/dirs.py deleted file mode 100644 index 7973126..0000000 --- a/ganarchy/dirs.py +++ /dev/null @@ -1,53 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -"""This module handles GAnarchy's config, data and cache directories. - -These are not XDG dirs. They're GAnarchy dirs. They're based on XDG -dirs but they're not XDG dirs. - -Attributes: - DATA_HOME (str): GAnarchy data home. - CACHE_HOME (str): GAnarchy cache home. - CONFIG_HOME (str): GAnarchy config home. - CONFIG_DIRS (list of str): GAnarchy config dirs. -""" - -import os - -# need to check for unset or empty, ``.get`` only handles unset. - -DATA_HOME = os.environ.get('XDG_DATA_HOME', '') -if not DATA_HOME: - DATA_HOME = os.environ['HOME'] + '/.local/share' -DATA_HOME = DATA_HOME + "/ganarchy" - -CACHE_HOME = os.environ.get('XDG_CACHE_HOME', '') -if not CACHE_HOME: - CACHE_HOME = os.environ['HOME'] + '/.cache' -CACHE_HOME = CACHE_HOME + "/ganarchy" - -CONFIG_HOME = os.environ.get('XDG_CONFIG_HOME', '') -if not CONFIG_HOME: - CONFIG_HOME = os.environ['HOME'] + '/.config' -CONFIG_HOME = CONFIG_HOME + "/ganarchy" - -CONFIG_DIRS = os.environ.get('XDG_CONFIG_DIRS', '') -if not CONFIG_DIRS: - CONFIG_DIRS = '/etc/xdg' -# TODO check if this is correct -CONFIG_DIRS = [config_dir + "/ganarchy" for config_dir in CONFIG_DIRS.split(':')] - diff --git a/ganarchy/git.py b/ganarchy/git.py deleted file mode 100644 index f8ccfcd..0000000 --- a/ganarchy/git.py +++ /dev/null @@ -1,173 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -"""Git abstraction. -""" - -# Errors are raised when we can't provide an otherwise valid result. -# For example, we return 0 for counts instead of raising, but raise -# instead of returning empty strings for commit hashes and messages. - -import subprocess - -class GitError(Exception): - """Raised when a git operation fails, generally due to a - missing commit or branch, or network connection issues. - """ - pass - -class Git: - def __init__(self, path): - self.path = path - self.base = ("git", "-C", path) - - def create(self): - """Creates the local repo. - - Can safely be called on an existing repo. - """ - subprocess.call(self.base + ("init", "-q")) - - - def check_history(self, local_head, commit): - """Checks if the local head contains commit in its history. - Raises if it doesn't. - - Args: - local_head (str): Name of local head. - commit (str): Commit hash. - - Raises: - GitError: If an error occurs. - """ - try: - subprocess.run( - self.base + ("merge-base", "--is-ancestor", commit, local_head), - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - except subprocess.CalledProcessError as e: - raise GitError("check history") from e - - def check_branchname(self, branchname): - """Checks if the given branchname is a valid branch name. - Raises if it isn't. - - Args: - branchname (str): Name of branch. - - Raises: - GitError: If an error occurs. - """ - try: - # TODO check that this rstrip is safe - out = subprocess.run( - self.base + ("check-ref-format", "--branch", branchname), - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ).stdout.decode("utf-8").rstrip('\r\n') - # protect against @{-1}/@{-n} ("previous checkout operation") - # is also fairly future-proofed, I hope? - if out != branchname: - raise GitError("check branchname", out, branchname) - except subprocess.CalledProcessError as e: - raise GitError("check branchname") from e - - def force_fetch(self, url, remote_head, local_head): - """Fetches a remote head into a local head. - - If the local head already exists, it is replaced. - - Args: - url (str): Remote url. - remote_head (str): Name of remote head. - local_head (str): Name of local head. - - Raises: - GitError: If an error occurs. - """ - try: - subprocess.run( - self.base + ("fetch", "-q", url, "+" + remote_head + ":" + local_head), - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - except subprocess.CalledProcessError as e: - raise GitError(e.output) from e - - def get_count(self, first_hash, last_hash): - """Returns a count of the commits added since ``first_hash`` - up to ``last_hash``. - - Args: - first_hash (str): A commit. - last_hash (str): Another commit. - - Returns: - int: A count of commits added between the hashes, or 0 - if an error occurs. - """ - try: - res = subprocess.run( - self.base + ("rev-list", "--count", first_hash + ".." + last_hash, "--"), - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ).stdout.decode("utf-8").strip() - return int(res) - except subprocess.CalledProcessError as e: - return 0 - - def get_hash(self, target): - """Returns the commit hash for a given target. - - Args: - target (str): a refspec. - - Raises: - GitError: If an error occurs. - """ - try: - return subprocess.run( - self.base + ("show", target, "-s", "--format=format:%H", "--"), - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ).stdout.decode("utf-8") - except subprocess.CalledProcessError as e: - raise GitError("") from e - - def get_commit_message(self, target): - """Returns the commit message for a given target. - - Args: - target (str): a refspec. - - Raises: - GitError: If an error occurs. - """ - try: - return subprocess.run( - self.base + ("show", target, "-s", "--format=format:%B", "--"), - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ).stdout.decode("utf-8", "replace") - except subprocess.CalledProcessError as e: - raise GitError("") from e diff --git a/ganarchy/templating/__init__.py b/ganarchy/templating/__init__.py deleted file mode 100644 index d6f6d0c..0000000 --- a/ganarchy/templating/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -"""Templates. - -""" - -# TODO write me diff --git a/ganarchy/templating/environment.py b/ganarchy/templating/environment.py deleted file mode 100644 index 0258f4d..0000000 --- a/ganarchy/templating/environment.py +++ /dev/null @@ -1,31 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -import jinja2 - -import ganarchy.templating.templates -import ganarchy.templating.toml - -def get_env(): - env = jinja2.Environment( - loader=ganarchy.templating.templates.get_template_loader(), - autoescape=False, - # aka please_stop_mangling_my_templates=True - keep_trailing_newline=True - ) - env.filters['tomlescape'] = ganarchy.templating.toml.tomlescape - env.filters['tomle'] = env.filters['tomlescape'] - return env diff --git a/ganarchy/templating/templates.py b/ganarchy/templating/templates.py deleted file mode 100644 index 1e9c074..0000000 --- a/ganarchy/templating/templates.py +++ /dev/null @@ -1,128 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -import jinja2 - -import ganarchy.dirs - -def get_template_loader(): - return jinja2.ChoiceLoader([ - jinja2.FileSystemLoader([ganarchy.dirs.CONFIG_HOME + "/templates"] + [config_dir + "/templates" for config_dir in ganarchy.dirs.CONFIG_DIRS]), - jinja2.DictLoader({ - ## index.html - 'index.html': """<!DOCTYPE html> -<html lang="en"> - <head> - <meta charset="utf-8" /> - <!-- - GAnarchy - project homepage generator - Copyright (C) 2019 Soni L. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <https://www.gnu.org/licenses/>. - --> - <title>{{ ganarchy.title|e }}</title> - <meta name="description" content="{{ ganarchy.title|e }}" /> - <!--if your browser doesn't like the following, use a different browser.--> - <script type="application/javascript" src="/index.js"></script> - </head> - <body> - <h1>{{ ganarchy.title|e }}</h1> - <p>This is {{ ganarchy.title|e }}. Currently tracking the following projects:</p> - <ul> - {% for project in ganarchy.projects -%}{% if project.exists -%} - <li><a href="{{ (ganarchy.base_url[:-1] + ganarchy.base_url[-1:].rsplit('/',1)[0])|e }}/project/{{ project.commit|e }}">{{ project.title|e }}</a>: {{ project.description|e }}</li> - {% endif -%}{% endfor -%} - </ul> - <p>Powered by <a href="https://ganarchy.autistic.space/">GAnarchy</a>. AGPLv3-licensed. <a href="https://cybre.tech/SoniEx2/ganarchy">Source Code</a>.</p> - <p> - <a href="{{ ganarchy.base_url|e }}" onclick="event.preventDefault(); navigator.registerProtocolHandler('web+ganarchy', this.href + '?url=%s', 'GAnarchy');">Register web+ganarchy: URI handler</a> - (Makes navigating between GAnarchy instances easier). - </p> - </body> -</html> -""", - ## index.toml - 'index.toml': """# Generated by GAnarchy - -{%- for project in database.list_projects() %} -[projects.{{project}}] -{%- for repo_url, branch, _head_commit in database.list_repobranches(project) %} -{%- if database.should_repo_federate(project, repo_url, branch) %} -"{{repo_url|tomle}}".{% if not branch %}HEAD{% else %}"{{branch|tomle}}"{% endif %} = { active=true } -{%- endif %} -{%- endfor %} -{% endfor -%} -""", - ## project.html - # FIXME convert to project.title/etc instead of project_title/etc. - 'project.html': """<!DOCTYPE html> -<html lang="en"> - <head> - <meta charset="utf-8" /> - <!-- - GAnarchy - project homepage generator - Copyright (C) 2019 Soni L. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <https://www.gnu.org/licenses/>. - --> - <title>{{ project_title|e }}</title> - {% if project_desc %}<meta name="description" content="{{ project_desc|e }}" />{% endif %} - <style type="text/css">.branchname { color: #808080; font-style: italic; }</style> - </head> - <body> - <h1>{{ project_title|e }}</h1> - <p>Tracking <span id="project_commit"><a href="web+ganarchy:{{ project_commit }}">{{ project_commit }}</a></span></p> - <div id="project_body"><p>{{ project_body|e|replace("\n\n", "</p><p>") }}</p></div> - <ul> - {% for url, msg, img, branch in repos -%} - <li><a href="{{ url|e }}">{{ url|e }}</a>{% if branch %} <span class="branchname">[{{ branch|e }}]</span>{% endif %}: {{ msg|e }}</li> - {% endfor -%} - </ul> - <p>Powered by <a href="https://ganarchy.autistic.space/">GAnarchy</a>. AGPLv3-licensed. <a href="https://cybre.tech/SoniEx2/ganarchy">Source Code</a>.</p> - <p> - <a href="{{ base_url|e }}">Main page</a>. - <a href="{{ base_url|e }}" onclick="event.preventDefault(); navigator.registerProtocolHandler('web+ganarchy', this.href + '?url=%s', 'GAnarchy');">Register web+ganarchy: URI handler</a> - (Makes navigating between GAnarchy instances easier). - </p> - </body> -</html> -""", - ## history.svg FIXME - 'history.svg': """""", - }) - ]) diff --git a/ganarchy/templating/toml.py b/ganarchy/templating/toml.py deleted file mode 100644 index 431125d..0000000 --- a/ganarchy/templating/toml.py +++ /dev/null @@ -1,33 +0,0 @@ -# This file is part of GAnarchy - decentralized project hub -# Copyright (C) 2020 Soni L. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -_tomletrans = str.maketrans({ - 0: '\\u0000', 1: '\\u0001', 2: '\\u0002', 3: '\\u0003', 4: '\\u0004', - 5: '\\u0005', 6: '\\u0006', 7: '\\u0007', 8: '\\b', 9: '\\t', 10: '\\n', - 11: '\\u000B', 12: '\\f', 13: '\\r', 14: '\\u000E', 15: '\\u000F', - 16: '\\u0010', 17: '\\u0011', 18: '\\u0012', 19: '\\u0013', 20: '\\u0014', - 21: '\\u0015', 22: '\\u0016', 23: '\\u0017', 24: '\\u0018', 25: '\\u0019', - 26: '\\u001A', 27: '\\u001B', 28: '\\u001C', 29: '\\u001D', 30: '\\u001E', - 31: '\\u001F', '"': '\\"', '\\': '\\\\' - }) - -def tomlescape(value): - """Escapes a string for use in a TOML string. - - Returns: - str: The escaped string. - """ - return value.translate(_tomletrans) |