Experimental new config parser

2 files changed, 152 insertions, 76 deletions

diff --git a/ganarchy.py b/ganarchy.py
index bcc094e..fe03860 100755
--- a/ganarchy.py
+++ b/ganarchy.py
@@ -16,15 +16,22 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-import sqlite3
-import click
-import os
-import subprocess
+import abc
+import builtins
 import hashlib
 import hmac
-import jinja2
+import os
 import re
+import sqlite3
+import subprocess
+
+import click
+import jinja2
 import qtoml
+import requests
+
+import abdl
+
 from collections import defaultdict
 from urllib.parse import urlparse
 
@@ -321,17 +328,18 @@ class Repo:
     def update_metadata(self):
         self.message = GIT.get_commit_message(self.branchname)
 
-    def update(self):
+    def update(self, updating=True):
         """
         Updates the git repo, returning new metadata.
         """
-        try:
-            subprocess.check_output(["git", "-C", cache_home, "fetch", "-q", self.url, "+" + self.head + ":" + self.branchname], stderr=subprocess.STDOUT)
-        except subprocess.CalledProcessError as e:
-            # This may error for various reasons, but some are important: dead links, etc
-            click.echo(e.output, err=True)
-            self.erroring = True
-            return None
+        if updating:
+            try:
+                subprocess.check_output(["git", "-C", cache_home, "fetch", "-q", self.url, "+" + self.head + ":" + self.branchname], stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                # This may error for various reasons, but some are important: dead links, etc
+                click.echo(e.output, err=True)
+                self.erroring = True
+                return None
         pre_hash = self.hash
         try:
             post_hash = GIT.get_hash(self.branchname)
@@ -348,7 +356,8 @@ class Repo:
         except subprocess.CalledProcessError:
             count = 0  # force-pushed
         try:
-            subprocess.check_call(["git", "-C", cache_home, "merge-base", "--is-ancestor", self.project_commit, self.branchname], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            if updating:
+                subprocess.check_call(["git", "-C", cache_home, "merge-base", "--is-ancestor", self.project_commit, self.branchname], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
             self.update_metadata()
             return count
         except (subprocess.CalledProcessError, GitError) as e:
@@ -392,9 +401,9 @@ class Project:
             self.title = None
             self.description = None
 
-    def update(self):
+    def update(self, updating=True):
         # TODO? check if working correctly
-        results = [(repo, repo.update()) for repo in self.repos]
+        results = [(repo, repo.update(updating)) for repo in self.repos]
         self.refresh_metadata()
         return results
 
@@ -424,17 +433,87 @@ class GAnarchy:
         if list_projects:
             projects = []
             with dbconn:
-                for (project,) in dbconn.execute('''SELECT DISTINCT "project" FROM "repos" '''): # FIXME? *maybe* sort by activity in the future
+                for (project,) in dbconn.execute('''SELECT DISTINCT "project" FROM "repos" '''):
                     projects.append(Project(dbconn, project, list_repos=list_repos))
             projects.sort(key=lambda project: project.title) # sort projects by title
             self.projects = projects
         else:
             self.projects = None
 
+class ConfigSource(abc.ABC):
+    @abc.abstractmethod
+    def update(self):
+        """Refreshes the config if necessary."""
+        pass
+
+    def is_domain_blocked(self, domain):
+        """Returns True if the given domain is blocked."""
+        return False
+
+    @abc.abstractmethod
+    def get_project_commit_tree_paths(self):
+        """Returns an iterator of (project, URI, branch, options) tuples.
+
+        project is the project commit hash, URI is the repo URI, branch is the branch name and
+        options are the options for the given project commit-tree path."""
+        pass
+
+    @abc.abstractmethod
+    def __getitem__(self, key):
+        raise KeyError
+
+class FileConfigSource(ConfigSource):
+    def __init__(self, filename):
+        self.exists = False
+        self.last_updated = None
+        self.filename = filename
+        self.tomlobj = None
+        self.update()
+
+    def update(self):
+        try:
+            updtime = self.last_updated
+            self.last_updated = os.stat(self.filename).st_mtime
+            if not self.exists or updtime != self.last_updated:
+                with open(self.filename) as f:
+                    self.tomlobj = qtoml.load(f)
+            self.exists = True
+        except OSError:
+            return
+
+    def get_project_commit_tree_paths(self):
+        for (project, pobj) in self.tomlobj['projects'].items():
+            for (uri, uobj) in pobj.items():
+                for (branch, options) in uobj.items():
+                    yield (project, uri, branch, options)
+
+    def __getitem__(self, key):
+        return super().__getitem__(self, key)
+
+class RemoteConfigSource(ConfigSource):
+    def __init__(self, path):
+        self.path = path
+
+    def update(self):
+        raise NotImplementedError
+
+    def get_project_commit_tree_paths(self):
+        for (project, pobj) in self.tomlobj['projects'].items():
+            for (uri, uobj) in pobj.items():
+                for (branch, options) in uobj.items():
+                    yield (project, uri, branch, options)
+
 class Config:
+    # sanitize = skip invalid entries
+    # validate = error on invalid entries
+    CONFIG_PATTERN_SANITIZE = abdl.compile("->commit/[0-9a-fA-F]{40}|[0-9a-fA-F]{64}/?:?$dict->url:?$dict->branch:?$dict", dict(vars(builtins), **globals()))
+    # TODO use a validating pattern instead?
+    CONFIG_PATTERN = abdl.compile("->commit->url->branch", dict(vars(builtins), **globals()))
+
     def __init__(self, toml_file, base=None, remove=True):
         self.projects = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(dict))))
         config_data = qtoml.load(toml_file)
+        self.remote_configs = config_data.get('config_srcs', [])
         self.title = config_data.get('title', '')
         self.base_url = config_data.get('base_url', '')
         # TODO blocked domains (but only read them from config_data if remove is True)
@@ -453,62 +532,36 @@ class Config:
         self._update_projects(projects, remove=remove)
 
     def _update_projects(self, projects, remove, sanitize=True):
-        for (project_commit, repos) in projects.items():
-            if sanitize and not isinstance(repos, dict):
-                # TODO emit warnings?
-                continue
-            if sanitize and not re.fullmatch("[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", project_commit): # future-proofing: sha256 support
-                # TODO emit warnings?
+        if sanitize:
+            m = Config.CONFIG_PATTERN_SANITIZE.match(projects)
+        else:
+            m = Config.CONFIG_PATTERN.match(projects)
+        for v in m:
+            commit, repo_url, branchname, options = v['commit'][0], v['url'][0], v['branch'][0], v['branch'][1]
+            try:
+                u = urlparse(repo_url)
+                if not u:
+                    raise ValueError
+                getattr(u, 'port') # raises ValueError if port is invalid
+                if u.scheme not in ('http', 'https'):
+                    raise ValueError
+                if (u.hostname in self.blocked_domains) or (u.hostname.endswith(self.blocked_domain_suffixes)):
+                    raise ValueError
+            except ValueError:
                 continue
-            project = self.projects[project_commit]
-            for (repo_url, branches) in repos.items():
-                if sanitize and not isinstance(branches, dict):
-                    # TODO emit warnings?
-                    continue
-                try:
-                    u = urlparse(repo_url)
-                    if not u:
-                        raise ValueError
-                    getattr(u, 'port') # raises ValueError if port is invalid
-                    if u.scheme in ('file', ''):
-                        raise ValueError
-                    if (u.hostname in self.blocked_domains) or (u.hostname.endswith(self.blocked_domain_suffixes)):
-                        raise ValueError
-                except ValueError:
-                    if sanitize:
-                        # TODO emit warnings?
-                        continue
-                    else:
-                        raise
-                repo = project[repo_url]
-                for (branchname, options) in branches.items():
-                    if sanitize and not isinstance(options, dict):
-                        # TODO emit warnings?
-                        continue
-                    if branchname == "HEAD":
-                        if sanitize:
-                            # feels weird, but generally makes things easier
-                            # DO NOT emit warnings here. this is deliberate.
-                            branchname = None
-                        else:
-                            raise ValueError
-                    branch = repo[branchname]
-                    active = options.get('active', False)
-                    if active not in (True, False):
-                        if sanitize:
-                            # TODO emit warnings?
-                            continue
-                        else:
-                            raise ValueError
-                    ## | remove | branch.active | options.active | result |
-                    ## |    x   |     false     |     false      |  false |
-                    ## |    x   |     false     |     true       |  true  |
-                    ## |    x   |     true      |     true       |  true  |
-                    ## |  false |     true      |     false      |  true  |
-                    ## |  true  |     true      |     false      |  false |
-                    branch['active'] = branch.get('active', False) or active
-                    if remove and not active:
-                        branch['active'] = False
+            if branchname == "HEAD":
+                branchname = None
+            active = options.get('active', False) == True
+            ## | remove | branch.active | options.active | result |
+            ## |    x   |     false     |     false      |  false |
+            ## |    x   |     false     |     true       |  true  |
+            ## |    x   |     true      |     true       |  true  |
+            ## |  false |     true      |     false      |  true  |
+            ## |  true  |     true      |     false      |  false |
+            branch = self.projects[commit][repo_url][branchname]
+            branch['active'] = branch.get('active', False) or active
+            if remove and not active:
+                branch['active'] = False
 
 def debug():
     @ganarchy.group()
@@ -522,6 +575,10 @@ def debug():
         click.echo('Cache home: {}'.format(cache_home))
         click.echo('Data home: {}'.format(data_home))
 
+    @debug.command()
+    def configs():
+        pass
+
 debug()
 
 @ganarchy.command()
@@ -542,9 +599,26 @@ def merge_configs(skip_errors, files):
         template = env.get_template('index.toml')
         click.echo(template.render(config=config))
 
+def update_remote_configs():
+    pass
+
+@ganarchy.command()
+@click.argument('out', required=True)
+def run(out):
+    """Runs ganarchy standalone.
+
+    This will run ganarchy so it regularly updates the output directory given by OUT.
+    Additionally, it'll also search for the following hooks in its config dirs:
+
+        - post_object_update_hook - executed after an object is updated.
+
+        - post_update_cycle_hook - executed after all objects in an update cycle are updated."""
+    pass
+
 @ganarchy.command()
+@click.option('--update/--no-update', default=True)
 @click.argument('project', required=False)
-def cron_target(project):
+def cron_target(update, project):
     """Runs ganarchy as a cron target."""
     conf = None
     # reverse order is intentional
@@ -585,7 +659,7 @@ def cron_target(project):
     generate_html = []
     c = conn.cursor()
     p = Project(conn, project, list_repos=True)
-    results = p.update()
+    results = p.update(update)
     for (repo, count) in results:
         if count is not None:
             entries.append((repo.url, count, repo.hash, repo.branch, project))
@@ -593,8 +667,9 @@ def cron_target(project):
     # sort stuff twice because reasons
     entries.sort(key=lambda x: x[1], reverse=True)
     generate_html.sort(key=lambda x: x[2], reverse=True)
-    c.executemany('''INSERT INTO "repo_history" ("url", "count", "head_commit", "branch", "project") VALUES (?, ?, ?, ?, ?)''', entries)
-    conn.commit()
+    if update:
+        c.executemany('''INSERT INTO "repo_history" ("url", "count", "head_commit", "branch", "project") VALUES (?, ?, ?, ?, ?)''', entries)
+        conn.commit()
     html_entries = []
     for (url, msg, count, branch) in generate_html:
         history = c.execute('''SELECT "count" FROM "repo_history" WHERE "url" = ? AND "branch" IS ? AND "project" IS ? ORDER BY "entry" ASC''', (url, branch, project)).fetchall()
diff --git a/requirements.txt b/requirements.txt
index 9ae62ea..2f964c0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@ Click==7.0
 Jinja2==2.10.1
 qtoml==0.2.4
 pyparsing==2.4.2
+requests==2.22.0