Restructure directory layout

To make this better than the unorganized mess it used to be.

obs/update_unstable.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2021 Luca Beltrame <lbeltrame@kde.org>
+# SPDX-License-Identifier: BSD-3-clause
+
+import argparse
+import logging
+import json
+from pathlib import Path
+from urllib.parse import quote
+
+import git
+import requests
+import sarge
+
+API_URL = "https://invent.kde.org/api/v4/projects/"
+
+
+class GitHashCache:
+
+    def __init__(self, cache_file):
+        self.cache = cache_file
+        self._data = dict()
+
+    def __getitem__(self, key):
+
+        if key not in self._data:
+            return ""
+
+        return self._data[key]
+
+    def __setitem__(self, key, value):
+        self._data[key] = value
+
+    def get(self, key, *args, **kwargs):
+        return self._data.get(key, *args, **kwargs)
+
+    def save(self):
+        logging.debug("Saving pickled data")
+        with open(self.cache, "w") as handle:
+            json.dump(self._data, handle, indent=4)
+
+    def load(self):
+
+        if not Path(self.cache).exists():
+            logging.debug("File cache not found, not loading")
+            return
+
+        with open(self.cache) as handle:
+            self._data = json.load(handle)
+
+
+def project_exists(project):
+    # We want / to get quoted, so put safe to ""
+    project_name = quote(project, safe="")
+    request = requests.get(API_URL + project_name)
+
+    if request:
+        return True
+
+    return False
+
+
+def lsremote(url):
+
+    remote_refs = {}
+    gitcmd = git.cmd.Git()
+    for ref in gitcmd.ls_remote(url).split('\n'):
+        hash_ref_list = ref.split('\t')
+        remote_refs[hash_ref_list[1]] = hash_ref_list[0]
+
+    return remote_refs
+
+
+def get_remote_hash(url, branch="master"):
+    refs = "refs/heads/{}".format(branch)
+    return lsremote(url)[refs]
+
+
+def run_osc(repository, package_name):
+
+    cmd = "osc service remoterun {0} {1}"
+    cmd = cmd.format(repository, package_name)
+
+    logging.debug("Running %s", cmd)
+    logging.info("Updating package %s", package_name)
+
+    pid = sarge.run(cmd)
+
+    if pid.returncode != 0:
+        logging.error("Error during service run, package %s", package_name)
+        return False
+
+    logging.debug("Package %s complete", package_name)
+    return True
+
+
+def update_package(hash_data, package_name,
+                   remote_name, obs_repository,
+                   branch):
+
+    repo_name = "https://invent.kde.org/{}".format(remote_name)
+
+    if not project_exists(remote_name):
+        logging.warning("Repository %s not found, skipping", remote_name)
+        return
+
+    remote_hash = get_remote_hash(repo_name, branch)
+    repo_hashes = hash_data.get(obs_repository)
+
+    if hash_data.get(obs_repository) is None:
+        logging.debug("No prior data - initializing empty")
+        hash_data[obs_repository] = dict()
+
+    current_hash = hash_data[obs_repository].get(remote_name, "")
+
+    logging.debug("Package %s, theirs %s, ours %s",
+                  remote_name, remote_hash, current_hash)
+
+    if remote_hash != current_hash:
+        logging.debug("Hash doesn't match, updating")
+        if run_osc(obs_repository, package_name):
+            hash_data[obs_repository][remote_name] = remote_hash
+            hash_data.save()
+
+
+def update_packages(cache_file, repo_mapping_file):
+
+    hash_data = GitHashCache(cache_file)
+    hash_data.load()
+
+    with open(repo_mapping_file, "r") as mapping:
+
+        repo_data = json.load(mapping)
+
+        for obs_repository, branch_data in repo_data.items():
+
+            logging.info("Updating packages for %s", obs_repository)
+
+            for package in branch_data:
+                kde_name = package["kde"]
+                obs_name = package["obs"]
+                branch = package["branch"]
+                package_name = Path(kde_name).name
+                logging.debug("Updating package %s (%s)",
+                              package_name, obs_name)
+                logging.debug("Using branch %s", branch)
+                update_package(hash_data, obs_name, kde_name, obs_repository,
+                               branch)
+
+    logging.debug("Saving data")
+    hash_data.save()
+
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--cache-file", help="Location of the cache file",
+        default=Path.home() / ".local/share/obs_repo_cache.json")
+    parser.add_argument("mapping_file", help="KDE:OBS repository mapping file")
+    parser.add_argument("--debug", help="Show debugging output",
+                        action="store_true")
+    options = parser.parse_args()
+
+    level = logging.INFO if not options.debug else logging.DEBUG
+
+    logging.basicConfig(format='%(levelname)s - %(message)s',
+                        level=level)
+
+    cache_file = options.cache_file
+
+    update_packages(cache_file, options.mapping_file)
+    logging.info("Complete")
+
+
+if __name__ == "__main__":
+    main()