#!/usr/bin/env python3 # SPDX-FileCopyrightText: 2021 Luca Beltrame # SPDX-License-Identifier: BSD-3-clause import argparse import logging import json from pathlib import Path from urllib.parse import quote import git import requests import sarge API_URL = "https://invent.kde.org/api/v4/projects/" class GitHashCache: def __init__(self, cache_file): self.cache = cache_file self._data = dict() def __getitem__(self, key): if key not in self._data: return "" return self._data[key] def __setitem__(self, key, value): self._data[key] = value def get(self, key, *args, **kwargs): return self._data.get(key, *args, **kwargs) def save(self): logging.debug("Saving pickled data") with open(self.cache, "w") as handle: json.dump(self._data, handle, indent=4) def load(self): if not Path(self.cache).exists(): logging.debug("File cache not found, not loading") return with open(self.cache) as handle: self._data = json.load(handle) def project_exists(project): # We want / to get quoted, so put safe to "" project_name = quote(project, safe="") request = requests.get(API_URL + project_name) if request: return True return False def lsremote(url): remote_refs = {} gitcmd = git.cmd.Git() for ref in gitcmd.ls_remote(url).split('\n'): hash_ref_list = ref.split('\t') remote_refs[hash_ref_list[1]] = hash_ref_list[0] return remote_refs def get_remote_hash(url, branch="master"): refs = "refs/heads/{}".format(branch) return lsremote(url)[refs] def run_osc(repository, package_name): cmd = "osc service remoterun {0} {1}" cmd = cmd.format(repository, package_name) logging.debug("Running %s", cmd) logging.info("Updating package %s", package_name) pid = sarge.run(cmd) if pid.returncode != 0: logging.error("Error during service run, package %s", package_name) return False logging.debug("Package %s complete", package_name) return True def update_package(hash_data, package_name, remote_name, obs_repository, branch): repo_name = "https://invent.kde.org/{}".format(remote_name) if not project_exists(remote_name): logging.warning("Repository %s not found, skipping", remote_name) return remote_hash = get_remote_hash(repo_name, branch) repo_hashes = hash_data.get(obs_repository) if hash_data.get(obs_repository) is None: logging.debug("No prior data - initializing empty") hash_data[obs_repository] = dict() current_hash = hash_data[obs_repository].get(remote_name, "") logging.debug("Package %s, theirs %s, ours %s", remote_name, remote_hash, current_hash) if remote_hash != current_hash: logging.debug("Hash doesn't match, updating") if run_osc(obs_repository, package_name): hash_data[obs_repository][remote_name] = remote_hash hash_data.save() def update_packages(cache_file, repo_mapping_file): hash_data = GitHashCache(cache_file) hash_data.load() with open(repo_mapping_file, "r") as mapping: repo_data = json.load(mapping) for obs_repository, branch_data in repo_data.items(): logging.info("Updating packages for %s", obs_repository) for package in branch_data: kde_name = package["kde"] obs_name = package["obs"] branch = package["branch"] package_name = Path(kde_name).name logging.debug("Updating package %s (%s)", package_name, obs_name) logging.debug("Using branch %s", branch) update_package(hash_data, obs_name, kde_name, obs_repository, branch) logging.debug("Saving data") hash_data.save() def main(): parser = argparse.ArgumentParser() parser.add_argument( "--cache-file", help="Location of the cache file", default=Path.home() / ".local/share/obs_repo_cache.json") parser.add_argument("mapping_file", help="KDE:OBS repository mapping file") parser.add_argument("--debug", help="Show debugging output", action="store_true") options = parser.parse_args() level = logging.INFO if not options.debug else logging.DEBUG logging.basicConfig(format='%(levelname)s - %(message)s', level=level) cache_file = options.cache_file update_packages(cache_file, options.mapping_file) logging.info("Complete") if __name__ == "__main__": main()