177 lines
4.7 KiB
Python
Executable file
177 lines
4.7 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# SPDX-FileCopyrightText: 2021 Luca Beltrame <lbeltrame@kde.org>
|
|
# SPDX-License-Identifier: BSD-3-clause
|
|
|
|
import argparse
|
|
import logging
|
|
import json
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
import git
|
|
import requests
|
|
import sarge
|
|
|
|
API_URL = "https://invent.kde.org/api/v4/projects/"
|
|
|
|
|
|
class GitHashCache:
|
|
|
|
def __init__(self, cache_file):
|
|
self.cache = cache_file
|
|
self._data = dict()
|
|
|
|
def __getitem__(self, key):
|
|
|
|
if key not in self._data:
|
|
return ""
|
|
|
|
return self._data[key]
|
|
|
|
def __setitem__(self, key, value):
|
|
self._data[key] = value
|
|
|
|
def get(self, key, *args, **kwargs):
|
|
return self._data.get(key, *args, **kwargs)
|
|
|
|
def save(self):
|
|
logging.debug("Saving pickled data")
|
|
with open(self.cache, "w") as handle:
|
|
json.dump(self._data, handle, indent=4)
|
|
|
|
def load(self):
|
|
|
|
if not Path(self.cache).exists():
|
|
logging.debug("File cache not found, not loading")
|
|
return
|
|
|
|
with open(self.cache) as handle:
|
|
self._data = json.load(handle)
|
|
|
|
|
|
def project_exists(project):
|
|
# We want / to get quoted, so put safe to ""
|
|
project_name = quote(project, safe="")
|
|
request = requests.get(API_URL + project_name)
|
|
|
|
if request:
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def lsremote(url):
|
|
|
|
remote_refs = {}
|
|
gitcmd = git.cmd.Git()
|
|
for ref in gitcmd.ls_remote(url).split('\n'):
|
|
hash_ref_list = ref.split('\t')
|
|
remote_refs[hash_ref_list[1]] = hash_ref_list[0]
|
|
|
|
return remote_refs
|
|
|
|
|
|
def get_remote_hash(url, branch="master"):
|
|
refs = "refs/heads/{}".format(branch)
|
|
return lsremote(url)[refs]
|
|
|
|
|
|
def run_osc(repository, package_name):
|
|
|
|
cmd = "osc service remoterun {0} {1}"
|
|
cmd = cmd.format(repository, package_name)
|
|
|
|
logging.debug("Running %s", cmd)
|
|
logging.info("Updating package %s", package_name)
|
|
|
|
pid = sarge.run(cmd)
|
|
|
|
if pid.returncode != 0:
|
|
logging.error("Error during service run, package %s", package_name)
|
|
return False
|
|
|
|
logging.debug("Package %s complete", package_name)
|
|
return True
|
|
|
|
|
|
def update_package(hash_data, package_name,
|
|
remote_name, obs_repository,
|
|
branch):
|
|
|
|
repo_name = "https://invent.kde.org/{}".format(remote_name)
|
|
|
|
if not project_exists(remote_name):
|
|
logging.warning("Repository %s not found, skipping", remote_name)
|
|
return
|
|
|
|
remote_hash = get_remote_hash(repo_name, branch)
|
|
repo_hashes = hash_data.get(obs_repository)
|
|
|
|
if hash_data.get(obs_repository) is None:
|
|
logging.debug("No prior data - initializing empty")
|
|
hash_data[obs_repository] = dict()
|
|
|
|
current_hash = hash_data[obs_repository].get(remote_name, "")
|
|
|
|
logging.debug("Package %s, theirs %s, ours %s",
|
|
remote_name, remote_hash, current_hash)
|
|
|
|
if remote_hash != current_hash:
|
|
logging.debug("Hash doesn't match, updating")
|
|
if run_osc(obs_repository, package_name):
|
|
hash_data[obs_repository][remote_name] = remote_hash
|
|
hash_data.save()
|
|
|
|
|
|
def update_packages(cache_file, repo_mapping_file):
|
|
|
|
hash_data = GitHashCache(cache_file)
|
|
hash_data.load()
|
|
|
|
with open(repo_mapping_file, "r") as mapping:
|
|
|
|
repo_data = json.load(mapping)
|
|
|
|
for obs_repository, branch_data in repo_data.items():
|
|
|
|
logging.info("Updating packages for %s", obs_repository)
|
|
|
|
for package in branch_data:
|
|
kde_name = package["kde"]
|
|
obs_name = package["obs"]
|
|
branch = package["branch"]
|
|
package_name = Path(kde_name).name
|
|
logging.debug("Updating package %s (%s)",
|
|
package_name, obs_name)
|
|
logging.debug("Using branch %s", branch)
|
|
update_package(hash_data, obs_name, kde_name, obs_repository,
|
|
branch)
|
|
|
|
logging.debug("Saving data")
|
|
hash_data.save()
|
|
|
|
|
|
def main():
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--cache-file", help="Location of the cache file",
|
|
default=Path.home() / ".local/share/obs_repo_cache.json")
|
|
parser.add_argument("mapping_file", help="KDE:OBS repository mapping file")
|
|
parser.add_argument("--debug", help="Show debugging output",
|
|
action="store_true")
|
|
options = parser.parse_args()
|
|
|
|
level = logging.INFO if not options.debug else logging.DEBUG
|
|
|
|
logging.basicConfig(format='%(levelname)s - %(message)s',
|
|
level=level)
|
|
|
|
cache_file = options.cache_file
|
|
|
|
update_packages(cache_file, options.mapping_file)
|
|
logging.info("Complete")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|