Source code for fedcloudclient.sites

"fedcloud site" commands will read site configurations and manipulate with them. If
the local site configurations exist at ~/.config/fedcloud/site-config/, fedcloud will
read them from there, otherwise the commands will read from GitHub repository.

By default, fedcloud does not save anything on local disk, users have to save the
site configuration to local disk explicitly via "fedcloud site save-config" command.
The advantage of having local site configurations, beside faster loading, is to give
users ability to make customizations, e.g. add additional VOs, remove sites they
do not have access, and so on.

import builtins
import json
from pathlib import Path
from urllib.request import Request, urlopen

import click
import pkg_resources
import yaml
from fedcloudclient.decorators import site_params, site_vo_params
from jsonschema import validate


__LOCAL_CONFIG_DIR = ".config/fedcloud/site-config/"

__site_config_data = []

__FILE_SIZE_LIMIT = 1024 * 1024  # Max size for config files

[docs]def read_site_schema(): """ Read schema.json for validating site configuration :return: JSON object from schema.json """ file = pkg_resources.resource_stream("fedcloudclient", "schema.json") schema = json.load(file) return schema
[docs]def read_site_config(): """ Read site configurations from local config dir if exist, otherwise from default GitHub location. Storing site configurations in global variable, that will be used by other functions. Call read_local_site_config() or read_default_site_config() :return: None """ if len(__site_config_data) > 0: return config_dir = Path.home() / __LOCAL_CONFIG_DIR if config_dir.exists(): read_local_site_config(config_dir) else: read_default_site_config()
[docs]def safe_read_yaml_from_url(url, max_length): """ Safe reading from URL. Check URL and size before reading :param url: :param max_length: :return: data from URL """ if isinstance(url, str) and url.lower().startswith( "https://" ): # Only read from HTTPS location req = Request(url) else: raise SystemExit("Error: remote filename not starting with https:// : %s" % url) # URLs already checked, so ignore bandit test data = None try: with urlopen(req) as yaml_file: # nosec if int(yaml_file.headers["Content-Length"]) > max_length: raise SystemExit( "Error: remote file %s is larger than limit %d " % (url, max_length) ) data = yaml.safe_load(yaml_file) except Exception as e: print("Error during reading data from %s" % url) raise SystemExit("Exception: %s" % e) return data
[docs]def read_default_site_config(): """ Read default site configurations from GitHub. Storing site configurations in a global variable, that will be used by other functions :return: None """ __site_config_data.clear() schema = read_site_schema() site_list = safe_read_yaml_from_url(__REMOTE_CONFIG_FILE, __FILE_SIZE_LIMIT) # Check data format of site list if not isinstance(site_list, builtins.list): raise SystemExit("Error: site list is in a wrong format") for filename in site_list: site_info = safe_read_yaml_from_url(filename, __FILE_SIZE_LIMIT) # Validating site config after reading try: validate(instance=site_info, schema=schema) except Exception as e: print("Site config in file %s is in wrong format" % filename) raise SystemExit("Exception: %s" % e) __site_config_data.append(site_info)
[docs]def read_local_site_config(config_dir): """ Read site configurations from local directory specified in config_dir. Storing site configurations in global variable, that will be used by other functions :param config_dir: path to directory containing site configuration :return: None """ __site_config_data.clear() schema = read_site_schema() config_dir = Path(config_dir) for f in sorted(config_dir.glob("*.yaml")): try: yaml_file = site_info = yaml.safe_load(yaml_file) validate(instance=site_info, schema=schema) __site_config_data.append(site_info) except Exception as e: print("Error during reading site config from %s" % f) raise SystemExit("Exception: %s" % e)
[docs]def save_site_config(config_dir): """ Save site configs to local directory specified in config_dir. Overwrite local configs if exist :param config_dir: path to directory containing site configuration :return: None """ config_dir = Path(config_dir) config_dir.mkdir(parents=True, exist_ok=True) for site_info in __site_config_data: config_file = config_dir / (site_info["gocdb"] + ".yaml") with"w", encoding="utf-8") as f: yaml.dump(site_info, f)
[docs]def list_sites(): """ List of all sites IDs in site configurations :return: list of site IDs """ read_site_config() result = [] for site_info in __site_config_data: result.append(site_info["gocdb"]) return result
[docs]def find_site_data(site_name): """ Return configuration of the correspondent site with site_name :param site_name: site ID in GOCDB :return: configuration of site if found, otherwise None """ read_site_config() for site_info in __site_config_data: if site_info["gocdb"] == site_name: return site_info return None
[docs]def find_endpoint_and_project_id(site_name, vo): """ Return Keystone endpoint and project ID from site name and VO according to site configuration :param site_name: site ID in GOCDB :param vo: VO name. None if finding only site endpoint :return: endpoint, project_id, protocol if the VO exist on the site, otherwise None, None, None """ site_info = find_site_data(site_name) if site_info is None: return None, None, None protocol = site_info.get("protocol") # If only site name is given, return endpoint without project ID if vo is None: return site_info["endpoint"], None, protocol for vo_info in site_info["vos"]: if vo_info["name"] == vo: return site_info["endpoint"], vo_info["auth"]["project_id"], protocol # Return None, None if VO not found return None, None, None def site(): """ Site command group for manipulation with site configurations """ pass @site.command() @site_params def show(site): """ Print configuration of the specified site """ site_info = find_site_data(site) if site_info: print(yaml.dump(site_info, sort_keys=True)) else: print("Site %s not found" % site) return 1 @site.command() @site_vo_params def show_project_id(site, vo): """ Printing Keystone endpoint and project ID of the VO on the site """ endpoint, project_id, protocol = find_endpoint_and_project_id(site, vo) if endpoint: print('export OS_AUTH_URL="%s"' % endpoint) print('export OS_PROJECT_ID="%s"' % project_id) else: print("VO %s not found on site %s" % (vo, site)) return 1 @site.command() def show_all(): """ Print all site configuration """ read_site_config() for site_info in __site_config_data: site_info_str = yaml.dump(site_info, sort_keys=True) print(site_info_str) @site.command() def save_config(): """ Read default site configs from GitHub and save them to local folder in $HOME Overwrite local configs if exist """ read_default_site_config() config_dir = Path.home() / __LOCAL_CONFIG_DIR print("Saving site configs to directory %s" % config_dir) save_site_config(config_dir) @site.command() def list(): """ List site IDs """ read_site_config() for site_info in __site_config_data: print(site_info["gocdb"])