From 535660fe6d576ba12181564a260e77ed3ac9b7bb Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Thu, 23 Jan 2025 10:57:44 +0100 Subject: [PATCH 01/19] Add code from experimental 'info' repo --- .dockerignore | 1 - .gitignore | 8 +- Dockerfile | 2 +- requirements.txt | 16 +- src/config.py | 53 +++++++ src/functions.py | 71 --------- src/job.py | 11 ++ src/tasks/check_changelist.py | 62 ++++++++ src/tasks/check_deadlocks.py | 24 +++ src/tasks/check_missing_apps.py | 35 +++++ src/tasks/get_app_info.py | 24 +++ src/tasks/get_package_info.py | 24 +++ src/utils/__init__.py | 0 src/utils/helper.py | 115 +++++++++++++++ src/utils/redis.py | 48 ++++++ src/utils/steam.py | 115 +++++++++++++++ src/utils/storage.py | 249 ++++++++++++++++++++++++++++++++ src/{main.py => web.py} | 1 + 18 files changed, 781 insertions(+), 78 deletions(-) create mode 100644 src/config.py create mode 100644 src/job.py create mode 100644 src/tasks/check_changelist.py create mode 100644 src/tasks/check_deadlocks.py create mode 100644 src/tasks/check_missing_apps.py create mode 100644 src/tasks/get_app_info.py create mode 100644 src/tasks/get_package_info.py create mode 100644 src/utils/__init__.py create mode 100644 src/utils/helper.py create mode 100644 src/utils/redis.py create mode 100644 src/utils/steam.py create mode 100644 src/utils/storage.py rename src/{main.py => web.py} (99%) diff --git a/.dockerignore b/.dockerignore index c5ef958..567fad5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ *.pyc __pycache__ .venv -.deta .env diff --git a/.gitignore b/.gitignore index c5ef958..e55a532 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,11 @@ +.DS_Store +.ruff_cache + *.pyc __pycache__ .venv -.deta .env + +_test.py +data/ +celerybeat-schedule.db \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3a46c80..2436dc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,4 +29,4 @@ COPY --chown=$USER:$USER src/ $HOME/ ##################### INSTALLATION END ##################### # Set default container command -CMD exec gunicorn main:app --max-requests 3000 --max-requests-jitter 150 --workers $WORKERS --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:$PORT \ No newline at end of file +CMD exec gunicorn web:app --max-requests 3000 --max-requests-jitter 150 --workers $WORKERS --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:$PORT \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 10495dc..71ff032 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,18 @@ -fastapi -redis -deta - +## general semver python-dotenv logfmter +## web +fastapi[standard] +redis +minio + +## steam steam[client] gevent + +## job +celery +celery-singleton +flower \ No newline at end of file diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..18df039 --- /dev/null +++ b/src/config.py @@ -0,0 +1,53 @@ +import utils.helper +import logging + +# fmt: off + +# Set variables based on environment +redis_host = utils.helper.read_env("REDIS_HOST", "localhost") +redis_port = utils.helper.read_env("REDIS_PORT", "6379") +redis_url = "redis://" + redis_host + ":" + redis_port + "/0" + +storage_type = utils.helper.read_env("STORAGE_TYPE", "local", choices=[ "local", "object" ]) +storage_directory = utils.helper.read_env("STORAGE_DIRECTORY", "data/", dependency={ "STORAGE_TYPE": "local" }) +storage_object_endpoint = utils.helper.read_env("STORAGE_OBJECT_ENDPOINT", dependency={ "STORAGE_TYPE": "object" }) +storage_object_access_key = utils.helper.read_env("STORAGE_OBJECT_ACCESS_KEY", dependency={ "STORAGE_TYPE": "object" }) +storage_object_secret_key = utils.helper.read_env("STORAGE_OBJECT_SECRET_KEY", dependency={ "STORAGE_TYPE": "object" }) +storage_object_bucket = utils.helper.read_env("STORAGE_OBJECT_BUCKET", dependency={ "STORAGE_TYPE": "object" }) +storage_object_secure = utils.helper.read_env("STORAGE_OBJECT_SECURE", True) +storage_object_region = utils.helper.read_env("STORAGE_OBJECT_REGION", False) + +# Set general settings +chunk_size = 10 + +# logging configuration +formatter = Logfmter(keys=["level"], mapping={"level": "levelname"}) +handler = logging.StreamHandler() +handler.setFormatter(formatter) +logging.basicConfig(handlers=[handler]) +if "LOG_LEVEL" in os.environ: + log_level(os.environ["LOG_LEVEL"]) + +# Set Celery configuration +timezone = "UTC" +broker_url = redis_url +broker_connection_retry_on_startup = True +beat_schedule = { + "check-changelist-every-5-seconds": { + "task": "check_changelist", + "schedule": 5.0 + }, + "check-missing-apps-every-30-minutes": { + "task": "check_missing_apps", + "schedule": 1800.0, + }, + "check-deadlocks-every-1-hour": { + "task": "check_deadlocks", + "schedule": 3600.0, + }, +} + +worker_concurrency = 4 + +# Dynamically import all tasks files +imports = utils.helper.list_tasks() diff --git a/src/functions.py b/src/functions.py index ff87f51..c13055b 100644 --- a/src/functions.py +++ b/src/functions.py @@ -9,7 +9,6 @@ import redis import logging from steam.client import SteamClient -from deta import Deta def app_info(app_id): @@ -68,8 +67,6 @@ def cache_read(app_id): if os.environ["CACHE_TYPE"] == "redis": return redis_read(app_id) - elif os.environ["CACHE_TYPE"] == "deta": - return deta_read(app_id) else: # print query parse error and return empty dict logging.error( @@ -88,8 +85,6 @@ def cache_write(app_id, data): if os.environ["CACHE_TYPE"] == "redis": return redis_write(app_id, data) - elif os.environ["CACHE_TYPE"] == "deta": - return deta_write(app_id, data) else: # print query parse error and return empty dict logging.error( @@ -206,69 +201,3 @@ def log_level(level): logging.getLogger().setLevel(logging.CRITICAL) case _: logging.getLogger().setLevel(logging.WARNING) - - -def deta_read(app_id): - """ - Read app info from Deta base cache. - """ - - # initialize with a project key - deta = Deta(os.environ["DETA_PROJECT_KEY"]) - - # connect (and create) database - dbs = deta.Base(os.environ["DETA_BASE_NAME"]) - - try: - # get info from cache - data = dbs.get(str(app_id)) - - # return if not found - if not data: - # return failed status - return False - - # return cached data - return data["data"] - - except Exception as read_error: - # print query parse error and return empty dict - print( - "The following error occured while trying to read and decode " - + "from Deta cache:" - ) - print("> " + str(read_error)) - - # return failed status - return False - - -def deta_write(app_id, data): - """ - Write app info to Deta base cache. - """ - - # initialize with a project key - deta = Deta(os.environ["DETA_PROJECT_KEY"]) - - # connect (and create) database - dbs = deta.Base(os.environ["DETA_BASE_NAME"]) - - # write cache data and set ttl - try: - # set expiration ttl - expiration = int(os.environ["CACHE_EXPIRATION"]) - - # insert data into cache - dbs.put({"data": data}, str(app_id), expire_in=expiration) - - # return succes status - return True - - except Exception as write_error: - # print query parse error and return empty dict - print("The following error occured while trying to write to Deta cache:") - print("> " + str(write_error)) - - # return fail status - return False diff --git a/src/job.py b/src/job.py new file mode 100644 index 0000000..0aeb531 --- /dev/null +++ b/src/job.py @@ -0,0 +1,11 @@ +from celery import Celery +import logging + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Create the Celery application +app = Celery() +app.config_from_object("config") +app.autodiscover_tasks(["main.tasks"]) diff --git a/src/tasks/check_changelist.py b/src/tasks/check_changelist.py new file mode 100644 index 0000000..1acfeec --- /dev/null +++ b/src/tasks/check_changelist.py @@ -0,0 +1,62 @@ +from main import app, logger +from celery_singleton import Singleton +from .get_app_info import get_app_info_task +from .get_package_info import get_package_info_task +import utils.steam +import utils.redis +import config +import json + + +@app.task(name="check_changelist", base=Singleton, lock_expiry=10) +def check_changelist_task(): + """ + Check for app and package changes between changelists + and start tasks to retrieve these changes. + """ + + previous_change_number = utils.redis.read("change_number") + latest_change_number = utils.steam.get_change_number() + + if not previous_change_number: + logger.warning("Previous changenumber could not be retrieved from Redis") + current_state = utils.storage.read("state/", "changes.json") + if current_state: + content = json.loads(current_state) + previous_change_number = content["change_number"] + else: + logger.warning( + "Previous changenumber could not be retrieved from statefile in storage" + ) + + if not previous_change_number: + utils.redis.write("change_number", latest_change_number) + + elif int(previous_change_number) == int(latest_change_number): + logger.info( + "The previous and current change number " + + str(latest_change_number) + + " are the same" + ) + pass + + else: + changes = utils.steam.get_changes_since_change_number(previous_change_number) + + for i in range(0, len(changes["apps"]), config.chunk_size): + chunk = changes["apps"][i : i + config.chunk_size] + get_app_info_task.delay(chunk) + + for i in range(0, len(changes["packages"]), config.chunk_size): + chunk = changes["packages"][i : i + config.chunk_size] + get_package_info_task.delay(chunk) + + utils.redis.write("change_number", latest_change_number) + + content = { + "changed_apps": len(changes["apps"]), + "changed_packages": len(changes["packages"]), + "change_number": latest_change_number, + } + content = json.dumps(content) + utils.storage.write(content, "state/", "changes.json") diff --git a/src/tasks/check_deadlocks.py b/src/tasks/check_deadlocks.py new file mode 100644 index 0000000..f0dda7b --- /dev/null +++ b/src/tasks/check_deadlocks.py @@ -0,0 +1,24 @@ +from main import app, logger +from celery_singleton import clear_locks + + +@app.task(name="check_deadlocks", bind=True) +def check_deadlocks_task(self): + """ + If no running tasks, purge all singleton locks + in Redis to avoid for ever deadlocks. + """ + + workers = app.control.inspect() + workers = workers.active() + + total_tasks = [] + for worker in workers: + active_tasks = workers[worker] + for task in active_tasks: + if task["id"] != self.request.id: + total_tasks.append(task) + + clear_locks(app) + + logger.info("Cleared locks. No tasks were running.") diff --git a/src/tasks/check_missing_apps.py b/src/tasks/check_missing_apps.py new file mode 100644 index 0000000..c85face --- /dev/null +++ b/src/tasks/check_missing_apps.py @@ -0,0 +1,35 @@ +from main import app, logger +from celery_singleton import Singleton +from .get_app_info import get_app_info_task +import utils.storage +import utils.steam +import config + + +@app.task(name="check_missing_apps", base=Singleton, lock_expiry=7200) +def check_missing_apps_task(): + """ + Check for missing stored apps by comparing them with + all available apps in Steam and start tasks to + retrieve the info for the missing apps. + """ + + steam_apps = utils.steam.get_app_list() + + stored_apps = utils.storage.list("app/") + stored_apps_list = [] + for app_obj in stored_apps: + app_id = app_obj.split(".")[0] + app_ext = app_obj.split(".")[1] + if app_ext == "json": + stored_apps_list.append(int(app_id)) + + diff = utils.helper.list_differences(steam_apps, stored_apps_list) + if len(diff) > 0: + logger.info( + "Compared stored apps and found " + str(len(diff)) + " missing apps" + ) + + for i in range(0, len(diff), config.chunk_size): + chunk = diff[i : i + config.chunk_size] + get_app_info_task.delay(chunk) diff --git a/src/tasks/get_app_info.py b/src/tasks/get_app_info.py new file mode 100644 index 0000000..1c352da --- /dev/null +++ b/src/tasks/get_app_info.py @@ -0,0 +1,24 @@ +from main import app, logger +import utils.storage +import utils.steam +import json + + +@app.task( + name="get_app_info", + time_limit=3, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 3, "countdown": 5}, +) +def get_app_info_task(apps=[]): + """ + Get app information of input list of apps, generate + separate json files and upload them to the store. + """ + + logger.info("Getting product info for following apps: " + str(apps)) + apps = utils.steam.get_apps_info(apps) + + for app_obj in apps: + content = json.dumps(apps[app_obj]) + utils.storage.write(content, "app/", str(app_obj) + ".json") diff --git a/src/tasks/get_package_info.py b/src/tasks/get_package_info.py new file mode 100644 index 0000000..7c8dbf1 --- /dev/null +++ b/src/tasks/get_package_info.py @@ -0,0 +1,24 @@ +from main import app, logger +import utils.storage +import utils.steam +import json + + +@app.task( + name="get_package_info", + time_limit=3, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 3, "countdown": 5}, +) +def get_package_info_task(packages=[]): + """ + Get package information of input list of packages, generate + separate json files and upload them to the store. + """ + + logger.info("Getting product info for following packages: " + str(packages)) + packages = utils.steam.get_packages_info(packages) + + for package_obj in packages: + content = json.dumps(packages[package_obj]) + utils.storage.write(content, "package/", str(package_obj) + ".json") diff --git a/src/utils/__init__.py b/src/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/utils/helper.py b/src/utils/helper.py new file mode 100644 index 0000000..0b804c7 --- /dev/null +++ b/src/utils/helper.py @@ -0,0 +1,115 @@ +#from main import logger +import logging +import sys +import os + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def list_tasks(): + """ + List Python files in local tasks directory + and return tuple of these files. + """ + + tasks_tuple = () + tasks_directory = normalize_directory("tasks") + for task in os.listdir(tasks_directory): + if os.path.splitext(task)[1] == ".py": + tasks_tuple = tasks_tuple + ("tasks." + os.path.splitext(task)[0],) + + return tasks_tuple + + +def read_env(name, default=False, dependency={}, choices=[]): + """ + Get value from environment variable and return + false if not exist. Optionally check if other + variable exists. + """ + + try: + value = os.environ[name] + if choices and value not in choices: + logger.critical( + "The value '" + + str(value) + + "' of variable '" + + str(name) + + "' is incorrect and can only be set to one of the these values: " + + str(choices) + ) + sys.exit(1) + + except KeyError: + if default: + value = default + else: + value = False + + for dep in dependency: + if read_env(dep) == dependency[dep]: + logger.critical( + "The variable '" + + str(name) + + "' must be set because it is required when '" + + str(dep) + + "' is set to '" + + str(dependency[dep]) + + "'" + ) + sys.exit(1) + + return value + + +def combine_paths(*args): + """ + Combine the input paths to 1 path and make sure + it is valid by checking the slashes. + """ + + combined_path = "" + for arg in args: + if arg[0] == "/": + arg = arg[1:] + + if arg[-1] != "/": + arg = arg + "/" + + combined_path = combined_path + arg + + combined_path = combined_path.replace("//", "/") + combined_path = combined_path.replace("///", "/") + + return combined_path + + +def normalize_directory(path): + """ + Makes sure that relative paths always start from + the root, have a full path and that all paths end + with a "/". + """ + + if not path[0] == "/": + root = os.getcwd() + path = root + "/" + path + + if path[-1] != "/": + path = path + "/" + + return path + + +def list_differences(list1, list2): + """ + Return the items that are in list1 but + not in list2. + """ + + difference = [item for item in list1 if item not in list2] + + return difference diff --git a/src/utils/redis.py b/src/utils/redis.py new file mode 100644 index 0000000..5379a7e --- /dev/null +++ b/src/utils/redis.py @@ -0,0 +1,48 @@ +from main import logger +import logging +import config +import redis + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def connect(): + """ + Parse redis config and connect. + """ + + try: + rds = redis.Redis(host=config.redis_host, port=config.redis_port) + + except Exception as error: + logger.error("Failed to connect to Redis with error: " + error) + return False + + return rds + + +def read(key): + """ + Read specified key from Redis. + """ + + rds = connect() + data = rds.get(key) + + if not data: + return False + data = data.decode("UTF-8") + + return data + + +def write(key, data): + """ + Write specified key to Redis. + """ + + rds = connect() + rds.set(key, data) + + return True diff --git a/src/utils/steam.py b/src/utils/steam.py new file mode 100644 index 0000000..f4e36be --- /dev/null +++ b/src/utils/steam.py @@ -0,0 +1,115 @@ +from main import logger +from steam.client import SteamClient +import requests + + +def init_client(): + """ + Initialize Steam client, login and + return the client. + """ + + client = SteamClient() + client.anonymous_login() + client.verbose_debug = False + + return client + + +def get_app_list(): + """ + Get list of id's of all current apps in + Steam and return them in a flat list. + """ + + response = requests.get("https://api.steampowered.com/ISteamApps/GetAppList/v2/") + response_json = response.json() + + apps = [] + if response.status_code != 200: + logger.error("The Steam GetAppList API endpoint returned a non-200 http code") + + else: + for app in response_json["applist"]["apps"]: + apps.append(app["appid"]) + + return apps + + +def get_change_number(): + """ + Get and return the latest change number. + """ + + client = init_client() + info = client.get_changes_since(1, app_changes=False, package_changes=False) + change_number = info.current_change_number + + return change_number + + +def get_changes_since_change_number(change_number): + """ + Get and return lists of changed apps and + packages since the specified change number. + """ + + client = init_client() + info = client.get_changes_since( + int(change_number), app_changes=True, package_changes=True + ) + + app_list = [] + if info.app_changes: + for app in info.app_changes: + app_list.append(app.appid) + + package_list = [] + if info.package_changes: + for package in info.package_changes: + package_list.append(package.packageid) + + changes = {"apps": app_list, "packages": package_list} + + return changes + + +def get_apps_info(apps=[]): + """ + Get product info for list of apps and + return the output untouched. + """ + + try: + client = init_client() + info = client.get_product_info(apps=apps, timeout=5) + info = info["apps"] + except Exception as err: + logger.error( + "Something went wrong while querying product info for apps: " + str(apps) + ) + logger.error(err) + return False + + return info + + +def get_packages_info(packages=[]): + """ + Get product info for list of packages and + return the output untouched. + """ + + try: + client = init_client() + info = client.get_product_info(packages=packages, timeout=5) + info = info["packages"] + except Exception as err: + logger.error( + "Something went wrong while querying product info for packages: " + + str(packages) + ) + logger.error(err) + return False + + return info diff --git a/src/utils/storage.py b/src/utils/storage.py new file mode 100644 index 0000000..04c82a4 --- /dev/null +++ b/src/utils/storage.py @@ -0,0 +1,249 @@ +from main import logger +import utils.helper +import config +import os +from minio import Minio +from io import BytesIO + + +## meta functions + + +def read(path, filename): + """ + Read file from specified directory. + """ + + match config.storage_type: + case "local": + response = local_read(path, filename) + case "object": + response = object_read(path, filename) + case _: + response = False + + return response + + +def write(content, path, filename): + """ + Write content to file in the specified + directory/path. + """ + + match config.storage_type: + case "local": + response = local_write(content, path, filename) + case "object": + response = object_write(content, path, filename) + case _: + response = False + + return response + + +def list(path): + """ + List files in specified directory and return + it as a list. + """ + + match config.storage_type: + case "local": + response = local_list(path) + case "object": + response = object_list(path) + case _: + response = False + + return response + + +## local storage functions + + +def local_read(path, filename): + """ + Read file from local specified directory. + """ + + path = utils.helper.combine_paths(config.storage_directory, path) + path = utils.helper.normalize_directory(path) + file = path + filename + + try: + content = open(file, "r") + content = content.read() + + except Exception: + logger.error("The following file could not be read: " + file) + return False + + return content + + +def local_write(content, path, filename): + """ + Write content to file to local specified + directory. + """ + + path = utils.helper.combine_paths(config.storage_directory, path) + path = utils.helper.normalize_directory(path) + file = path + filename + + try: + f = open(file, "w") + f.write(content) + f.close() + logger.info("Written the following file: " + file) + + except Exception: + logger.error("The following file could not be written locally: " + file) + return False + + return True + + +def local_delete(path, filename): + """ + Delete file in local specified directory. + """ + + path = utils.helper.combine_paths(config.storage_directory, path) + path = utils.helper.normalize_directory(path) + file = path + filename + + try: + os.remove(file) + except OSError as err: + logger.error( + "The following file could not be deleted locally: " + + err.filename + + ". The following error occured: " + + err.strerror + ) + return False + + return True + + +def local_list(path): + """ + List files in specified local directory and + return it as a list. + """ + + path = utils.helper.combine_paths(config.storage_directory, path) + path = utils.helper.normalize_directory(path) + + try: + content = os.listdir(path) + + except Exception: + logger.error("The following directory could not be read: " + path) + return False + + return content + + +## object store functions + + +def object_connect(): + """ + Connect to object store with credentials set + in config environment variables. + """ + + arguments = { + "endpoint": config.storage_object_endpoint, + "access_key": config.storage_object_access_key, + "secret_key": config.storage_object_secret_key, + "secure": str(config.storage_object_secure).lower() == "true", + } + + if config.storage_object_region: + arguments["region"] = config.storage_object_region + + client = Minio(**arguments) + + return client + + +def object_read(path, filename): + """ + Read file from object storage from specified + directory. + """ + + file = path + filename + conn = object_connect() + + try: + content = conn.get_object(config.storage_object_bucket, file) + + except Exception: + logger.error("The following file could not be retrieved: " + file) + return False + + return content + + +def object_write(content, path, filename): + """ + Write content to file to object storage in + specified directory. + """ + + content = content.encode("utf-8") + content = BytesIO(content) + + file = path + filename + conn = object_connect() + resp = conn.put_object( + config.storage_object_bucket, + file, + content, + length=-1, + part_size=10485760, + content_type="application/json", + ) + + return resp + + +def object_delete(path, filename): + """ + Delete file in object storage specified directory. + """ + + file = path + filename + conn = object_connect() + resp = conn.remove_object(config.storage_object_bucket, file) + + return resp + + +def object_list(path, details=False): + """ + List files in specified directory in object + storage and return it as a list. + """ + + conn = object_connect() + + try: + files = conn.list_objects(config.storage_object_bucket, prefix=path) + file_list = [] + + for file in files: + file = file.object_name + file = file.split("/")[-1] + file_list.append(file) + + except Exception: + logger.error("The files in following directory could not be listed: " + path) + return False + + return file_list diff --git a/src/main.py b/src/web.py similarity index 99% rename from src/main.py rename to src/web.py index d86d0fd..8905390 100644 --- a/src/main.py +++ b/src/web.py @@ -5,6 +5,7 @@ # import modules import os import json +import config import semver import typing import logging From 4726432516a009f904d01c8131628fc475a79af3 Mon Sep 17 00:00:00 2001 From: Thuis Date: Thu, 23 Jan 2025 23:40:42 +0100 Subject: [PATCH 02/19] Combined Redis functions and rewrote logging --- requirements.txt | 1 - src/config.py | 16 ++++++++++------ src/functions.py | 20 -------------------- src/utils/general.py | 21 +++++++++++++++++++++ src/utils/helper.py | 4 ---- src/utils/redis.py | 23 +++++++++++++++++------ src/web.py | 18 +----------------- 7 files changed, 49 insertions(+), 54 deletions(-) create mode 100644 src/utils/general.py diff --git a/requirements.txt b/requirements.txt index 71ff032..0c04916 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ ## general semver -python-dotenv logfmter ## web diff --git a/src/config.py b/src/config.py index 18df039..235b4cc 100644 --- a/src/config.py +++ b/src/config.py @@ -1,12 +1,17 @@ +import utils.general import utils.helper import logging +import config +import os +from logfmter import Logfmter # fmt: off # Set variables based on environment +redis_url = utils.helper.read_env("REDIS_URL") redis_host = utils.helper.read_env("REDIS_HOST", "localhost") redis_port = utils.helper.read_env("REDIS_PORT", "6379") -redis_url = "redis://" + redis_host + ":" + redis_port + "/0" +redis_password = utils.helper.read_env("REDIS_PASSWORD") storage_type = utils.helper.read_env("STORAGE_TYPE", "local", choices=[ "local", "object" ]) storage_directory = utils.helper.read_env("STORAGE_DIRECTORY", "data/", dependency={ "STORAGE_TYPE": "local" }) @@ -17,16 +22,16 @@ storage_object_secure = utils.helper.read_env("STORAGE_OBJECT_SECURE", True) storage_object_region = utils.helper.read_env("STORAGE_OBJECT_REGION", False) +log_level = utils.helper.read_env("LOG_LEVEL", "info", choices=[ "debug", "info", "warning", "error", "critical" ]) + # Set general settings chunk_size = 10 -# logging configuration +# Logging configuration formatter = Logfmter(keys=["level"], mapping={"level": "levelname"}) handler = logging.StreamHandler() handler.setFormatter(formatter) -logging.basicConfig(handlers=[handler]) -if "LOG_LEVEL" in os.environ: - log_level(os.environ["LOG_LEVEL"]) +logging.basicConfig(handlers=[handler], level=utils.general.log_level(config.log_level)) # Set Celery configuration timezone = "UTC" @@ -46,7 +51,6 @@ "schedule": 3600.0, }, } - worker_concurrency = 4 # Dynamically import all tasks files diff --git a/src/functions.py b/src/functions.py index c13055b..91ecd70 100644 --- a/src/functions.py +++ b/src/functions.py @@ -181,23 +181,3 @@ def redis_write(app_id, data): # return fail status return False - - -def log_level(level): - """ - Sets lowest level to log. - """ - - match level: - case "debug": - logging.getLogger().setLevel(logging.DEBUG) - case "info": - logging.getLogger().setLevel(logging.INFO) - case "warning": - logging.getLogger().setLevel(logging.WARNING) - case "error": - logging.getLogger().setLevel(logging.ERROR) - case "critical": - logging.getLogger().setLevel(logging.CRITICAL) - case _: - logging.getLogger().setLevel(logging.WARNING) diff --git a/src/utils/general.py b/src/utils/general.py new file mode 100644 index 0000000..6fcdbe8 --- /dev/null +++ b/src/utils/general.py @@ -0,0 +1,21 @@ +import logging + + +def log_level(level): + """ + Sets lowest level to log. + """ + + match level: + case "debug": + return logging.DEBUG + case "info": + return logging.INFO + case "warning": + return logging.WARNING + case "error": + return logging.ERROR + case "critical": + return logging.CRITICAL + case _: + return logging.WARNING diff --git a/src/utils/helper.py b/src/utils/helper.py index 0b804c7..bbe4823 100644 --- a/src/utils/helper.py +++ b/src/utils/helper.py @@ -3,10 +3,6 @@ import sys import os -# Set up logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - def list_tasks(): """ diff --git a/src/utils/redis.py b/src/utils/redis.py index 5379a7e..dd84b29 100644 --- a/src/utils/redis.py +++ b/src/utils/redis.py @@ -1,11 +1,7 @@ -from main import logger import logging import config import redis -# Set up logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) def connect(): """ @@ -13,15 +9,30 @@ def connect(): """ try: - rds = redis.Redis(host=config.redis_host, port=config.redis_port) + # try connection string, or default to separate REDIS_* env vars + if config.redis_url: + rds = redis.Redis.from_url(config.redis_url) + + elif config.redis_password: + rds = redis.Redis( + host=config.redis_host, + port=config.redis_port, + password=config.redis_password + ) + else: + rds = redis.Redis( + host=config.redis_host, + port=config.redis_port + ) except Exception as error: - logger.error("Failed to connect to Redis with error: " + error) + logging.error("Failed to connect to Redis with error: " + error) return False return rds + def read(key): """ Read specified key from Redis. diff --git a/src/web.py b/src/web.py index 8905390..cc86535 100644 --- a/src/web.py +++ b/src/web.py @@ -10,27 +10,11 @@ import typing import logging from fastapi import FastAPI, Response -from functions import app_info, cache_read, cache_write, log_level -from logfmter import Logfmter - -# load configuration -from dotenv import load_dotenv - -load_dotenv() +from functions import app_info, cache_read, cache_write # initialise app app = FastAPI() -# set logformat -formatter = Logfmter(keys=["level"], mapping={"level": "levelname"}) -handler = logging.StreamHandler() -handler.setFormatter(formatter) -logging.basicConfig(handlers=[handler]) - -if "LOG_LEVEL" in os.environ: - log_level(os.environ["LOG_LEVEL"]) - - # include "pretty" for backwards compatibility class PrettyJSONResponse(Response): media_type = "application/json" From 4dfa3f5ce92225c6b2fea2c7ebc2596ef4493af1 Mon Sep 17 00:00:00 2001 From: Thuis Date: Tue, 28 Jan 2025 00:12:16 +0100 Subject: [PATCH 03/19] Further playing around and merging functions --- README.md | 4 ---- docker-compose.yml | 4 ++-- src/config.py | 7 +++++++ src/functions.py | 39 ++++++++++++--------------------------- src/utils/helper.py | 4 ++-- src/utils/redis.py | 7 ++++--- src/utils/steam.py | 1 + src/web.py | 8 ++++---- 8 files changed, 32 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 0d1faa4..90f0562 100644 --- a/README.md +++ b/README.md @@ -109,10 +109,6 @@ REDIS_URL="redis://YourUsername:YourRedisP@ssword!@your.redis.host.example.com:6 # logging LOG_LEVEL=info - -# deta -DETA_BASE_NAME="steamcmd" -DETA_PROJECT_KEY="YourDet@ProjectKey!" ``` ## Development diff --git a/docker-compose.yml b/docker-compose.yml index 893a12e..7c723ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: web: build: . - command: "gunicorn main:app --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 --reload" + command: "gunicorn web:app --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 --reload" ports: - "8000:8000" volumes: @@ -10,7 +10,7 @@ services: PORT: 8000 WORKERS: 4 VERSION: 9.9.9 - CACHE: True + CACHE: "True" CACHE_TYPE: redis CACHE_EXPIRATION: 120 REDIS_HOST: redis diff --git a/src/config.py b/src/config.py index 235b4cc..61edde5 100644 --- a/src/config.py +++ b/src/config.py @@ -8,10 +8,16 @@ # fmt: off # Set variables based on environment +cache = utils.helper.read_env("CACHE", "False", choices=[ "True", "False" ]) +cache_type = utils.helper.read_env("CACHE_TYPE", "redis", choices=[ "redis" ]) +cache_expiration = utils.helper.read_env("CACHE_EXPIRATION", "120") + redis_url = utils.helper.read_env("REDIS_URL") redis_host = utils.helper.read_env("REDIS_HOST", "localhost") redis_port = utils.helper.read_env("REDIS_PORT", "6379") redis_password = utils.helper.read_env("REDIS_PASSWORD") +redis_database_web = utils.helper.read_env("REDIS_DATABASE_WEB", "0") +redis_database_job = utils.helper.read_env("REDIS_DATABASE_JOB", "1") storage_type = utils.helper.read_env("STORAGE_TYPE", "local", choices=[ "local", "object" ]) storage_directory = utils.helper.read_env("STORAGE_DIRECTORY", "data/", dependency={ "STORAGE_TYPE": "local" }) @@ -23,6 +29,7 @@ storage_object_region = utils.helper.read_env("STORAGE_OBJECT_REGION", False) log_level = utils.helper.read_env("LOG_LEVEL", "info", choices=[ "debug", "info", "warning", "error", "critical" ]) +version = utils.helper.read_env("VERSION", "9.9.9") # Set general settings chunk_size = 10 diff --git a/src/functions.py b/src/functions.py index 91ecd70..14ad35e 100644 --- a/src/functions.py +++ b/src/functions.py @@ -3,6 +3,8 @@ """ # import modules +import utils.redis +import config import os import json import gevent @@ -65,7 +67,7 @@ def cache_read(app_id): Read app info from chosen cache. """ - if os.environ["CACHE_TYPE"] == "redis": + if config.cache_type == "redis": return redis_read(app_id) else: # print query parse error and return empty dict @@ -83,7 +85,7 @@ def cache_write(app_id, data): write app info to chosen cache. """ - if os.environ["CACHE_TYPE"] == "redis": + if config.cache_type == "redis": return redis_write(app_id, data) else: # print query parse error and return empty dict @@ -96,33 +98,12 @@ def cache_write(app_id, data): return False -def redis_connection(): - """ - Parse redis config and connect. - """ - - # try connection string, or default to separate REDIS_* env vars - if "REDIS_URL" in os.environ: - rds = redis.Redis.from_url(os.environ["REDIS_URL"]) - elif "REDIS_PASSWORD" in os.environ: - rds = redis.Redis( - host=os.environ["REDIS_HOST"], - port=os.environ["REDIS_PORT"], - password=os.environ["REDIS_PASSWORD"], - ) - else: - rds = redis.Redis(host=os.environ["REDIS_HOST"], port=os.environ["REDIS_PORT"]) - - # return connection - return rds - - def redis_read(app_id): """ Read app info from Redis cache. """ - rds = redis_connection() + rds = utils.redis.connect() try: # get info from cache @@ -158,7 +139,7 @@ def redis_write(app_id, data): Write app info to Redis cache. """ - rds = redis_connection() + rds = utils.redis.connect() # write cache data and set ttl try: @@ -166,8 +147,12 @@ def redis_write(app_id, data): data = json.dumps(data) # insert data into cache - expiration = int(os.environ["CACHE_EXPIRATION"]) - rds.set(app_id, data, ex=expiration) + if int(config.cache_expiration) == 0: + rds.set(app_id, data) + + else: + expiration = int(config.cache_expiration) + rds.set(app_id, data, ex=expiration) # return succes status return True diff --git a/src/utils/helper.py b/src/utils/helper.py index bbe4823..cdd446d 100644 --- a/src/utils/helper.py +++ b/src/utils/helper.py @@ -29,7 +29,7 @@ def read_env(name, default=False, dependency={}, choices=[]): try: value = os.environ[name] if choices and value not in choices: - logger.critical( + logging.critical( "The value '" + str(value) + "' of variable '" @@ -47,7 +47,7 @@ def read_env(name, default=False, dependency={}, choices=[]): for dep in dependency: if read_env(dep) == dependency[dep]: - logger.critical( + logging.critical( "The variable '" + str(name) + "' must be set because it is required when '" diff --git a/src/utils/redis.py b/src/utils/redis.py index dd84b29..b09f647 100644 --- a/src/utils/redis.py +++ b/src/utils/redis.py @@ -17,12 +17,14 @@ def connect(): rds = redis.Redis( host=config.redis_host, port=config.redis_port, - password=config.redis_password + password=config.redis_password, + #db=str(redis_database_web) ) else: rds = redis.Redis( host=config.redis_host, - port=config.redis_port + port=config.redis_port, + #db=str(redis_database_web) ) except Exception as error: @@ -32,7 +34,6 @@ def connect(): return rds - def read(key): """ Read specified key from Redis. diff --git a/src/utils/steam.py b/src/utils/steam.py index f4e36be..628eb1c 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -28,6 +28,7 @@ def get_app_list(): apps = [] if response.status_code != 200: logger.error("The Steam GetAppList API endpoint returned a non-200 http code") + return False else: for app in response_json["applist"]["apps"]: diff --git a/src/web.py b/src/web.py index cc86535..f0e0cbc 100644 --- a/src/web.py +++ b/src/web.py @@ -3,9 +3,9 @@ """ # import modules +import config import os import json -import config import semver import typing import logging @@ -34,7 +34,7 @@ def render(self, content: typing.Any) -> bytes: def read_app(app_id: int, pretty: bool = False): logging.info("Requested app info", extra={"app_id": app_id}) - if "CACHE" in os.environ and os.environ["CACHE"]: + if config.cache == "True": info = cache_read(app_id) if not info: @@ -77,10 +77,10 @@ def read_item(pretty: bool = False): logging.info("Requested api version") # check if version succesfully read and parsed - if "VERSION" in os.environ and os.environ["VERSION"]: + if config.version: return { "status": "success", - "data": semver.parse(os.environ["VERSION"]), + "data": semver.parse(config.version), "pretty": pretty, } else: From 30fb09faa714cf34e7392031f8ded8b2c75e717f Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Tue, 28 Jan 2025 20:13:43 +0100 Subject: [PATCH 04/19] WIP Redis functions --- src/utils/redis.py | 50 +++++++++++++++++++++++++++++++++++++++++++--- src/web.py | 4 ++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/utils/redis.py b/src/utils/redis.py index b09f647..c9dca5e 100644 --- a/src/utils/redis.py +++ b/src/utils/redis.py @@ -11,20 +11,22 @@ def connect(): try: # try connection string, or default to separate REDIS_* env vars if config.redis_url: - rds = redis.Redis.from_url(config.redis_url) + rds = redis.Redis.from_url(config.redis_url, db=config.redis_database_web) + #print(rds) + #print('----------') elif config.redis_password: rds = redis.Redis( host=config.redis_host, port=config.redis_port, password=config.redis_password, - #db=str(redis_database_web) + db=config.redis_database_web ) else: rds = redis.Redis( host=config.redis_host, port=config.redis_port, - #db=str(redis_database_web) + db=config.redis_database_web ) except Exception as error: @@ -58,3 +60,45 @@ def write(key, data): rds.set(key, data) return True + + +def remove_database_from_url(url): + """ + Remove database if specified in the given + connection url and return the result. + """ + + last_element = url.split('/')[-1] + + try: + specified_database = int(last_element) + #except TypeError: + except ValueError: + specified_database = 0 + print('There is no Redis database specified in the given connection string or it is not specified as an integer!') + + return url + + +def change_url_database(url, database): + """ + Remove any specified database from Redis + connection url and return edited url. + """ + + last_element = url.split('/')[-1] + + try: + specified_database = int(last_element) + #except TypeError: + except ValueError: + specified_database = 0 + print('There is no Redis database specified in the given connection string or it is not specified as an integer!') + + print(specified_database) + + #print(type(last_element)) + #if int(last_element) == int(database): + + + return url \ No newline at end of file diff --git a/src/web.py b/src/web.py index f0e0cbc..e9db3e3 100644 --- a/src/web.py +++ b/src/web.py @@ -9,9 +9,13 @@ import semver import typing import logging +from dotenv import load_dotenv from fastapi import FastAPI, Response from functions import app_info, cache_read, cache_write +# load configuration +load_dotenv() + # initialise app app = FastAPI() From 01f94291dce783758eb64328e234f1485d21eea1 Mon Sep 17 00:00:00 2001 From: Thuis Date: Tue, 28 Jan 2025 23:06:31 +0100 Subject: [PATCH 05/19] WIP --- src/config.py | 3 +-- src/functions.py | 42 -------------------------------------- src/utils/redis.py | 50 +++------------------------------------------- src/web.py | 9 ++++++--- 4 files changed, 10 insertions(+), 94 deletions(-) diff --git a/src/config.py b/src/config.py index 61edde5..0dc02af 100644 --- a/src/config.py +++ b/src/config.py @@ -16,8 +16,7 @@ redis_host = utils.helper.read_env("REDIS_HOST", "localhost") redis_port = utils.helper.read_env("REDIS_PORT", "6379") redis_password = utils.helper.read_env("REDIS_PASSWORD") -redis_database_web = utils.helper.read_env("REDIS_DATABASE_WEB", "0") -redis_database_job = utils.helper.read_env("REDIS_DATABASE_JOB", "1") +redis_database = utils.helper.read_env("REDIS_DATABASE", "0") storage_type = utils.helper.read_env("STORAGE_TYPE", "local", choices=[ "local", "object" ]) storage_directory = utils.helper.read_env("STORAGE_DIRECTORY", "data/", dependency={ "STORAGE_TYPE": "local" }) diff --git a/src/functions.py b/src/functions.py index 14ad35e..bb37c6f 100644 --- a/src/functions.py +++ b/src/functions.py @@ -62,42 +62,6 @@ def app_info(app_id): logging.error(err, extra={"app_id": app_id}) -def cache_read(app_id): - """ - Read app info from chosen cache. - """ - - if config.cache_type == "redis": - return redis_read(app_id) - else: - # print query parse error and return empty dict - logging.error( - "Set incorrect cache type", - extra={"app_id": app_id, "cache_type": os.environ["CACHE_TYPE"]}, - ) - - # return failed status - return False - - -def cache_write(app_id, data): - """ - write app info to chosen cache. - """ - - if config.cache_type == "redis": - return redis_write(app_id, data) - else: - # print query parse error and return empty dict - logging.error( - "Set incorrect cache type", - extra={"app_id": app_id, "cache_type": os.environ["CACHE_TYPE"]}, - ) - - # return failed status - return False - - def redis_read(app_id): """ Read app info from Redis cache. @@ -117,9 +81,6 @@ def redis_read(app_id): # decode bytes to str data = data.decode("UTF-8") - # convert json to python dict - data = json.loads(data) - # return cached data return data @@ -143,9 +104,6 @@ def redis_write(app_id, data): # write cache data and set ttl try: - # convert dict to json - data = json.dumps(data) - # insert data into cache if int(config.cache_expiration) == 0: rds.set(app_id, data) diff --git a/src/utils/redis.py b/src/utils/redis.py index c9dca5e..0dbfe11 100644 --- a/src/utils/redis.py +++ b/src/utils/redis.py @@ -11,22 +11,20 @@ def connect(): try: # try connection string, or default to separate REDIS_* env vars if config.redis_url: - rds = redis.Redis.from_url(config.redis_url, db=config.redis_database_web) - #print(rds) - #print('----------') + rds = redis.Redis.from_url(config.redis_url, db=config.redis_database) elif config.redis_password: rds = redis.Redis( host=config.redis_host, port=config.redis_port, password=config.redis_password, - db=config.redis_database_web + db=config.redis_database ) else: rds = redis.Redis( host=config.redis_host, port=config.redis_port, - db=config.redis_database_web + db=config.redis_database ) except Exception as error: @@ -60,45 +58,3 @@ def write(key, data): rds.set(key, data) return True - - -def remove_database_from_url(url): - """ - Remove database if specified in the given - connection url and return the result. - """ - - last_element = url.split('/')[-1] - - try: - specified_database = int(last_element) - #except TypeError: - except ValueError: - specified_database = 0 - print('There is no Redis database specified in the given connection string or it is not specified as an integer!') - - return url - - -def change_url_database(url, database): - """ - Remove any specified database from Redis - connection url and return edited url. - """ - - last_element = url.split('/')[-1] - - try: - specified_database = int(last_element) - #except TypeError: - except ValueError: - specified_database = 0 - print('There is no Redis database specified in the given connection string or it is not specified as an integer!') - - print(specified_database) - - #print(type(last_element)) - #if int(last_element) == int(database): - - - return url \ No newline at end of file diff --git a/src/web.py b/src/web.py index e9db3e3..e759fa6 100644 --- a/src/web.py +++ b/src/web.py @@ -11,7 +11,7 @@ import logging from dotenv import load_dotenv from fastapi import FastAPI, Response -from functions import app_info, cache_read, cache_write +from functions import app_info, redis_read, redis_write # load configuration load_dotenv() @@ -39,14 +39,17 @@ def read_app(app_id: int, pretty: bool = False): logging.info("Requested app info", extra={"app_id": app_id}) if config.cache == "True": - info = cache_read(app_id) + info = redis_read(app_id) + if info: + info = json.loads(info) if not info: logging.info( "App info could not be found in cache", extra={"app_id": app_id} ) info = app_info(app_id) - cache_write(app_id, info) + data = json.dumps(info) + redis_write(app_id, data) else: logging.info( "App info succesfully retrieved from cache", From 54da8647c176d4d9f0b3fe202eb96701e70072aa Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Wed, 29 Jan 2025 02:01:49 +0100 Subject: [PATCH 06/19] WIP --- src/config.py | 8 +++- src/functions.py | 93 +++++++++++++--------------------------------- src/utils/redis.py | 51 +++++++++++++++++++++---- src/utils/steam.py | 55 ++++++++++++++++++++++----- src/web.py | 36 ++++++++---------- 5 files changed, 138 insertions(+), 105 deletions(-) diff --git a/src/config.py b/src/config.py index 0dc02af..fd6bf04 100644 --- a/src/config.py +++ b/src/config.py @@ -3,15 +3,19 @@ import logging import config import os +from dotenv import load_dotenv from logfmter import Logfmter # fmt: off +# Load values from .env file +load_dotenv() + # Set variables based on environment cache = utils.helper.read_env("CACHE", "False", choices=[ "True", "False" ]) cache_type = utils.helper.read_env("CACHE_TYPE", "redis", choices=[ "redis" ]) cache_expiration = utils.helper.read_env("CACHE_EXPIRATION", "120") - + redis_url = utils.helper.read_env("REDIS_URL") redis_host = utils.helper.read_env("REDIS_HOST", "localhost") redis_port = utils.helper.read_env("REDIS_PORT", "6379") @@ -37,7 +41,7 @@ formatter = Logfmter(keys=["level"], mapping={"level": "levelname"}) handler = logging.StreamHandler() handler.setFormatter(formatter) -logging.basicConfig(handlers=[handler], level=utils.general.log_level(config.log_level)) +logging.basicConfig(handlers=[handler], level=utils.general.log_level(log_level)) # Set Celery configuration timezone = "UTC" diff --git a/src/functions.py b/src/functions.py index bb37c6f..ead7ad6 100644 --- a/src/functions.py +++ b/src/functions.py @@ -5,19 +5,21 @@ # import modules import utils.redis import config -import os -import json import gevent -import redis import logging from steam.client import SteamClient -def app_info(app_id): +def app_info(apps=[]): + """ + Get product info for list of apps and + return the output untouched. + """ + connect_retries = 2 connect_timeout = 3 - logging.info("Started requesting app info", extra={"app_id": app_id}) + logging.info("Started requesting app info", extra={"apps": str(apps)}) try: # Sometimes it hangs for 30+ seconds. Normal connection takes about 500ms @@ -28,7 +30,7 @@ def app_info(app_id): with gevent.Timeout(connect_timeout): logging.info( "Retrieving app info from steamclient", - extra={"app_id": app_id, "retry_count": count}, + extra={"apps": str(apps), "retry_count": count}, ) logging.debug("Connecting via steamclient to steam api") @@ -36,8 +38,8 @@ def app_info(app_id): client.anonymous_login() client.verbose_debug = False - logging.debug("Requesting app info from steam api") - info = client.get_product_info(apps=[app_id], timeout=1) + logging.debug("Requesting app info from steam api", extra={"apps": str(apps)}) + info = client.get_product_info(apps=apps, timeout=1) return info @@ -48,79 +50,36 @@ def app_info(app_id): client._connecting = False else: - logging.info("Succesfully retrieved app info", extra={"app_id": app_id}) + logging.info("Succesfully retrieved app info", extra={"apps": str(apps)}) break else: logging.error( "Max connect retries exceeded", - extra={"connect_retries": connect_retries}, + extra={"apps": str(apps), "connect_retries": connect_retries}, ) raise Exception(f"Max connect retries ({connect_retries}) exceeded") except Exception as err: - logging.error("Failed in retrieving app info", extra={"app_id": app_id}) - logging.error(err, extra={"app_id": app_id}) - - -def redis_read(app_id): - """ - Read app info from Redis cache. - """ - - rds = utils.redis.connect() - - try: - # get info from cache - data = rds.get(app_id) - - # return if not found - if not data: - # return failed status - return False - - # decode bytes to str - data = data.decode("UTF-8") - - # return cached data - return data - - except Exception as redis_error: - # print query parse error and return empty dict - logging.error( - "An error occured while trying to read and decode from Redis cache", - extra={"app_id": app_id, "error_msg": redis_error}, - ) - - # return failed status + logging.error("Failed in retrieving app info with error: " + str(err), extra={"apps": str(apps)}) return False + return info -def redis_write(app_id, data): +def get_apps_info(apps=[]): """ - Write app info to Redis cache. + Get product info for list of apps and + return the output untouched. """ - rds = utils.redis.connect() - - # write cache data and set ttl try: - # insert data into cache - if int(config.cache_expiration) == 0: - rds.set(app_id, data) - - else: - expiration = int(config.cache_expiration) - rds.set(app_id, data, ex=expiration) - - # return succes status - return True - - except Exception as redis_error: - # print query parse error and return empty dict - logging.error( - "An error occured while trying to write to Redis cache", - extra={"app_id": app_id, "error_msg": redis_error}, + client = init_client() + info = client.get_product_info(apps=apps, timeout=5) + info = info["apps"] + except Exception as err: + logger.error( + "Something went wrong while querying product info", extra={"apps": str(apps)} ) + logger.error(err) + return False - # return fail status - return False + return info \ No newline at end of file diff --git a/src/utils/redis.py b/src/utils/redis.py index 0dbfe11..1944bfe 100644 --- a/src/utils/redis.py +++ b/src/utils/redis.py @@ -40,13 +40,30 @@ def read(key): """ rds = connect() - data = rds.get(key) - if not data: - return False - data = data.decode("UTF-8") + try: + # get info from cache + data = rds.get(key) + + # return False if not found + if not data: + return False + + # decode bytes to str + data = data.decode("UTF-8") + + # return data from Redis + return data + + except Exception as redis_error: + # print query parse error and return empty dict + logging.error( + "An error occured while trying to read and decode from Redis", + extra={"key": key, "error_msg": redis_error}, + ) - return data + # return failed status + return False def write(key, data): @@ -55,6 +72,26 @@ def write(key, data): """ rds = connect() - rds.set(key, data) - return True + # write data and set ttl + try: + expiration = int(config.cache_expiration) + + # insert data into Redis + if expiration == 0: + rds.set(key, data) + else: + rds.set(key, data, ex=expiration) + + # return succes status + return True + + except Exception as redis_error: + # print query parse error and return empty dict + logging.error( + "An error occured while trying to write to Redis cache", + extra={"key": key, "error_msg": redis_error}, + ) + + # return fail status + return False diff --git a/src/utils/steam.py b/src/utils/steam.py index 628eb1c..cb85deb 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -1,6 +1,8 @@ -from main import logger -from steam.client import SteamClient +import config +import gevent +import logging import requests +from steam.client import SteamClient def init_client(): @@ -9,6 +11,7 @@ def init_client(): return the client. """ + logging.debug("Connecting via steamclient to steam api") client = SteamClient() client.anonymous_login() client.verbose_debug = False @@ -81,15 +84,49 @@ def get_apps_info(apps=[]): return the output untouched. """ + connect_retries = 2 + connect_timeout = 3 + + logging.info("Started requesting app info", extra={"apps": str(apps)}) + try: - client = init_client() - info = client.get_product_info(apps=apps, timeout=5) - info = info["apps"] + # Sometimes it hangs for 30+ seconds. Normal connection takes about 500ms + for _ in range(connect_retries): + count = str(_) + + try: + with gevent.Timeout(connect_timeout): + logging.info( + "Retrieving app info from steamclient", + extra={"apps": str(apps), "retry_count": count}, + ) + + client = init_client() + + logging.debug("Requesting app info from steam api", extra={"apps": str(apps)}) + info = client.get_product_info(apps=apps, timeout=1) + info = info["apps"] + + return info + + except gevent.timeout.Timeout: + logging.warning( + "Encountered timeout when trying to connect to steam api. Retrying.." + ) + client._connecting = False + + else: + logging.info("Succesfully retrieved app info", extra={"apps": str(apps)}) + break + else: + logging.error( + "Max connect retries exceeded", + extra={"apps": str(apps), "connect_retries": connect_retries}, + ) + raise Exception(f"Max connect retries ({connect_retries}) exceeded") + except Exception as err: - logger.error( - "Something went wrong while querying product info for apps: " + str(apps) - ) - logger.error(err) + logging.error("Failed in retrieving app info with error: " + str(err), extra={"apps": str(apps)}) return False return info diff --git a/src/web.py b/src/web.py index e759fa6..5c6aa11 100644 --- a/src/web.py +++ b/src/web.py @@ -3,18 +3,15 @@ """ # import modules +import utils.redis +import utils.steam import config -import os import json import semver import typing import logging -from dotenv import load_dotenv from fastapi import FastAPI, Response -from functions import app_info, redis_read, redis_write - -# load configuration -load_dotenv() +from functions import app_info # initialise app app = FastAPI() @@ -36,47 +33,46 @@ def render(self, content: typing.Any) -> bytes: @app.get("/v1/info/{app_id}", response_class=PrettyJSONResponse) def read_app(app_id: int, pretty: bool = False): - logging.info("Requested app info", extra={"app_id": app_id}) + logging.info("Requested app info", extra={"apps": str([app_id])}) if config.cache == "True": - info = redis_read(app_id) - if info: - info = json.loads(info) + info = utils.redis.read(app_id) if not info: logging.info( - "App info could not be found in cache", extra={"app_id": app_id} + "App info could not be found in cache", extra={"apps": str([app_id])} ) - info = app_info(app_id) + info = utils.steam.get_apps_info([app_id]) data = json.dumps(info) - redis_write(app_id, data) + utils.redis.write(app_id, data) else: + info = json.loads(info) logging.info( "App info succesfully retrieved from cache", - extra={"app_id": app_id}, + extra={"apps": str([app_id])}, ) else: - info = app_info(app_id) + info = utils.steam.get_apps_info([app_id]) if info is None: logging.info( "The SteamCMD backend returned no actual data and failed", - extra={"app_id": app_id}, + extra={"apps": str([app_id])}, ) # return empty result for not found app return {"data": {app_id: {}}, "status": "failed", "pretty": pretty} - if not info["apps"]: + if not info: logging.info( "No app has been found at Steam but the request was succesfull", - extra={"app_id": app_id}, + extra={"apps": str([app_id])}, ) # return empty result for not found app return {"data": {app_id: {}}, "status": "success", "pretty": pretty} - logging.info("Succesfully retrieved app info", extra={"app_id": app_id}) - return {"data": info["apps"], "status": "success", "pretty": pretty} + logging.info("Succesfully retrieved app info", extra={"apps": str([app_id])}) + return {"data": info, "status": "success", "pretty": pretty} @app.get("/v1/version", response_class=PrettyJSONResponse) From 716fd15a58940e783c5bbdcd6c5908aed8c16b7e Mon Sep 17 00:00:00 2001 From: Thuis Date: Wed, 29 Jan 2025 15:08:41 +0100 Subject: [PATCH 07/19] Rewrite --- src/config.py | 9 ++-- src/functions.py | 85 --------------------------------- src/job.py | 11 +++-- src/tasks/check_changelist.py | 40 +++++----------- src/tasks/check_deadlocks.py | 5 +- src/tasks/check_missing_apps.py | 2 +- src/tasks/get_app_info.py | 4 +- src/tasks/get_package_info.py | 4 +- src/utils/helper.py | 1 - src/utils/redis.py | 27 +++++++++++ src/utils/steam.py | 6 +-- src/utils/storage.py | 15 +++--- src/web.py | 7 ++- 13 files changed, 74 insertions(+), 142 deletions(-) delete mode 100644 src/functions.py diff --git a/src/config.py b/src/config.py index fd6bf04..97d88da 100644 --- a/src/config.py +++ b/src/config.py @@ -2,7 +2,6 @@ import utils.helper import logging import config -import os from dotenv import load_dotenv from logfmter import Logfmter @@ -52,10 +51,10 @@ "task": "check_changelist", "schedule": 5.0 }, - "check-missing-apps-every-30-minutes": { - "task": "check_missing_apps", - "schedule": 1800.0, - }, + #"check-missing-apps-every-30-minutes": { + # "task": "check_missing_apps", + # "schedule": 1800.0, + #}, "check-deadlocks-every-1-hour": { "task": "check_deadlocks", "schedule": 3600.0, diff --git a/src/functions.py b/src/functions.py deleted file mode 100644 index ead7ad6..0000000 --- a/src/functions.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -General Functions -""" - -# import modules -import utils.redis -import config -import gevent -import logging -from steam.client import SteamClient - - -def app_info(apps=[]): - """ - Get product info for list of apps and - return the output untouched. - """ - - connect_retries = 2 - connect_timeout = 3 - - logging.info("Started requesting app info", extra={"apps": str(apps)}) - - try: - # Sometimes it hangs for 30+ seconds. Normal connection takes about 500ms - for _ in range(connect_retries): - count = str(_) - - try: - with gevent.Timeout(connect_timeout): - logging.info( - "Retrieving app info from steamclient", - extra={"apps": str(apps), "retry_count": count}, - ) - - logging.debug("Connecting via steamclient to steam api") - client = SteamClient() - client.anonymous_login() - client.verbose_debug = False - - logging.debug("Requesting app info from steam api", extra={"apps": str(apps)}) - info = client.get_product_info(apps=apps, timeout=1) - - return info - - except gevent.timeout.Timeout: - logging.warning( - "Encountered timeout when trying to connect to steam api. Retrying.." - ) - client._connecting = False - - else: - logging.info("Succesfully retrieved app info", extra={"apps": str(apps)}) - break - else: - logging.error( - "Max connect retries exceeded", - extra={"apps": str(apps), "connect_retries": connect_retries}, - ) - raise Exception(f"Max connect retries ({connect_retries}) exceeded") - - except Exception as err: - logging.error("Failed in retrieving app info with error: " + str(err), extra={"apps": str(apps)}) - return False - - return info - -def get_apps_info(apps=[]): - """ - Get product info for list of apps and - return the output untouched. - """ - - try: - client = init_client() - info = client.get_product_info(apps=apps, timeout=5) - info = info["apps"] - except Exception as err: - logger.error( - "Something went wrong while querying product info", extra={"apps": str(apps)} - ) - logger.error(err) - return False - - return info \ No newline at end of file diff --git a/src/job.py b/src/job.py index 0aeb531..decf66c 100644 --- a/src/job.py +++ b/src/job.py @@ -1,11 +1,16 @@ +""" +Job Service and background worker. +""" + +# import modules from celery import Celery import logging -# Set up logging +# set up logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -# Create the Celery application +# initialise Celery application app = Celery() app.config_from_object("config") -app.autodiscover_tasks(["main.tasks"]) +app.autodiscover_tasks(["job.tasks"]) diff --git a/src/tasks/check_changelist.py b/src/tasks/check_changelist.py index 1acfeec..f781263 100644 --- a/src/tasks/check_changelist.py +++ b/src/tasks/check_changelist.py @@ -1,9 +1,10 @@ -from main import app, logger +from job import app, logger from celery_singleton import Singleton from .get_app_info import get_app_info_task from .get_package_info import get_package_info_task import utils.steam import utils.redis +import logging import config import json @@ -15,25 +16,15 @@ def check_changelist_task(): and start tasks to retrieve these changes. """ - previous_change_number = utils.redis.read("change_number") + previous_change_number = utils.redis.read("_state.change_number") latest_change_number = utils.steam.get_change_number() if not previous_change_number: - logger.warning("Previous changenumber could not be retrieved from Redis") - current_state = utils.storage.read("state/", "changes.json") - if current_state: - content = json.loads(current_state) - previous_change_number = content["change_number"] - else: - logger.warning( - "Previous changenumber could not be retrieved from statefile in storage" - ) - - if not previous_change_number: - utils.redis.write("change_number", latest_change_number) + logging.warning("Previous changenumber could not be retrieved from Redis") + utils.redis.write("_state.change_number", latest_change_number) elif int(previous_change_number) == int(latest_change_number): - logger.info( + logging.info( "The previous and current change number " + str(latest_change_number) + " are the same" @@ -41,22 +32,17 @@ def check_changelist_task(): pass else: + logging.info("The changenumber has been updated from " + str(previous_change_number) + " to " + str(latest_change_number)) changes = utils.steam.get_changes_since_change_number(previous_change_number) for i in range(0, len(changes["apps"]), config.chunk_size): chunk = changes["apps"][i : i + config.chunk_size] get_app_info_task.delay(chunk) - for i in range(0, len(changes["packages"]), config.chunk_size): - chunk = changes["packages"][i : i + config.chunk_size] - get_package_info_task.delay(chunk) - - utils.redis.write("change_number", latest_change_number) + #for i in range(0, len(changes["packages"]), config.chunk_size): + # chunk = changes["packages"][i : i + config.chunk_size] + # get_package_info_task.delay(chunk) - content = { - "changed_apps": len(changes["apps"]), - "changed_packages": len(changes["packages"]), - "change_number": latest_change_number, - } - content = json.dumps(content) - utils.storage.write(content, "state/", "changes.json") + utils.redis.write("_state.change_number", latest_change_number) + utils.redis.increment("_state.changed_apps", len(changes["apps"])) + utils.redis.increment("_state.changed_packages", len(changes["packages"])) diff --git a/src/tasks/check_deadlocks.py b/src/tasks/check_deadlocks.py index f0dda7b..d118f0b 100644 --- a/src/tasks/check_deadlocks.py +++ b/src/tasks/check_deadlocks.py @@ -1,4 +1,5 @@ -from main import app, logger +from job import app +import logging from celery_singleton import clear_locks @@ -21,4 +22,4 @@ def check_deadlocks_task(self): clear_locks(app) - logger.info("Cleared locks. No tasks were running.") + logging.info("Cleared locks. No tasks were running.") diff --git a/src/tasks/check_missing_apps.py b/src/tasks/check_missing_apps.py index c85face..fcfc47a 100644 --- a/src/tasks/check_missing_apps.py +++ b/src/tasks/check_missing_apps.py @@ -1,4 +1,4 @@ -from main import app, logger +from job import app, logger from celery_singleton import Singleton from .get_app_info import get_app_info_task import utils.storage diff --git a/src/tasks/get_app_info.py b/src/tasks/get_app_info.py index 1c352da..e220bbe 100644 --- a/src/tasks/get_app_info.py +++ b/src/tasks/get_app_info.py @@ -1,4 +1,4 @@ -from main import app, logger +from job import app, logger import utils.storage import utils.steam import json @@ -21,4 +21,4 @@ def get_app_info_task(apps=[]): for app_obj in apps: content = json.dumps(apps[app_obj]) - utils.storage.write(content, "app/", str(app_obj) + ".json") + utils.redis.write("app." + str(app_obj), content) diff --git a/src/tasks/get_package_info.py b/src/tasks/get_package_info.py index 7c8dbf1..746ea2f 100644 --- a/src/tasks/get_package_info.py +++ b/src/tasks/get_package_info.py @@ -1,4 +1,4 @@ -from main import app, logger +from job import app, logger import utils.storage import utils.steam import json @@ -21,4 +21,4 @@ def get_package_info_task(packages=[]): for package_obj in packages: content = json.dumps(packages[package_obj]) - utils.storage.write(content, "package/", str(package_obj) + ".json") + utils.redis.write("package." + str(package_obj), content) \ No newline at end of file diff --git a/src/utils/helper.py b/src/utils/helper.py index cdd446d..4c04529 100644 --- a/src/utils/helper.py +++ b/src/utils/helper.py @@ -1,4 +1,3 @@ -#from main import logger import logging import sys import os diff --git a/src/utils/redis.py b/src/utils/redis.py index 1944bfe..ffb3811 100644 --- a/src/utils/redis.py +++ b/src/utils/redis.py @@ -95,3 +95,30 @@ def write(key, data): # return fail status return False + + +def increment(key, amount=1): + """ + Increment value of amount to + specified key to Redis. + """ + + rds = connect() + + # increment data of key + try: + # increment and set new value + rds.incrby(key, amount) + + # return succes status + return True + + except Exception as redis_error: + # print query parse error and return empty dict + logging.error( + "An error occured while trying to increment value in Redis cache", + extra={"key": key, "error_msg": redis_error}, + ) + + # return fail status + return False diff --git a/src/utils/steam.py b/src/utils/steam.py index cb85deb..a878961 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -30,7 +30,7 @@ def get_app_list(): apps = [] if response.status_code != 200: - logger.error("The Steam GetAppList API endpoint returned a non-200 http code") + logging.error("The Steam GetAppList API endpoint returned a non-200 http code") return False else: @@ -143,11 +143,11 @@ def get_packages_info(packages=[]): info = client.get_product_info(packages=packages, timeout=5) info = info["packages"] except Exception as err: - logger.error( + logging.error( "Something went wrong while querying product info for packages: " + str(packages) ) - logger.error(err) + logging.error(err) return False return info diff --git a/src/utils/storage.py b/src/utils/storage.py index 04c82a4..efc137c 100644 --- a/src/utils/storage.py +++ b/src/utils/storage.py @@ -1,6 +1,7 @@ -from main import logger +from job import logger import utils.helper import config +import logging import os from minio import Minio from io import BytesIO @@ -76,7 +77,7 @@ def local_read(path, filename): content = content.read() except Exception: - logger.error("The following file could not be read: " + file) + logging.error("The following file could not be read: " + file) return False return content @@ -96,7 +97,7 @@ def local_write(content, path, filename): f = open(file, "w") f.write(content) f.close() - logger.info("Written the following file: " + file) + logging.info("Written the following file: " + file) except Exception: logger.error("The following file could not be written locally: " + file) @@ -117,7 +118,7 @@ def local_delete(path, filename): try: os.remove(file) except OSError as err: - logger.error( + logging.error( "The following file could not be deleted locally: " + err.filename + ". The following error occured: " @@ -141,7 +142,7 @@ def local_list(path): content = os.listdir(path) except Exception: - logger.error("The following directory could not be read: " + path) + logging.error("The following directory could not be read: " + path) return False return content @@ -184,7 +185,7 @@ def object_read(path, filename): content = conn.get_object(config.storage_object_bucket, file) except Exception: - logger.error("The following file could not be retrieved: " + file) + logging.error("The following file could not be retrieved: " + file) return False return content @@ -243,7 +244,7 @@ def object_list(path, details=False): file_list.append(file) except Exception: - logger.error("The files in following directory could not be listed: " + path) + logging.error("The files in following directory could not be listed: " + path) return False return file_list diff --git a/src/web.py b/src/web.py index 5c6aa11..327f0bb 100644 --- a/src/web.py +++ b/src/web.py @@ -1,5 +1,5 @@ """ -Main application and entrypoint. +Web Service and main API. """ # import modules @@ -11,7 +11,6 @@ import typing import logging from fastapi import FastAPI, Response -from functions import app_info # initialise app app = FastAPI() @@ -36,7 +35,7 @@ def read_app(app_id: int, pretty: bool = False): logging.info("Requested app info", extra={"apps": str([app_id])}) if config.cache == "True": - info = utils.redis.read(app_id) + info = utils.redis.read("app." + str(app_id)) if not info: logging.info( @@ -44,7 +43,7 @@ def read_app(app_id: int, pretty: bool = False): ) info = utils.steam.get_apps_info([app_id]) data = json.dumps(info) - utils.redis.write(app_id, data) + utils.redis.write("app." + str(app_id), data) else: info = json.loads(info) logging.info( From 3b7bfc27e7324e6feeb0dc06211169d410b34632 Mon Sep 17 00:00:00 2001 From: Thuis Date: Wed, 29 Jan 2025 15:20:16 +0100 Subject: [PATCH 08/19] Rewrite --- .gitignore | 3 ++- README.md | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index e55a532..fb19efb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ __pycache__ _test.py data/ -celerybeat-schedule.db \ No newline at end of file +celerybeat-schedule.db +celerybeat-schedule \ No newline at end of file diff --git a/README.md b/README.md index 90f0562..64a06b6 100644 --- a/README.md +++ b/README.md @@ -113,14 +113,23 @@ LOG_LEVEL=info ## Development -Run the api locally by installing a web server like uvicorn and running it: +Run the Web Service (FastAPI) locally by running the FastAPI development server: ```bash python3 -m venv .venv source .venv/bin/activate pip install -r requirements.txt -pip install uvicorn cd src/ -uvicorn main:app --reload +fastapi dev web.py +``` +Now you can reach the SteamCMD API locally on [http://localhost:8000](http://localhost:8000). + +Run the Job Service (Celery) locally by running celery directly: +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +cd src/ +celery -A job worker --loglevel=info --concurrency=2 --beat ``` The easiest way to spin up a complete development environment is using Docker From f4c21a8df805f24face5677cbb38ac1702985bfb Mon Sep 17 00:00:00 2001 From: Thuis Date: Wed, 29 Jan 2025 19:36:24 +0100 Subject: [PATCH 09/19] WIP --- src/utils/steam.py | 155 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 137 insertions(+), 18 deletions(-) diff --git a/src/utils/steam.py b/src/utils/steam.py index a878961..c044d2c 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -19,6 +19,42 @@ def init_client(): return client +#def init_client(): +# """ +# Initialize Steam client, login and +# return the client. +# """ +# +# connect_retries = 2 +# connect_timeout = 3 +# +# try: +# # Sometimes it hangs for 30+ seconds. Normal connection takes about 500ms +# for _ in range(connect_retries): +# count = str(_) +# +# try: +# with gevent.Timeout(connect_timeout): +# +# logging.debug("Connecting via steamclient to steam api") +# client = SteamClient() +# client.anonymous_login() +# client.verbose_debug = False +# +# return client +# +# except gevent.timeout.Timeout: +# client._connecting = False +# +# else: +# break +# else: +# raise Exception(f"Max connect retries ({connect_retries}) exceeded") +# +# except Exception as err: +# return False + + def get_app_list(): """ Get list of id's of all current apps in @@ -40,16 +76,76 @@ def get_app_list(): return apps +#def get_change_number(): +# """ +# Get and return the latest change number. +# """ +# +# client = init_client() +# info = client.get_changes_since(1, app_changes=False, package_changes=False) +# change_number = info.current_change_number +# +# return change_number + + def get_change_number(): """ Get and return the latest change number. """ - client = init_client() - info = client.get_changes_since(1, app_changes=False, package_changes=False) - change_number = info.current_change_number + connect_retries = 2 + connect_timeout = 3 + + try: + # Sometimes it hangs for 30+ seconds. Normal connection takes about 500ms + for _ in range(connect_retries): + count = str(_) + + try: + with gevent.Timeout(connect_timeout): + + client = init_client() + info = client.get_changes_since(1, app_changes=False, package_changes=False) + change_number = info.current_change_number + + return change_number + + except gevent.timeout.Timeout: + client._connecting = False + + else: + break + else: + raise Exception(f"Max connect retries ({connect_retries}) exceeded") + + except Exception as err: + return False + - return change_number +#def get_changes_since_change_number(change_number): +# """ +# Get and return lists of changed apps and +# packages since the specified change number. +# """ +# +# client = init_client() +# info = client.get_changes_since( +# int(change_number), app_changes=True, package_changes=True +# ) +# +# app_list = [] +# if info.app_changes: +# for app in info.app_changes: +# app_list.append(app.appid) +# +# package_list = [] +# if info.package_changes: +# for package in info.package_changes: +# package_list.append(package.packageid) +# +# changes = {"apps": app_list, "packages": package_list} +# +# return changes def get_changes_since_change_number(change_number): @@ -58,24 +154,47 @@ def get_changes_since_change_number(change_number): packages since the specified change number. """ - client = init_client() - info = client.get_changes_since( - int(change_number), app_changes=True, package_changes=True - ) + connect_retries = 2 + connect_timeout = 3 + + try: + # Sometimes it hangs for 30+ seconds. Normal connection takes about 500ms + for _ in range(connect_retries): + count = str(_) + + try: + with gevent.Timeout(connect_timeout): - app_list = [] - if info.app_changes: - for app in info.app_changes: - app_list.append(app.appid) - package_list = [] - if info.package_changes: - for package in info.package_changes: - package_list.append(package.packageid) + client = init_client() + info = client.get_changes_since( + int(change_number), app_changes=True, package_changes=True + ) - changes = {"apps": app_list, "packages": package_list} + app_list = [] + if info.app_changes: + for app in info.app_changes: + app_list.append(app.appid) - return changes + package_list = [] + if info.package_changes: + for package in info.package_changes: + package_list.append(package.packageid) + + changes = {"apps": app_list, "packages": package_list} + + return changes + + except gevent.timeout.Timeout: + client._connecting = False + + else: + break + else: + raise Exception(f"Max connect retries ({connect_retries}) exceeded") + + except Exception as err: + return False def get_apps_info(apps=[]): From 50fe97bbc44fc716191e411a3939ef3cff6b5339 Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Mon, 3 Feb 2025 19:42:20 +0100 Subject: [PATCH 10/19] Init WIP --- src/tasks/check_changelist.py | 12 ++++ src/tasks/get_app_info.py | 5 +- src/utils/redis.py | 2 + src/utils/steam.py | 120 ++++++++++++---------------------- 4 files changed, 58 insertions(+), 81 deletions(-) diff --git a/src/tasks/check_changelist.py b/src/tasks/check_changelist.py index f781263..1563b4f 100644 --- a/src/tasks/check_changelist.py +++ b/src/tasks/check_changelist.py @@ -7,6 +7,7 @@ import logging import config import json +import time @app.task(name="check_changelist", base=Singleton, lock_expiry=10) @@ -23,6 +24,13 @@ def check_changelist_task(): logging.warning("Previous changenumber could not be retrieved from Redis") utils.redis.write("_state.change_number", latest_change_number) + elif not latest_change_number: + logging.error( + "The current change number could not be retrieved. Instead got: " + + str(latest_change_number) + ) + pass + elif int(previous_change_number) == int(latest_change_number): logging.info( "The previous and current change number " @@ -35,6 +43,10 @@ def check_changelist_task(): logging.info("The changenumber has been updated from " + str(previous_change_number) + " to " + str(latest_change_number)) changes = utils.steam.get_changes_since_change_number(previous_change_number) + while not changes: + changes = utils.steam.get_changes_since_change_number(previous_change_number) + time.sleep(1) + for i in range(0, len(changes["apps"]), config.chunk_size): chunk = changes["apps"][i : i + config.chunk_size] get_app_info_task.delay(chunk) diff --git a/src/tasks/get_app_info.py b/src/tasks/get_app_info.py index e220bbe..148496b 100644 --- a/src/tasks/get_app_info.py +++ b/src/tasks/get_app_info.py @@ -6,7 +6,7 @@ @app.task( name="get_app_info", - time_limit=3, + time_limit=15, autoretry_for=(Exception,), retry_kwargs={"max_retries": 3, "countdown": 5}, ) @@ -20,5 +20,6 @@ def get_app_info_task(apps=[]): apps = utils.steam.get_apps_info(apps) for app_obj in apps: - content = json.dumps(apps[app_obj]) + content = { app_obj : apps[app_obj] } + content = json.dumps(content) utils.redis.write("app." + str(app_obj), content) diff --git a/src/utils/redis.py b/src/utils/redis.py index ffb3811..93bb1f5 100644 --- a/src/utils/redis.py +++ b/src/utils/redis.py @@ -61,6 +61,7 @@ def read(key): "An error occured while trying to read and decode from Redis", extra={"key": key, "error_msg": redis_error}, ) + logging.error(redis_error) # return failed status return False @@ -92,6 +93,7 @@ def write(key, data): "An error occured while trying to write to Redis cache", extra={"key": key, "error_msg": redis_error}, ) + logging.error(redis_error) # return fail status return False diff --git a/src/utils/steam.py b/src/utils/steam.py index c044d2c..ed04331 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -19,42 +19,6 @@ def init_client(): return client -#def init_client(): -# """ -# Initialize Steam client, login and -# return the client. -# """ -# -# connect_retries = 2 -# connect_timeout = 3 -# -# try: -# # Sometimes it hangs for 30+ seconds. Normal connection takes about 500ms -# for _ in range(connect_retries): -# count = str(_) -# -# try: -# with gevent.Timeout(connect_timeout): -# -# logging.debug("Connecting via steamclient to steam api") -# client = SteamClient() -# client.anonymous_login() -# client.verbose_debug = False -# -# return client -# -# except gevent.timeout.Timeout: -# client._connecting = False -# -# else: -# break -# else: -# raise Exception(f"Max connect retries ({connect_retries}) exceeded") -# -# except Exception as err: -# return False - - def get_app_list(): """ Get list of id's of all current apps in @@ -76,18 +40,6 @@ def get_app_list(): return apps -#def get_change_number(): -# """ -# Get and return the latest change number. -# """ -# -# client = init_client() -# info = client.get_changes_since(1, app_changes=False, package_changes=False) -# change_number = info.current_change_number -# -# return change_number - - def get_change_number(): """ Get and return the latest change number. @@ -95,6 +47,7 @@ def get_change_number(): connect_retries = 2 connect_timeout = 3 + client = None try: # Sometimes it hangs for 30+ seconds. Normal connection takes about 500ms @@ -108,44 +61,33 @@ def get_change_number(): info = client.get_changes_since(1, app_changes=False, package_changes=False) change_number = info.current_change_number + client.logout() + return change_number except gevent.timeout.Timeout: - client._connecting = False + if client: + client._connecting = False + client.logout() else: break else: + if client: + client._connecting = False + client.logout() raise Exception(f"Max connect retries ({connect_retries}) exceeded") except Exception as err: - return False + if client: + client._connecting = False + client.logout() - -#def get_changes_since_change_number(change_number): -# """ -# Get and return lists of changed apps and -# packages since the specified change number. -# """ -# -# client = init_client() -# info = client.get_changes_since( -# int(change_number), app_changes=True, package_changes=True -# ) -# -# app_list = [] -# if info.app_changes: -# for app in info.app_changes: -# app_list.append(app.appid) -# -# package_list = [] -# if info.package_changes: -# for package in info.package_changes: -# package_list.append(package.packageid) -# -# changes = {"apps": app_list, "packages": package_list} -# -# return changes + logging.error( + "Encountered the following error when trying to retrieve latest change number: " + + str(err) + ) + return False def get_changes_since_change_number(change_number): @@ -156,6 +98,7 @@ def get_changes_since_change_number(change_number): connect_retries = 2 connect_timeout = 3 + client = None try: # Sometimes it hangs for 30+ seconds. Normal connection takes about 500ms @@ -183,17 +126,27 @@ def get_changes_since_change_number(change_number): changes = {"apps": app_list, "packages": package_list} + client.logout() + return changes except gevent.timeout.Timeout: - client._connecting = False + if client: + client._connecting = False + client.logout() else: break else: + if client: + client._connecting = False + client.logout() raise Exception(f"Max connect retries ({connect_retries}) exceeded") except Exception as err: + if client: + client._connecting = False + client.logout() return False @@ -205,6 +158,7 @@ def get_apps_info(apps=[]): connect_retries = 2 connect_timeout = 3 + client = None logging.info("Started requesting app info", extra={"apps": str(apps)}) @@ -226,18 +180,25 @@ def get_apps_info(apps=[]): info = client.get_product_info(apps=apps, timeout=1) info = info["apps"] + client.logout() + return info except gevent.timeout.Timeout: logging.warning( "Encountered timeout when trying to connect to steam api. Retrying.." ) - client._connecting = False + if client: + client._connecting = False + client.logout() else: logging.info("Succesfully retrieved app info", extra={"apps": str(apps)}) break else: + if client: + client._connecting = False + client.logout() logging.error( "Max connect retries exceeded", extra={"apps": str(apps), "connect_retries": connect_retries}, @@ -245,11 +206,12 @@ def get_apps_info(apps=[]): raise Exception(f"Max connect retries ({connect_retries}) exceeded") except Exception as err: + if client: + client._connecting = False + client.logout() logging.error("Failed in retrieving app info with error: " + str(err), extra={"apps": str(apps)}) return False - return info - def get_packages_info(packages=[]): """ From 7b98cbcc05bdd54b4f7dfc86f8a58bca1f4948f4 Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Sat, 22 Feb 2025 23:25:49 +0100 Subject: [PATCH 11/19] Add check to fix incorrect cached data --- .gitignore | 3 ++- src/config.py | 4 +++ src/tasks/check_incorrect_apps.py | 43 +++++++++++++++++++++++++++++++ src/tasks/check_missing_apps.py | 1 - 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/tasks/check_incorrect_apps.py diff --git a/.gitignore b/.gitignore index fb19efb..4e12a83 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ __pycache__ _test.py data/ celerybeat-schedule.db -celerybeat-schedule \ No newline at end of file +celerybeat-schedule-* +celerybeat-schedule diff --git a/src/config.py b/src/config.py index 97d88da..4105b0f 100644 --- a/src/config.py +++ b/src/config.py @@ -55,6 +55,10 @@ # "task": "check_missing_apps", # "schedule": 1800.0, #}, + "check-incorrect-apps-every-30-minutes": { + "task": "check_incorrect_apps", + "schedule": 1800.0, + }, "check-deadlocks-every-1-hour": { "task": "check_deadlocks", "schedule": 3600.0, diff --git a/src/tasks/check_incorrect_apps.py b/src/tasks/check_incorrect_apps.py new file mode 100644 index 0000000..ce87f06 --- /dev/null +++ b/src/tasks/check_incorrect_apps.py @@ -0,0 +1,43 @@ +from job import app, logger +from celery_singleton import Singleton +from .get_app_info import get_app_info_task +import utils.redis +import utils.steam +import config + + +@app.task(name="check_incorrect_apps", base=Singleton, lock_expiry=7200) +def check_incorrect_apps_task(): + """ + Check for stored apps that have the value of "false" and + remove them from the cache. Then start tasks to retrieve + the info for these apps. + """ + + # connecting to Redis + rds = utils.redis.connect() + + # initialize cursor + cursor = 0 + false_apps = [] + + # use SCAN to iterate through keys that start with "app." + while True: + cursor, apps = rds.scan(cursor, match='app.*') + for app in apps: + # Check if the value of the app is "false" + if rds.get(app) == b'false': + app = app.decode("UTF-8") + app = app.split(".")[1] + false_apps.append(int(app)) + + if cursor == 0: + break + + if len(false_apps) > 0: + logger.warning("Found " + str(len(false_apps)) + " apps that have a stored value of 'false'") + + for i in range(0, len(false_apps), config.chunk_size): + chunk = false_apps[i : i + config.chunk_size] + logger.warning("Deleting " + str(chunk) + " apps from cache and starting app info retrieval again") + get_app_info_task.delay(chunk) diff --git a/src/tasks/check_missing_apps.py b/src/tasks/check_missing_apps.py index fcfc47a..da7ca9f 100644 --- a/src/tasks/check_missing_apps.py +++ b/src/tasks/check_missing_apps.py @@ -1,7 +1,6 @@ from job import app, logger from celery_singleton import Singleton from .get_app_info import get_app_info_task -import utils.storage import utils.steam import config From 74b8527429517e2a7467a30c2794a3b56e8808b4 Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Sat, 22 Feb 2025 23:37:33 +0100 Subject: [PATCH 12/19] Add celery files to docker ignore --- .dockerignore | 5 +++++ .gitignore | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 567fad5..cee03ca 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,8 @@ __pycache__ .venv .env + +_test.py +celerybeat-schedule.db +celerybeat-schedule-* +celerybeat-schedule diff --git a/.gitignore b/.gitignore index 4e12a83..aa25c57 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ __pycache__ .env _test.py -data/ celerybeat-schedule.db celerybeat-schedule-* celerybeat-schedule From 798773250f7a574db4ad105d654860395c38366f Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Mon, 24 Feb 2025 20:13:43 +0100 Subject: [PATCH 13/19] Reformat for black formatting --- src/tasks/check_changelist.py | 13 ++++++++++--- src/tasks/check_incorrect_apps.py | 16 ++++++++++++---- src/tasks/get_app_info.py | 2 +- src/tasks/get_package_info.py | 2 +- src/utils/redis.py | 6 ++---- src/utils/steam.py | 18 +++++++++++++----- src/web.py | 1 + 7 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/tasks/check_changelist.py b/src/tasks/check_changelist.py index 1563b4f..f5710ef 100644 --- a/src/tasks/check_changelist.py +++ b/src/tasks/check_changelist.py @@ -40,18 +40,25 @@ def check_changelist_task(): pass else: - logging.info("The changenumber has been updated from " + str(previous_change_number) + " to " + str(latest_change_number)) + logging.info( + "The changenumber has been updated from " + + str(previous_change_number) + + " to " + + str(latest_change_number) + ) changes = utils.steam.get_changes_since_change_number(previous_change_number) while not changes: - changes = utils.steam.get_changes_since_change_number(previous_change_number) + changes = utils.steam.get_changes_since_change_number( + previous_change_number + ) time.sleep(1) for i in range(0, len(changes["apps"]), config.chunk_size): chunk = changes["apps"][i : i + config.chunk_size] get_app_info_task.delay(chunk) - #for i in range(0, len(changes["packages"]), config.chunk_size): + # for i in range(0, len(changes["packages"]), config.chunk_size): # chunk = changes["packages"][i : i + config.chunk_size] # get_package_info_task.delay(chunk) diff --git a/src/tasks/check_incorrect_apps.py b/src/tasks/check_incorrect_apps.py index ce87f06..e8a1500 100644 --- a/src/tasks/check_incorrect_apps.py +++ b/src/tasks/check_incorrect_apps.py @@ -23,10 +23,10 @@ def check_incorrect_apps_task(): # use SCAN to iterate through keys that start with "app." while True: - cursor, apps = rds.scan(cursor, match='app.*') + cursor, apps = rds.scan(cursor, match="app.*") for app in apps: # Check if the value of the app is "false" - if rds.get(app) == b'false': + if rds.get(app) == b"false": app = app.decode("UTF-8") app = app.split(".")[1] false_apps.append(int(app)) @@ -35,9 +35,17 @@ def check_incorrect_apps_task(): break if len(false_apps) > 0: - logger.warning("Found " + str(len(false_apps)) + " apps that have a stored value of 'false'") + logger.warning( + "Found " + + str(len(false_apps)) + + " apps that have a stored value of 'false'" + ) for i in range(0, len(false_apps), config.chunk_size): chunk = false_apps[i : i + config.chunk_size] - logger.warning("Deleting " + str(chunk) + " apps from cache and starting app info retrieval again") + logger.warning( + "Deleting " + + str(chunk) + + " apps from cache and starting app info retrieval again" + ) get_app_info_task.delay(chunk) diff --git a/src/tasks/get_app_info.py b/src/tasks/get_app_info.py index 148496b..42b05c5 100644 --- a/src/tasks/get_app_info.py +++ b/src/tasks/get_app_info.py @@ -20,6 +20,6 @@ def get_app_info_task(apps=[]): apps = utils.steam.get_apps_info(apps) for app_obj in apps: - content = { app_obj : apps[app_obj] } + content = {app_obj: apps[app_obj]} content = json.dumps(content) utils.redis.write("app." + str(app_obj), content) diff --git a/src/tasks/get_package_info.py b/src/tasks/get_package_info.py index 746ea2f..0f3a167 100644 --- a/src/tasks/get_package_info.py +++ b/src/tasks/get_package_info.py @@ -21,4 +21,4 @@ def get_package_info_task(packages=[]): for package_obj in packages: content = json.dumps(packages[package_obj]) - utils.redis.write("package." + str(package_obj), content) \ No newline at end of file + utils.redis.write("package." + str(package_obj), content) diff --git a/src/utils/redis.py b/src/utils/redis.py index 93bb1f5..09f0341 100644 --- a/src/utils/redis.py +++ b/src/utils/redis.py @@ -18,13 +18,11 @@ def connect(): host=config.redis_host, port=config.redis_port, password=config.redis_password, - db=config.redis_database + db=config.redis_database, ) else: rds = redis.Redis( - host=config.redis_host, - port=config.redis_port, - db=config.redis_database + host=config.redis_host, port=config.redis_port, db=config.redis_database ) except Exception as error: diff --git a/src/utils/steam.py b/src/utils/steam.py index ed04331..88c617a 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -58,7 +58,9 @@ def get_change_number(): with gevent.Timeout(connect_timeout): client = init_client() - info = client.get_changes_since(1, app_changes=False, package_changes=False) + info = client.get_changes_since( + 1, app_changes=False, package_changes=False + ) change_number = info.current_change_number client.logout() @@ -108,7 +110,6 @@ def get_changes_since_change_number(change_number): try: with gevent.Timeout(connect_timeout): - client = init_client() info = client.get_changes_since( int(change_number), app_changes=True, package_changes=True @@ -176,7 +177,9 @@ def get_apps_info(apps=[]): client = init_client() - logging.debug("Requesting app info from steam api", extra={"apps": str(apps)}) + logging.debug( + "Requesting app info from steam api", extra={"apps": str(apps)} + ) info = client.get_product_info(apps=apps, timeout=1) info = info["apps"] @@ -193,7 +196,9 @@ def get_apps_info(apps=[]): client.logout() else: - logging.info("Succesfully retrieved app info", extra={"apps": str(apps)}) + logging.info( + "Succesfully retrieved app info", extra={"apps": str(apps)} + ) break else: if client: @@ -209,7 +214,10 @@ def get_apps_info(apps=[]): if client: client._connecting = False client.logout() - logging.error("Failed in retrieving app info with error: " + str(err), extra={"apps": str(apps)}) + logging.error( + "Failed in retrieving app info with error: " + str(err), + extra={"apps": str(apps)}, + ) return False diff --git a/src/web.py b/src/web.py index 327f0bb..825ee79 100644 --- a/src/web.py +++ b/src/web.py @@ -15,6 +15,7 @@ # initialise app app = FastAPI() + # include "pretty" for backwards compatibility class PrettyJSONResponse(Response): media_type = "application/json" From 905510bf4b478da0c63fa8d985282ae006df87dc Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Sat, 1 Mar 2025 13:10:04 +0100 Subject: [PATCH 14/19] remove obsolete docs --- README.md | 64 ++++++++----------------------------------------------- 1 file changed, 9 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 64a06b6..ce3f98c 100644 --- a/README.md +++ b/README.md @@ -11,53 +11,12 @@ # SteamCMD API -Read-only API interface for steamcmd app_info. Updates of this code are -automatically deployed via [Github Actions](https://github.com/steamcmd/api/actions) -when a new version has been created on Github. +Read-only API interface for steamcmd app_info. The official API is reachable on +[api.steamcmd.net](https://api.steamcmd.net) but can be fairly easily self-hosted. +Read more about the public API on [www.steamcmd.net](https://www.steamcmd.net). ## Self-hosting -The easiest way to host the API yourself is using the free cloud platform -[Fly.io](https://fly.io). Install the CLI according to the documentation: -[https://fly.io/docs/hands-on/install-flyctl/](https://fly.io/docs/hands-on/install-flyctl/). - -After installing, authenticate locally with the `flyctl` cli: -```bash -fly auth login -``` -Create the app and redis instances (choose your own names): -```bash -fly apps create -fly redis create -``` -Retrieve the Redis connection URL (you will need this later): -```bash -fly redis status - -Redis - ID = xxxxxxxxxxxxxxxxxx - Name = api - Plan = Free - Primary Region = ams - Read Regions = None - Eviction = Enabled - Private URL = redis://default:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@fly-api.upstash.io <== Write the password down -``` -Set the required configuration environment variables: -```bash -fly secrets set --app \ - CACHE=True \ - CACHE_TYPE=redis \ - CACHE_EXPIRATION=120 \ - REDIS_HOST="fly-api.upstash.io" \ - REDIS_PORT=6379 \ - REDIS_PASSWORD="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" -``` -Finally deploy the API Docker image with the latest code: -```bash -fly deploy --app --image steamcmd/api:latest -e VERSION=1.0.0 -``` -The version is optional and currently only required for the `/v1/version` endpoint. ## Container @@ -113,11 +72,16 @@ LOG_LEVEL=info ## Development -Run the Web Service (FastAPI) locally by running the FastAPI development server: +To develop locally start by creating a Python virtual environment and install the prerequisites: ```bash python3 -m venv .venv source .venv/bin/activate pip install -r requirements.txt +``` + +Run the Web Service (FastAPI) locally by running the FastAPI development server: +```bash +source .venv/bin/activate cd src/ fastapi dev web.py ``` @@ -132,16 +96,6 @@ cd src/ celery -A job worker --loglevel=info --concurrency=2 --beat ``` -The easiest way to spin up a complete development environment is using Docker -compose. This will build the image locally, mount the correct directory (`src`) -and set the required environment variables. If you are on windows you should -store the repository in the WSL filesystem or it will fail. Execute compose up -in the root: -```bash -docker compose up -``` -Now you can reach the SteamCMD API locally on [http://localhost:8000](http://localhost:8000). - ### Black To keep things simple, [Black](https://github.com/python/black) is used for code From a4d6636292af0c0c4d1a31c78b97fe49d7e2355f Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Tue, 21 Jan 2025 12:45:50 +0100 Subject: [PATCH 15/19] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ce3f98c..534540e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Discord Online](https://img.shields.io/discord/928592378711912488.svg)](https://discord.steamcmd.net) [![Mastodon Follow](https://img.shields.io/mastodon/follow/109302774947550572?domain=https%3A%2F%2Ffosstodon.org&style=flat)](https://fosstodon.org/@steamcmd) [![Image Size](https://img.shields.io/docker/image-size/steamcmd/api/latest.svg)](https://hub.docker.com/r/steamcmd/api) -[![Better Uptime](https://betteruptime.com/status-badges/v1/monitor/ln3p.svg)](https://status.steamcmd.net) +[![Uptime Robot Uptime](https://img.shields.io/uptimerobot/ratio/m782827237-5067fd1d69e3b1b2e4e40fff)](https://status.steamcmd.net) [![GitHub Release](https://img.shields.io/github/v/release/steamcmd/api?label=version)](https://github.com/steamcmd/api/releases) [![GitHub Sponsors](https://img.shields.io/github/sponsors/steamcmd)](https://github.com/sponsors/steamcmd) [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) From ef1214b8d2298c060777b8eb6d2f29116ba775e0 Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Sat, 1 Mar 2025 13:18:08 +0100 Subject: [PATCH 16/19] Remove obsolete deploy tasks and add GHCR registry --- .github/workflows/deploy.yml | 45 +++++++++++------------------------- .github/workflows/test.yml | 4 ++-- src/_test.py | 30 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 34 deletions(-) create mode 100755 src/_test.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5785e6e..e210819 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -8,7 +8,7 @@ on: jobs: check-requirements: name: Check Requirements - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Set Version Tag run: echo "API_TAG=$(echo $GITHUB_REF | awk -F '/' '{print $NF}')" >> $GITHUB_ENV @@ -17,25 +17,31 @@ jobs: build-image: name: Build Image - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: check-requirements steps: - uses: actions/checkout@v4 - name: Parse API Version run: echo "API_VERSION=$(echo $GITHUB_REF | awk -F '/' '{print $NF}' | cut -c 2-)" >> $GITHUB_ENV - name: Docker Login - run: echo ${{ secrets.DOCKER_ACCESS_TOKEN }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + run: | + echo ${{ secrets.DOCKER_ACCESS_TOKEN }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + echo ${{ secrets.GHCRIO_ACCESS_TOKEN }} | docker login ghcr.io -u ${{ secrets.GHCRIO_USERNAME }} --password-stdin - name: Build Image - run: docker build -t steamcmd/api:latest . + run: docker build -t steamcmd/api:latest -t ghcr.io/steamcmd/api:latest . # deploy - name: Tag Image - run: docker tag steamcmd/api:latest steamcmd/api:$API_VERSION + run: | + docker tag steamcmd/api:latest steamcmd/api:$API_VERSION + docker tag ghcr.io/steamcmd/api:latest ghcr.io/steamcmd/api:$API_VERSION - name: Push Image - run: docker push steamcmd/api --all-tags + run: | + docker push steamcmd/api --all-tags + docker push ghcr.io/steamcmd/api --all-tags update-readme: name: Update Readme - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Update Docker Hub Description @@ -44,28 +50,3 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} DOCKERHUB_REPOSITORY: steamcmd/api - - deploy-fly: - name: Deploy Fly.io - runs-on: ubuntu-22.04 - needs: [check-requirements, build-image] - steps: - - uses: actions/checkout@v4 - - uses: superfly/flyctl-actions/setup-flyctl@master - - name: Parse API Version - run: echo "API_VERSION=$(echo $GITHUB_REF | awk -F '/' '{print $NF}' | cut -c 2-)" >> $GITHUB_ENV - - name: Deploy API on Fly.io - run: flyctl deploy --app steamcmd --image steamcmd/api:${{ env.API_VERSION }} -e VERSION=${{ env.API_VERSION }} - env: - FLY_API_TOKEN: ${{ secrets.FLY_ACCESS_TOKEN }} - - deploy-render: - name: Deploy Render.com - runs-on: ubuntu-22.04 - needs: [check-requirements, build-image] - steps: - - uses: actions/checkout@v4 - - name: Parse API Version - run: echo "API_VERSION=$(echo $GITHUB_REF | awk -F '/' '{print $NF}' | cut -c 2-)" >> $GITHUB_ENV - - name: Deploy API on Render.com - run: curl https://api.render.com/deploy/${{ secrets.RENDER_SERVICE_ID }}?key=${{ secrets.RENDER_API_KEY }}&imgURL=docker.io%2Fsteamcmd%2Fapi%40${{ env.API_VERSION }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f6bbbc5..19bce21 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ on: jobs: test-image: name: Test Image - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Build Image @@ -19,7 +19,7 @@ jobs: python-lint: name: Python Lint - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - uses: jpetrucciani/ruff-check@main diff --git a/src/_test.py b/src/_test.py new file mode 100755 index 0000000..f2425b6 --- /dev/null +++ b/src/_test.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import redis + +rds = redis.Redis( + host="steamcmd-redis-svc", + port=6379, + password="4jch9e2w4xptUqCSTq8lnyX", + db=0 +) + +# Initialize cursor +cursor = 0 +false_apps = [] + +# Use SCAN to iterate through keys that start with "app." +while True: + cursor, apps = rds.scan(cursor, match='app.*') + for app in apps: + # Check if the value of the app is "false" + if rds.get(app) == b'false': + app = app.decode("UTF-8") + app = app.split(".")[1] + false_apps.append(int(app)) + + if cursor == 0: + break + +for app in false_apps: + print(app) \ No newline at end of file From a7f8c852337cf1e594d5dd9b07acd9a95e962d7d Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Sat, 15 Mar 2025 20:34:30 +0100 Subject: [PATCH 17/19] Delete test file --- src/_test.py | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100755 src/_test.py diff --git a/src/_test.py b/src/_test.py deleted file mode 100755 index f2425b6..0000000 --- a/src/_test.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python3 - -import redis - -rds = redis.Redis( - host="steamcmd-redis-svc", - port=6379, - password="4jch9e2w4xptUqCSTq8lnyX", - db=0 -) - -# Initialize cursor -cursor = 0 -false_apps = [] - -# Use SCAN to iterate through keys that start with "app." -while True: - cursor, apps = rds.scan(cursor, match='app.*') - for app in apps: - # Check if the value of the app is "false" - if rds.get(app) == b'false': - app = app.decode("UTF-8") - app = app.split(".")[1] - false_apps.append(int(app)) - - if cursor == 0: - break - -for app in false_apps: - print(app) \ No newline at end of file From e50e37a0ccad835e9fe916c8a5096378a07b3cc3 Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Sat, 15 Mar 2025 21:55:45 +0100 Subject: [PATCH 18/19] Rewrite part of the README to reflect the web and job service setup --- README.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 534540e..93182ac 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,12 @@ # SteamCMD API Read-only API interface for steamcmd app_info. The official API is reachable on -[api.steamcmd.net](https://api.steamcmd.net) but can be fairly easily self-hosted. -Read more about the public API on [www.steamcmd.net](https://www.steamcmd.net). +[api.steamcmd.net](https://api.steamcmd.net) and it's documentation can be found +on [www.steamcmd.net](https://www.steamcmd.net). ## Self-hosting - -## Container - -The API can easily be run via a Docker image which contains the API code and the +The API can easily be run via a container image which contains the API code and the `uvicorn` tool to be able to respond to web requests. With every new version of the API the Docker images is automatically rebuild and pushed to the Docker Hub: ```bash @@ -32,8 +29,16 @@ docker pull steamcmd/api:1.10.0 ```bash docker run -p 8000:8000 -d steamcmd/api:latest ``` -However during development, using Docker Compose is preferred. See the -[Development](#development) section for information. +The API consists of 2 services; the **Web** and the **Job** service and the Redis +cache. The **Job** service and the Redis cache are both optional but are both required +if you want to run the **Job** service. + +Details on how the official API is hosted can be found in the +[platform](https://github.com/steamcmd/platform) repository. This repository contains +all the infrastructure as code that is used to deploy the API on a Kubernetes cluster. + +See the [Development](#development) section for more information on running +the API and Job services directly via Python. ## Configuration @@ -48,7 +53,7 @@ that you will need to set the corresponding cache settings for that type as well when using the **redis** type). All the available options in an `.env` file: -``` +```shell # general VERSION=1.0.0 From 2a2312dac18b5eeadf25c88c3c7cc5af912c1fc2 Mon Sep 17 00:00:00 2001 From: Jona Koudijs Date: Sat, 15 Mar 2025 22:35:30 +0100 Subject: [PATCH 19/19] Satisfy part of Python tests --- src/config.py | 1 - src/tasks/check_changelist.py | 8 +------- src/utils/steam.py | 5 ++++- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/config.py b/src/config.py index 4105b0f..8123ff1 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,6 @@ import utils.general import utils.helper import logging -import config from dotenv import load_dotenv from logfmter import Logfmter diff --git a/src/tasks/check_changelist.py b/src/tasks/check_changelist.py index f5710ef..56fab17 100644 --- a/src/tasks/check_changelist.py +++ b/src/tasks/check_changelist.py @@ -1,12 +1,10 @@ -from job import app, logger +from job import app from celery_singleton import Singleton from .get_app_info import get_app_info_task -from .get_package_info import get_package_info_task import utils.steam import utils.redis import logging import config -import json import time @@ -58,10 +56,6 @@ def check_changelist_task(): chunk = changes["apps"][i : i + config.chunk_size] get_app_info_task.delay(chunk) - # for i in range(0, len(changes["packages"]), config.chunk_size): - # chunk = changes["packages"][i : i + config.chunk_size] - # get_package_info_task.delay(chunk) - utils.redis.write("_state.change_number", latest_change_number) utils.redis.increment("_state.changed_apps", len(changes["apps"])) utils.redis.increment("_state.changed_packages", len(changes["packages"])) diff --git a/src/utils/steam.py b/src/utils/steam.py index 88c617a..4822f4f 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -1,4 +1,3 @@ -import config import gevent import logging import requests @@ -148,6 +147,10 @@ def get_changes_since_change_number(change_number): if client: client._connecting = False client.logout() + logging.error( + "Failed in get changes since last change number with error: " + str(err), + extra={"changenumber": str(change_number)}, + ) return False