From 8ae8f8df0c954ac9346f5281f8253244f1f4280f Mon Sep 17 00:00:00 2001 From: AuxXxilium Date: Mon, 3 Feb 2025 18:49:04 +0100 Subject: [PATCH] page: update Signed-off-by: AuxXxilium --- .github/workflows/data.yml | 17 +-- scripts/func.py | 230 --------------------------------- scripts/functions.py | 253 +++++++++++++++++++++++++++++++++++++ 3 files changed, 258 insertions(+), 242 deletions(-) delete mode 100755 scripts/func.py create mode 100755 scripts/functions.py diff --git a/.github/workflows/data.yml b/.github/workflows/data.yml index 2ca91e32..18649274 100644 --- a/.github/workflows/data.yml +++ b/.github/workflows/data.yml @@ -74,15 +74,8 @@ jobs: - name: Get Config Data for Arc run: | REPO="https://github.com/AuxXxilium/arc-configs" - PRERELEASE="true" - - TAG="" - if [ "${PRERELEASE}" = "true" ]; then - TAG="$(curl -skL --connect-timeout 10 "${REPO}/tags" | grep /refs/tags/.*\.zip | sed -r 's/.*\/refs\/tags\/(.*)\.zip.*$/\1/' | sort -rV | head -1)" - else - LATESTURL="$(curl -skL --connect-timeout 10 -w %{url_effective} -o /dev/null "${REPO}/releases/latest")" - TAG="${LATESTURL##*/}" - fi + LATESTURL="$(curl -skL --connect-timeout 10 -w %{url_effective} -o /dev/null "${REPO}/releases/latest")" + TAG="${LATESTURL##*/}" [ "${TAG:0:1}" = "v" ] && TAG="${TAG:1}" rm -f configs.zip STATUS=$(curl -kL --connect-timeout 10 -w "%{http_code}" "${REPO}/releases/download/${TAG}/configs-${TAG}.zip" -o "configs.zip") @@ -98,9 +91,9 @@ jobs: - name: Get data run: | pip install -r scripts/requirements.txt - python scripts/func.py getmodels -w "." -j "docs/models.json" - python scripts/func.py getaddons -w "." -j "docs/addons.json" - python scripts/func.py getpats -w "." -j "docs/pats.json" + python scripts/functions.py getmodels -w "." -j "docs/models.json" + python scripts/functions.py getaddons -w "." -j "docs/addons.json" + python scripts/functions.py getpats -w "." -j "docs/pats.json" - name: Upload to Artifacts if: success() diff --git a/scripts/func.py b/scripts/func.py deleted file mode 100755 index c2d4294e..00000000 --- a/scripts/func.py +++ /dev/null @@ -1,230 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2023 AuxXxilium and Ing -# -# This is free software, licensed under the MIT License. -# See /LICENSE for more information. -# - -import os, sys, glob, json, click, shutil, tarfile, kmodule, requests -import yaml -from openpyxl import Workbook - - -@click.group() -def cli(): - """ - The CLI is a commands to Arc. - """ - pass - - -@cli.command() -@click.option("-w", "--workpath", type=str, required=True, help="The workpath of Arc.") -@click.option("-j", "--jsonpath", type=str, required=True, help="The output path of jsonfile.") -def getmodels(workpath, jsonpath): - - models = {} - with open("{}/mnt/p3/configs/platforms.yml".format(workpath), "r") as f: - P_data = yaml.safe_load(f) - P_platforms = P_data.get("platforms", []) - for P in P_platforms: - productvers = {} - for V in P_platforms[P]["productvers"]: - if P_platforms[P]["productvers"][V].get("kpre", "") != "": - productvers[V] = (P_platforms[P]["productvers"][V].get("kpre", "") + "-" + P_platforms[P]["productvers"][V].get("kver", "")) - else: - productvers[V] = P_platforms[P]["productvers"][V].get("kver", "") - models[P] = {"productvers": productvers, "models": []} - - req = requests.get("https://autoupdate.synology.com/os/v2") - req.encoding = "utf-8" - data = json.loads(req.text) - - for I in data["channel"]["item"]: - if not I["title"].startswith("DSM"): - continue - for J in I["model"]: - arch = J["mUnique"].split("_")[1].lower() - name = J["mLink"].split("/")[-1].split("_")[1].replace("%2B", "+") - if arch not in models.keys(): - continue - if name in (A for B in models for A in models[B]["models"]): - continue - models[arch]["models"].append(name) - - if jsonpath: - with open(jsonpath, "w") as f: - json.dump(models, f, indent=4, ensure_ascii=False) - -@cli.command() -@click.option("-w", "--workpath", type=str, required=True, help="The workpath of Arc.") -@click.option("-j", "--jsonpath", type=str, required=True, help="The output path of jsonfile.") -def getpats(workpath, jsonpath): - def __fullversion(ver): - out = ver - arr = ver.split('-') - if len(arr) > 0: - a = arr[0].split('.')[0] if len(arr[0].split('.')) > 0 else '0' - b = arr[0].split('.')[1] if len(arr[0].split('.')) > 1 else '0' - c = arr[0].split('.')[2] if len(arr[0].split('.')) > 2 else '0' - d = arr[1] if len(arr) > 1 else '00000' - e = arr[2] if len(arr) > 2 else '0' - out = '{}.{}.{}-{}-{}'.format(a,b,c,d,e) - return out - - platforms = [] - models = [] - with open("{}/mnt/p3/configs/platforms.yml".format(workpath), "r") as f: - data = yaml.safe_load(f) - platforms = data.get("platforms", []) - - req = requests.get("https://autoupdate.synology.com/os/v2") - req.encoding = "utf-8" - data = json.loads(req.text) - - for I in data["channel"]["item"]: - if not I["title"].startswith("DSM"): - continue - for J in I["model"]: - arch = J["mUnique"].split("_")[1].lower() - name = J["mLink"].split("/")[-1].split("_")[1].replace("%2B", "+") - if arch not in platforms: - continue - if name in models: - continue - models.append(name) - - pats = {} - for M in models: - pats[M] = {} - version = '7' - urlInfo = "https://www.synology.com/api/support/findDownloadInfo?lang=en-us" - urlSteps = "https://www.synology.com/api/support/findUpgradeSteps?" - #urlInfo = "https://www.synology.cn/api/support/findDownloadInfo?lang=zh-cn" - #urlSteps = "https://www.synology.cn/api/support/findUpgradeSteps?" - - major = "&major={}".format(version.split('.')[0]) if len(version.split('.')) > 0 else "" - minor = "&minor={}".format(version.split('.')[1]) if len(version.split('.')) > 1 else "" - req = requests.get("{}&product={}{}{}".format(urlInfo, M.replace("+", "%2B"), major, minor)) - req.encoding = "utf-8" - data = json.loads(req.text) - - build_ver = data['info']['system']['detail'][0]['items'][0]['build_ver'] - build_num = data['info']['system']['detail'][0]['items'][0]['build_num'] - buildnano = data['info']['system']['detail'][0]['items'][0]['nano'] - V=__fullversion("{}-{}-{}".format(build_ver, build_num, buildnano)) - if not V in pats[M]: - pats[M][V]={} - pats[M][V]['url'] = data['info']['system']['detail'][0]['items'][0]['files'][0]['url'].split('?')[0] - pats[M][V]['sum'] = data['info']['system']['detail'][0]['items'][0]['files'][0]['checksum'] - - from_ver=0 - for I in data['info']['pubVers']: - if from_ver == 0 or I['build'] < from_ver: from_ver = I['build'] - - for I in data['info']['productVers']: - if not I['version'].startswith(version): continue - if major == "" or minor == "": - majorTmp = "&major={}".format(I['version'].split('.')[0]) if len(I['version'].split('.')) > 0 else "" - minorTmp = "&minor={}".format(I['version'].split('.')[1]) if len(I['version'].split('.')) > 1 else "" - reqTmp = requests.get("{}&product={}{}{}".format(urlInfo, M.replace("+", "%2B"), majorTmp, minorTmp)) - reqTmp.encoding = "utf-8" - dataTmp = json.loads(reqTmp.text) - - build_ver = dataTmp['info']['system']['detail'][0]['items'][0]['build_ver'] - build_num = dataTmp['info']['system']['detail'][0]['items'][0]['build_num'] - buildnano = dataTmp['info']['system']['detail'][0]['items'][0]['nano'] - V=__fullversion("{}-{}-{}".format(build_ver, build_num, buildnano)) - if not V in pats[M]: - pats[M][V]={} - pats[M][V]['url'] = dataTmp['info']['system']['detail'][0]['items'][0]['files'][0]['url'].split('?')[0] - pats[M][V]['sum'] = dataTmp['info']['system']['detail'][0]['items'][0]['files'][0]['checksum'] - - for J in I['versions']: - to_ver=J['build'] - reqSteps = requests.get("{}&product={}&from_ver={}&to_ver={}".format(urlSteps, M.replace("+", "%2B"), from_ver, to_ver)) - if reqSteps.status_code != 200: continue - reqSteps.encoding = "utf-8" - dataSteps = json.loads(reqSteps.text) - for S in dataSteps['upgrade_steps']: - if not 'full_patch' in S or S['full_patch'] is False: continue - if not 'build_ver' in S or not S['build_ver'].startswith(version): continue - V=__fullversion("{}-{}-{}".format(S['build_ver'], S['build_num'], S['nano'])) - if not V in pats[M]: - pats[M][V] = {} - pats[M][V]['url'] = S['files'][0]['url'].split('?')[0] - pats[M][V]['sum'] = S['files'][0]['checksum'] - - if jsonpath: - with open(jsonpath, "w") as f: - json.dump(pats, f, indent=4, ensure_ascii=False) - -@cli.command() -@click.option("-w", "--workpath", type=str, required=True, help="The workpath of Arc.") -@click.option("-j", "--jsonpath", type=str, required=True, help="The output path of jsonfile.") -def getaddons(workpath, jsonpath): - # Read the manifest.yml file - AS = glob.glob("{}/mnt/p3/addons/*/manifest.yml".format(workpath)) - AS.sort() - addons = {} - for A in AS: - with open(A, "r") as file: - A_data = yaml.safe_load(file) - A_name = A_data.get("name", "") - A_system = A_data.get("system", False) - A_description = A_data.get("description", "") - A_apollolake = A_data.get("apollolake", "") - A_broadwell = A_data.get("broadwell", "") - A_broadwellnk = A_data.get("broadwellnk", "") - A_broadwellnkv2 = A_data.get("broadwellnkv2", "") - A_broadwellntbap = A_data.get("broadwellntbap", "") - A_denverton = A_data.get("denverton", "") - A_geminilake = A_data.get("geminilake", "") - A_purley = A_data.get("purley", "") - A_v1000 = A_data.get("v1000", "") - A_r1000 = A_data.get("r1000", "") - A_epyc7002 = A_data.get("epyc7002", "") - addons[A_name] = {"system": A_system, "description": A_description, "apollolake": A_apollolake, "broadwell": A_broadwell, "broadwellnk": A_broadwellnk, "broadwellnkv2": A_broadwellnkv2, "broadwellntbap": A_broadwellntbap, "denverton": A_denverton, "geminilake": A_geminilake, "purley": A_purley, "v1000": A_v1000, "r1000": A_r1000, "epyc7002": A_epyc7002} - if jsonpath: - with open(jsonpath, "w") as f: - json.dump(addons, f, indent=4, ensure_ascii=False) - - -@cli.command() -@click.option("-w", "--workpath", type=str, required=True, help="The workpath of Arc.") -@click.option("-j", "--jsonpath", type=str, required=True, help="The output path of jsonfile.") -def getmodules(workpath, jsonpath): - # Read the module files - MS = glob.glob("{}/mnt/p3/modules/*.tgz".format(workpath)) - MS.sort() - modules = {} - TMP_PATH = "/tmp/modules" - if os.path.exists(TMP_PATH): - shutil.rmtree(TMP_PATH) - for M in MS: - M_name = os.path.splitext(os.path.basename(M))[0] - M_modules = {} - # Extract the module - os.makedirs(TMP_PATH) - with tarfile.open(M, "r") as tar: - tar.extractall(TMP_PATH) - # Traverse the extracted files - KS = glob.glob("{}/*.ko".format(TMP_PATH)) - KS.sort() - for K in KS: - K_name = os.path.splitext(os.path.basename(K))[0] - K_info = kmodule.modinfo(K, basedir=os.path.dirname(K), kernel=None)[0] - K_description = K_info.get("description", "") - K_depends = K_info.get("depends", "") - M_modules[K_name] = {"description": K_description, "depends": K_depends} - modules[M_name] = M_modules - if os.path.exists(TMP_PATH): - shutil.rmtree(TMP_PATH) - if jsonpath: - with open(jsonpath, "w") as file: - json.dump(modules, file, indent=4, ensure_ascii=False) - - -if __name__ == "__main__": - cli() diff --git a/scripts/functions.py b/scripts/functions.py new file mode 100755 index 00000000..08074679 --- /dev/null +++ b/scripts/functions.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2025 AuxXxilium +# +# This is free software, licensed under the MIT License. +# See /LICENSE for more information. +# + +import os, click + +WORK_PATH = os.path.abspath(os.path.dirname(__file__)) + + +@click.group() +def cli(): + """ + The CLI is a commands to Arc. + """ + pass + + +def mutually_exclusive_options(ctx, param, value): + other_option = "file" if param.name == "data" else "data" + if value is not None and ctx.params.get(other_option) is not None: + raise click.UsageError(f"Illegal usage: `{param.name}` is mutually exclusive with `{other_option}`.") + return value + + +def validate_required_param(ctx, param, value): + if not value and "file" not in ctx.params and "data" not in ctx.params: + raise click.MissingParameter(param_decls=[param.name]) + return value + +def __fullversion(ver): + out = ver + arr = ver.split('-') + if len(arr) > 0: + a = arr[0].split('.')[0] if len(arr[0].split('.')) > 0 else '0' + b = arr[0].split('.')[1] if len(arr[0].split('.')) > 1 else '0' + c = arr[0].split('.')[2] if len(arr[0].split('.')) > 2 else '0' + d = arr[1] if len(arr) > 1 else '00000' + e = arr[2] if len(arr) > 2 else '0' + out = '{}.{}.{}-{}-{}'.format(a,b,c,d,e) + return out + +@cli.command() +@click.option("-p", "--platforms", type=str, help="The platforms of Syno.") +def getmodels(platforms=None): + """ + Get Syno Models. + """ + import re, json, requests, urllib3 + from requests.adapters import HTTPAdapter + from requests.packages.urllib3.util.retry import Retry # type: ignore + + adapter = HTTPAdapter(max_retries=Retry(total=3, backoff_factor=1, status_forcelist=[500, 502, 503, 504])) + session = requests.Session() + session.mount("http://", adapter) + session.mount("https://", adapter) + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + PS = platforms.lower().replace(",", " ").split() if platforms else [] + + models = [] + try: + url = "http://update7.synology.com/autoupdate/genRSS.php?include_beta=1" + #url = "https://update7.synology.com/autoupdate/genRSS.php?include_beta=1" + + req = session.get(url, timeout=10, verify=False) + req.encoding = "utf-8" + p = re.compile(r"(.*?).*?(.*?)", re.MULTILINE | re.DOTALL) + + data = p.findall(req.text) + for item in data: + if not "DSM" in item[1]: + continue + arch = item[0].split("_")[1] + name = item[1].split("/")[-1].split("_")[1].replace("%2B", "+") + if PS and arch.lower() not in PS: + continue + if not any(m["name"] == name for m in models): + models.append({"name": name, "arch": arch}) + + models.sort(key=lambda k: (k["arch"], k["name"])) + + except Exception as e: + # click.echo(f"Error: {e}") + pass + + print(json.dumps(models, indent=4)) + +@cli.command() +@click.option("-m", "--model", type=str, required=True, help="The model of Syno.") +@click.option("-v", "--version", type=str, required=True, help="The version of Syno.") +def getpats4mv(model, version): + import json, requests, urllib3 + from requests.adapters import HTTPAdapter + from requests.packages.urllib3.util.retry import Retry # type: ignore + + adapter = HTTPAdapter(max_retries=Retry(total=3, backoff_factor=1, status_forcelist=[500, 502, 503, 504])) + session = requests.Session() + session.mount("http://", adapter) + session.mount("https://", adapter) + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + pats = {} + try: + urlInfo = "https://www.synology.com/api/support/findDownloadInfo?lang=en-us" + urlSteps = "https://www.synology.com/api/support/findUpgradeSteps?" + #urlInfo = "https://www.synology.cn/api/support/findDownloadInfo?lang=zh-cn" + #urlSteps = "https://www.synology.cn/api/support/findUpgradeSteps?" + + major = f"&major={version.split('.')[0]}" if len(version.split('.')) > 0 else "" + minor = f"&minor={version.split('.')[1]}" if len(version.split('.')) > 1 else "" + req = session.get(f"{urlInfo}&product={model.replace('+', '%2B')}{major}{minor}", timeout=10, verify=False) + req.encoding = "utf-8" + data = json.loads(req.text) + + build_ver = data['info']['system']['detail'][0]['items'][0]['build_ver'] + build_num = data['info']['system']['detail'][0]['items'][0]['build_num'] + buildnano = data['info']['system']['detail'][0]['items'][0]['nano'] + V = __fullversion(f"{build_ver}-{build_num}-{buildnano}") + if V not in pats: + pats[V] = { + 'url': data['info']['system']['detail'][0]['items'][0]['files'][0]['url'].split('?')[0], + 'sum': data['info']['system']['detail'][0]['items'][0]['files'][0]['checksum'] + } + + from_ver = min(I['build'] for I in data['info']['pubVers']) + + for I in data['info']['productVers']: + if not I['version'].startswith(version): + continue + if not major or not minor: + majorTmp = f"&major={I['version'].split('.')[0]}" if len(I['version'].split('.')) > 0 else "" + minorTmp = f"&minor={I['version'].split('.')[1]}" if len(I['version'].split('.')) > 1 else "" + reqTmp = session.get(f"{urlInfo}&product={model.replace('+', '%2B')}{majorTmp}{minorTmp}", timeout=10, verify=False) + reqTmp.encoding = "utf-8" + dataTmp = json.loads(reqTmp.text) + + build_ver = dataTmp['info']['system']['detail'][0]['items'][0]['build_ver'] + build_num = dataTmp['info']['system']['detail'][0]['items'][0]['build_num'] + buildnano = dataTmp['info']['system']['detail'][0]['items'][0]['nano'] + V = __fullversion(f"{build_ver}-{build_num}-{buildnano}") + if V not in pats: + pats[V] = { + 'url': dataTmp['info']['system']['detail'][0]['items'][0]['files'][0]['url'].split('?')[0], + 'sum': dataTmp['info']['system']['detail'][0]['items'][0]['files'][0]['checksum'] + } + + for J in I['versions']: + to_ver = J['build'] + reqSteps = session.get(f"{urlSteps}&product={model.replace('+', '%2B')}&from_ver={from_ver}&to_ver={to_ver}", timeout=10, verify=False) + if reqSteps.status_code != 200: + continue + reqSteps.encoding = "utf-8" + dataSteps = json.loads(reqSteps.text) + for S in dataSteps['upgrade_steps']: + if not S.get('full_patch') or not S['build_ver'].startswith(version): + continue + V = __fullversion(f"{S['build_ver']}-{S['build_num']}-{S['nano']}") + if V not in pats: + pats[V] = { + 'url': S['files'][0]['url'].split('?')[0], + 'sum': S['files'][0]['checksum'] + } + except Exception as e: + # click.echo(f"Error: {e}") + pass + + pats = {k: pats[k] for k in sorted(pats.keys(), reverse=True)} + print(json.dumps(pats, indent=4)) + + +@cli.command() +@click.option("-p", "--models", type=str, help="The models of Syno.") +def getpats(models=None): + import re, json, requests, urllib3 + from bs4 import BeautifulSoup + from requests.adapters import HTTPAdapter + from requests.packages.urllib3.util.retry import Retry # type: ignore + + adapter = HTTPAdapter(max_retries=Retry(total=3, backoff_factor=1, status_forcelist=[500, 502, 503, 504])) + session = requests.Session() + session.mount("http://", adapter) + session.mount("https://", adapter) + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + MS = models.lower().replace(",", " ").split() if models else [] + + pats = {} + try: + req = session.get('https://archive.synology.com/download/Os/DSM', timeout=10, verify=False) + req.encoding = 'utf-8' + bs = BeautifulSoup(req.text, 'html.parser') + p = re.compile(r"(.*?)-(.*?)", re.MULTILINE | re.DOTALL) + l = bs.find_all('a', string=p) + for i in l: + ver = i.attrs['href'].split('/')[-1] + if not ver.startswith('7'): + continue + req = session.get(f'https://archive.synology.com{i.attrs["href"]}', timeout=10, verify=False) + req.encoding = 'utf-8' + bs = BeautifulSoup(req.text, 'html.parser') + p = re.compile(r"DSM_(.*?)_(.*?).pat", re.MULTILINE | re.DOTALL) + data = bs.find_all('a', string=p) + for item in data: + rels = p.search(item.attrs['href']) + if rels: + model, _ = rels.groups() + model = model.replace('%2B', '+') + if MS and model.lower() not in MS: + continue + if model not in pats: + pats[model] = {} + pats[model][__fullversion(ver)] = item.attrs['href'] + except Exception as e: + # click.echo(f"Error: {e}") + pass + + print(json.dumps(pats, indent=4)) + +@cli.command() +@click.option("-p", "--platforms", type=str, help="The platforms of Syno.") +def getmodelsoffline(platforms=None): + """ + Get Syno Models. + """ + import json + import os + + PS = platforms.lower().replace(",", " ").split() if platforms else [] + + with open(os.path.join('/mnt/p3/configs', "offline.json")) as user_file: + data = json.load(user_file) + + models = [] + for item in data["channel"]["item"]: + if not item["title"].startswith("DSM"): + continue + for model in item["model"]: + arch = model["mUnique"].split("_")[1] + name = model["mLink"].split("/")[-1].split("_")[1].replace("%2B", "+") + if PS and arch.lower() not in PS: + continue + if not any(m["name"] == name for m in models): + models.append({"name": name, "arch": arch}) + + models = sorted(models, key=lambda k: (k["arch"], k["name"])) + print(json.dumps(models, indent=4)) + +if __name__ == "__main__": + cli() \ No newline at end of file