diff --git a/linux/.gitignore b/linux/.gitignore index 675737a636..92f1ab9954 100644 --- a/linux/.gitignore +++ b/linux/.gitignore @@ -28,7 +28,7 @@ __pycache__ build-* deb-* builddebs -make_deb +keyman-config/make_deb tmp/ dist/ *~ @@ -54,5 +54,7 @@ ibus-kmfl/m4/visibility.m4 ibus-kmfl/m4/wchar_t.m4 ibus-kmfl/m4/wint_t.m4 ibus-kmfl/m4/xsize.m4 -keyman-config/debian/man keyman-config/build +keyman-config/debian/man +keyman-config/experiments/*.txt +test.sh diff --git a/linux/keyman-config/experiments/check_kmp.py b/linux/keyman-config/experiments/check_kmp.py new file mode 100755 index 0000000000..767b462d4f --- /dev/null +++ b/linux/keyman-config/experiments/check_kmp.py @@ -0,0 +1,45 @@ +#!/usr/bin/python3 + +import datetime +import logging +import os +import requests +import requests_cache +import subprocess +import tempfile +import time +from dirlist import list_keyboards +from keyman_config.install_kmp import get_metadata, extract_kmp +from keyman_config.kmpmetadata import KMFileTypes +from keyman_config.get_kmp import keyman_cache_dir + +#TODO check if any files in files list in cached kmps are KM_UNKNOWN + +def main(): + logging.basicConfig(level=logging.INFO, format='%(levelname)s:%(message)s') + km_cache=keyman_cache_dir() + logging.info("looking for KM_UNKNOWN files in cached kmps") + keyboarddata = list_keyboards() + if keyboarddata: + with open('./unknownfiles.txt', 'wt') as unknownfiles: + print("files found in cached kmps of type KM_UNKNOWN", file=unknownfiles) + for kbid in keyboarddata: + kmpfile = os.path.join(km_cache, kbid+".kmp") + if os.path.exists(kmpfile): + with tempfile.TemporaryDirectory() as tmpdirname: + extract_kmp(kmpfile, tmpdirname) + try: + info, system, options, keyboards, files = get_metadata(tmpdirname) + if files: + for kbfile in files: + if kbfile['type'] == KMFileTypes.KM_UNKNOWN: + print(kbfile['name'], file=unknownfiles) + except Exception as e: + print(type(e)) # the exception instance + print(e.args) # arguments stored in .args + print(e) # __str__ allows args to be printed directly, pass + pass + + +if __name__ == "__main__": + main() diff --git a/linux/keyman-config/experiments/dirlist.py b/linux/keyman-config/experiments/dirlist.py new file mode 100755 index 0000000000..ebdc5186af --- /dev/null +++ b/linux/keyman-config/experiments/dirlist.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 + +from bs4 import BeautifulSoup +import datetime +import logging +import requests +import requests_cache +import os +import time +from keyman_config.get_kmp import keyman_cache_dir + +def get_keyboard_dir_page(kb_url): + logging.info("Getting keyboard list") + logging.debug("At URL %s", kb_url) + cache_dir = keyman_cache_dir() + current_dir = os.getcwd() + expire_after = datetime.timedelta(days=7) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + os.chdir(cache_dir) + requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after) + now = time.ctime(int(time.time())) + response = requests.get(kb_url) + logging.debug("Time: {0} / Used Cache: {1}".format(now, response.from_cache)) + os.chdir(current_dir) + requests_cache.core.uninstall_cache() + if response.status_code == 200: + return response.text + else: + return None + + +def get_dir_list(): + url = "https://downloads.keyman.com/keyboards/" + page = get_keyboard_dir_page(url) +# print(page) + soup = BeautifulSoup(page, 'html.parser') +# return [url + '/' + node.get('href') for node in soup.find_all('a') if node.get('href').endswith(ext)] + return [url + node.get('href') for node in soup.find_all('a')] + +def list_keyboards(): + kblist = [] + for file in get_dir_list(): + logging.debug(file) + kb = os.path.basename(os.path.dirname(file)) + if kb != "keyboards": + kblist.append(kb) + return kblist diff --git a/linux/keyman-config/experiments/findkmninkmp.sh b/linux/keyman-config/experiments/findkmninkmp.sh new file mode 100755 index 0000000000..73df3af809 --- /dev/null +++ b/linux/keyman-config/experiments/findkmninkmp.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +echo "cached kmp with kmn files inside them:" +find ~/.cache/keyman/ -print0| while read -d $'\0' file +do + unzip -q -t "$file" *.kmn 2>&1 + if [ "$?" == "0" ]; then + basename "$file" .kmp +# else +# echo "no kmn in $file" + fi +done diff --git a/linux/keyman-config/experiments/keymankeyboards.py b/linux/keyman-config/experiments/keymankeyboards.py index 774486549e..a07a2cb1bf 100755 --- a/linux/keyman-config/experiments/keymankeyboards.py +++ b/linux/keyman-config/experiments/keymankeyboards.py @@ -22,6 +22,7 @@ import requests import requests_cache import os from pathlib import Path +from keyman_config.get_kmp import keyman_cache_dir def get_api_keyboards(verbose=False): """ @@ -33,11 +34,11 @@ def get_api_keyboards(verbose=False): dict: Keyboard data None: if http request not successful """ - api_url = "https://api.keyman.com/cloud/4.0/keyboards" + api_url = "https://api.keyman.com/cloud/4.0/keyboards?version=10.0" headers = {'Content-Type': 'application/json', 'Accept-Encoding': 'gzip, deflate, br'} home = str(Path.home()) - cache_dir = os.path.join(home, ".local/share/keyman") + cache_dir = keyman_cache_dir() current_dir = os.getcwd() expire_after = datetime.timedelta(days=1) if not os.path.isdir(cache_dir): diff --git a/linux/keyman-config/experiments/list_kmp.py b/linux/keyman-config/experiments/list_kmp.py index c6eb7cf0fe..d67f635dab 100755 --- a/linux/keyman-config/experiments/list_kmp.py +++ b/linux/keyman-config/experiments/list_kmp.py @@ -1,42 +1,117 @@ #!/usr/bin/python3 +import datetime +import logging +import os +import requests +import requests_cache +import subprocess import tempfile -from keymankeyboards import get_api_keyboards -from get_kmp import get_keyboard_data, get_kmp_file -from install_kmp import get_metadata, get_infdata, extract_kmp +import time +from dirlist import list_keyboards +#from keymankeyboards import get_api_keyboards + +#from keyman_config import get_kmp, install_kmp +from keyman_config.get_kmp import get_keyboard_data, get_kmp_file, keyman_cache_dir +from keyman_config.install_kmp import get_metadata, get_infdata, extract_kmp + +#TODO check for kmn and check if it is compilable +#TODO extra output files jsonkmpnokmn, jsonkmpbadkmn, goodjsonkmpkmn and for inf as well + +def get_kmn(kbid, sourcePath): + base_url = "https://raw.github.com/keymanapp/keyboards/master/" + sourcePath + kmn_url = base_url + "/source/" + kbid + ".kmn" + + cache_dir = keyman_cache_dir() + current_dir = os.getcwd() + expire_after = datetime.timedelta(days=7) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + os.chdir(cache_dir) + requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after) + now = time.ctime(int(time.time())) + response = requests.get(kmn_url) + logging.debug("Time: {0} / Used Cache: {1}".format(now, response.from_cache)) + os.chdir(current_dir) + requests_cache.core.uninstall_cache() + + return requests.get(kmn_url) def main(): - keyboarddata = get_api_keyboards() + logging.basicConfig(level=logging.DEBUG, format='%(levelname)s:%(message)s') + keyboarddata = list_keyboards() if keyboarddata: with open('./nokmp.txt', 'wt') as nokmp, \ open('./infnokeyboard.txt', 'wt') as infnokeyboard, \ + open('./goodjsonkmpkmn.txt', 'wt') as goodjsonkmpkmn, \ + open('./jsonkmpnokmn.txt', 'wt') as jsonkmpnokmn, \ + open('./jsonkmpbadkmn.txt', 'wt') as jsonkmpbadkmn, \ + open('./jsonkmpmissingkmn.txt', 'wt') as jsonkmpmissingkmn, \ open('./brokeninf.txt', 'wt') as brokeninf, \ open('./nodata.txt', 'wt') as nodata, \ - open('./goodjsonkmp.txt', 'wt') as goodjsonkmp, \ open('./goodinfkmp.txt', 'wt') as goodinfkmp: - for kb in keyboarddata['keyboard']: - kbdata = get_keyboard_data(kb['id']) - print(kb['id']) + + print("Keyboard: will work in kmfl :)", file=goodjsonkmpkmn) # goodjsonkmpkmn + print("Keyboard: has uncompilable kmn", file=jsonkmpbadkmn) # jsonkmpbadkmn + print("Keyboard: has json in kmp but can't find the kmn on github", file=jsonkmpmissingkmn) # jsonkmpmissingkmn + print("Keyboard: has json in kmp but has no sourcePath to look for kmn on github", file=jsonkmpnokmn) # jsonkmpnokmn + print("Keyboard: has kmp with kmp.inf", file=goodinfkmp) + print("Keyboard: has kmp with kmp.inf but it has no Keyboard", file=infnokeyboard) + print("Keyboard: has kmp but no kmp.json and no or broken kmp.inf", file=brokeninf) + print("Keyboard: does not have kmp so mobile/web only", file=nokmp) + print("Keyboard: has no data", file=nodata) + + + for kbid in keyboarddata: + kbdata = get_keyboard_data(kbid, True) + print(kbid) if kbdata: if 'packageFilename' in kbdata: - kmpfile = get_kmp_file(kbdata) + kmpfile = get_kmp_file(kbdata, True) with tempfile.TemporaryDirectory() as tmpdirname: extract_kmp(kmpfile, tmpdirname) - info, system, options, keyboards, files = get_metadata(tmpdirname) - if keyboards: - print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "with kmp.json", file=goodjsonkmp) - else: - info, system, options, keyboards, files = get_infdata(tmpdirname) + try: + info, system, options, keyboards, files = get_metadata(tmpdirname) if keyboards: - print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "with kmp.inf", file=goodinfkmp) - elif files: - print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "with kmp.inf but it has no Keyboard", file=infnokeyboard) + if 'sourcePath' in kbdata: + response = get_kmn(kbid, kbdata['sourcePath']) + if response.status_code == 200: + kmndownloadfile = os.path.join(tmpdirname, kbid + ".kmn") + with open(kmndownloadfile, 'wb') as f: + f.write(response.content) + subprocess.run(["kmflcomp", kmndownloadfile], stdout=subprocess.PIPE, stderr= subprocess.STDOUT) + kmfl_file = os.path.join(tmpdirname, kbid + ".kmfl") + if os.path.isfile(kmfl_file): + logging.debug("goodjsonkmpkmn") + print(kbid, file=goodjsonkmpkmn) # goodjsonkmpkmn + else: + logging.debug("jsonkmpbadkmn") + print(kbid, file=jsonkmpbadkmn) # jsonkmpbadkmn + else: + logging.debug("jsonkmpmissingkmn") + print(kbid, file=jsonkmpmissingkmn) # jsonkmpmissingkmn + else: + logging.debug("jsonkmpnokmn") + print(kbid, file=jsonkmpnokmn) # jsonkmpnokmn else: - print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "but no kmp.json and no or broken kmp.inf", file=brokeninf) + info, system, options, keyboards, files = get_infdata(tmpdirname) + if keyboards: + logging.debug("infnokeyboard") + print(kbid, file=goodinfkmp) + elif files: + logging.debug("goodinfkmp") + print(kbid, file=infnokeyboard) + else: + print(kbid, file=brokeninf) + except KeyError: + logging.debug("brokeninf") + print(kbid, file=brokeninf) else: - print("Keyboard:", kb['id'], "does not have kmp", file=nokmp) + logging.debug("nokmp") + print(kbid, file=nokmp) else: - print("Keyboard:", kb['id'], "has no data", file=nodata) + logging.debug("nodata") + print(kbid, file=nodata) if __name__ == "__main__": diff --git a/linux/keyman-config/keyman_config/get_kmp.py b/linux/keyman-config/keyman_config/get_kmp.py index 5442064c1b..3d831a3a02 100755 --- a/linux/keyman-config/keyman_config/get_kmp.py +++ b/linux/keyman-config/keyman_config/get_kmp.py @@ -11,12 +11,13 @@ import os from pathlib import Path -def get_keyboard_data(keyboardid): +def get_keyboard_data(keyboardid, weekCache=False): """ Get Keyboard data from web api. Args: keyboardid (str): Keyboard ID + weekCache (bool) : cache data for 1 week, default is 1 day Returns: dict: Keyboard data """ @@ -26,7 +27,10 @@ def get_keyboard_data(keyboardid): home = str(Path.home()) cache_dir = keyman_cache_dir() current_dir = os.getcwd() - expire_after = datetime.timedelta(days=1) + if weekCache: + expire_after = datetime.timedelta(days=7) + else: + expire_after = datetime.timedelta(days=1) os.chdir(cache_dir) requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after) now = time.ctime(int(time.time())) @@ -78,12 +82,13 @@ def user_keyboard_dir(keyboardid): return os.path.join(user_keyman_dir(), keyboardid) -def get_kmp_file(kbdata): +def get_kmp_file(kbdata, cache=False): """ Get info from keyboard data to download kmp then download it. Args: kbdata (dict): Keyboard data + cache (bool): Whether to cache the kmp file web request Returns: str: path where kmp file has been downloaded """ @@ -93,9 +98,9 @@ def get_kmp_file(kbdata): kmp_url = "https://downloads.keyman.com/keyboards/" + kbdata['id'] + "/" + kbdata['version'] + "/" + kbdata['packageFilename'] downloadfile = os.path.join(get_download_folder(), kbdata['packageFilename']) - return download_kmp_file(kmp_url, downloadfile) + return download_kmp_file(kmp_url, downloadfile, cache) -def download_kmp_file(url, kmpfile): +def download_kmp_file(url, kmpfile, cache=False): """ Download kmp file. @@ -104,12 +109,30 @@ def download_kmp_file(url, kmpfile): kmpfile (str): Where to save the kmp file. currently it does no checks on this location assumes that is in users keyman cache dir + cache(bool): Whether to cache the kmp file web request for a week Returns: str: path where kmp file has been downloaded """ logging.info("Download URL: %s", url) downloadfile = None + + if cache: + cache_dir = keyman_cache_dir() + current_dir = os.getcwd() + expire_after = datetime.timedelta(days=7) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + os.chdir(cache_dir) + requests_cache.install_cache(cache_name='keyman_kmp_cache', backend='sqlite', expire_after=expire_after) + now = time.ctime(int(time.time())) + response = requests.get(url) #, stream=True) + + if cache: + logging.debug("Time: {0} / Used Cache: {1}".format(now, response.from_cache)) + os.chdir(current_dir) + requests_cache.core.uninstall_cache() + if response.status_code == 200: with open(kmpfile, 'wb') as f: f.write(response.content)