From 750ba31f918e049400221aa8271bc50f53b7be8c Mon Sep 17 00:00:00 2001 From: glasseyes Date: Thu, 20 Sep 2018 15:36:45 +0700 Subject: [PATCH 1/7] [linux] list_kmp: get list known keyboards from downloads.km.com scrapes keyboards index page to get ids of listed keyboards findkmninkmp to find kmp that have kmn in them in cache dir --- linux/keyman-config/experiments/dirlist.py | 49 ++++++++ .../keyman-config/experiments/findkmninkmp.sh | 22 ++++ .../experiments/keymankeyboards.py | 2 +- linux/keyman-config/experiments/list_kmp.py | 117 +++++++++++++++--- linux/keyman-config/keyman_config/get_kmp.py | 24 +++- 5 files changed, 190 insertions(+), 24 deletions(-) create mode 100755 linux/keyman-config/experiments/dirlist.py create mode 100755 linux/keyman-config/experiments/findkmninkmp.sh diff --git a/linux/keyman-config/experiments/dirlist.py b/linux/keyman-config/experiments/dirlist.py new file mode 100755 index 0000000000..847ed904c6 --- /dev/null +++ b/linux/keyman-config/experiments/dirlist.py @@ -0,0 +1,49 @@ +#!/usr/bin/python3 + +from bs4 import BeautifulSoup +import datetime +import logging +import requests +import requests_cache +import os +import time + +def get_keyboard_dir_page(kb_url): + logging.info("Getting keyboard list") + logging.debug("At URL %s", kb_url) + home = os.path.expanduser("~") + datahome = os.environ.get("XDG_DATA_HOME", os.path.join(home, ".local", "share")) + cache_dir = os.path.join(datahome, "keyman") + current_dir = os.getcwd() + expire_after = datetime.timedelta(days=1) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + os.chdir(cache_dir) + requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after) + now = time.ctime(int(time.time())) + response = requests.get(kb_url) + logging.debug("Time: {0} / Used Cache: {1}".format(now, response.from_cache)) + os.chdir(current_dir) + requests_cache.core.uninstall_cache() + if response.status_code == 200: + return response.text + else: + return None + + +def get_dir_list(): + url = "https://downloads.keyman.com/keyboards/" + page = get_keyboard_dir_page(url) +# print(page) + soup = BeautifulSoup(page, 'html.parser') +# return [url + '/' + node.get('href') for node in soup.find_all('a') if node.get('href').endswith(ext)] + return [url + node.get('href') for node in soup.find_all('a')] + +def list_keyboards(): + kblist = [] + for file in get_dir_list(): + print(file) + kb = os.path.basename(os.path.dirname(file)) + if kb != "keyboards": + kblist.append(kb) + return kblist diff --git a/linux/keyman-config/experiments/findkmninkmp.sh b/linux/keyman-config/experiments/findkmninkmp.sh new file mode 100755 index 0000000000..ecdb1d473a --- /dev/null +++ b/linux/keyman-config/experiments/findkmninkmp.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +echo "cached kmp with kmn files inside them:" +find ~/.cache/keyman/ -print0| while read -d $'\0' file +do + unzip -q -t "$file" *.kmn 2>&1 + if [ "$?" == "0" ]; then + basename "$file" .kmp +# else +# echo "no kmn in $file" + fi +done + +#for file in `find ~/.cache/keyman -name "*.kmp"` +#do +# unzip -q -t "$file" *.kmn 2>&1 +# if [ "$?" == "0" ]; then +# echo "$file" +## else +## echo "no kmn in $file" +# fi +#done diff --git a/linux/keyman-config/experiments/keymankeyboards.py b/linux/keyman-config/experiments/keymankeyboards.py index 774486549e..d2163fdfb5 100755 --- a/linux/keyman-config/experiments/keymankeyboards.py +++ b/linux/keyman-config/experiments/keymankeyboards.py @@ -33,7 +33,7 @@ def get_api_keyboards(verbose=False): dict: Keyboard data None: if http request not successful """ - api_url = "https://api.keyman.com/cloud/4.0/keyboards" + api_url = "https://api.keyman.com/cloud/4.0/keyboards?version=10.0" headers = {'Content-Type': 'application/json', 'Accept-Encoding': 'gzip, deflate, br'} home = str(Path.home()) diff --git a/linux/keyman-config/experiments/list_kmp.py b/linux/keyman-config/experiments/list_kmp.py index c6eb7cf0fe..38024b82d1 100755 --- a/linux/keyman-config/experiments/list_kmp.py +++ b/linux/keyman-config/experiments/list_kmp.py @@ -1,42 +1,119 @@ #!/usr/bin/python3 +import datetime +import logging +import os +import requests +import requests_cache +import subprocess import tempfile -from keymankeyboards import get_api_keyboards -from get_kmp import get_keyboard_data, get_kmp_file -from install_kmp import get_metadata, get_infdata, extract_kmp +import time +from dirlist import list_keyboards +#from keymankeyboards import get_api_keyboards + +#from keyman_config import get_kmp, install_kmp +from keyman_config.get_kmp import get_keyboard_data, get_kmp_file +from keyman_config.install_kmp import get_metadata, get_infdata, extract_kmp + +#TODO check for kmn and check if it is compilable +#TODO extra output files jsonkmpnokmn, jsonkmpbadkmn, goodjsonkmpkmn and for inf as well + +def get_kmn(kbid, sourcePath): + base_url = "https://raw.github.com/keymanapp/keyboards/master/" + sourcePath + kmn_url = base_url + "/source/" + kbid + ".kmn" + + home = os.path.expanduser("~") + datahome = os.environ.get("XDG_DATA_HOME", os.path.join(home, ".local", "share")) + cache_dir = os.path.join(datahome, "keyman") + current_dir = os.getcwd() + expire_after = datetime.timedelta(days=1) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + os.chdir(cache_dir) + requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after) + now = time.ctime(int(time.time())) + response = requests.get(kmn_url) + logging.debug("Time: {0} / Used Cache: {1}".format(now, response.from_cache)) + os.chdir(current_dir) + requests_cache.core.uninstall_cache() + + return requests.get(kmn_url) def main(): - keyboarddata = get_api_keyboards() + logging.basicConfig(level=logging.DEBUG, format='%(levelname)s:%(message)s') + keyboarddata = list_keyboards() if keyboarddata: with open('./nokmp.txt', 'wt') as nokmp, \ open('./infnokeyboard.txt', 'wt') as infnokeyboard, \ + open('./goodjsonkmpkmn.txt', 'wt') as goodjsonkmpkmn, \ + open('./jsonkmpnokmn.txt', 'wt') as jsonkmpnokmn, \ + open('./jsonkmpbadkmn.txt', 'wt') as jsonkmpbadkmn, \ + open('./jsonkmpmissingkmn.txt', 'wt') as jsonkmpmissingkmn, \ open('./brokeninf.txt', 'wt') as brokeninf, \ open('./nodata.txt', 'wt') as nodata, \ - open('./goodjsonkmp.txt', 'wt') as goodjsonkmp, \ open('./goodinfkmp.txt', 'wt') as goodinfkmp: - for kb in keyboarddata['keyboard']: - kbdata = get_keyboard_data(kb['id']) - print(kb['id']) + + print("Keyboard: will work in kmfl :)", file=goodjsonkmpkmn) # goodjsonkmpkmn + print("Keyboard: has uncompilable kmn", file=jsonkmpbadkmn) # jsonkmpbadkmn + print("Keyboard: has json in kmp but can't find the kmn on github", file=jsonkmpmissingkmn) # jsonkmpmissingkmn + print("Keyboard: has json in kmp but has no sourcePath to look for kmn on github", file=jsonkmpnokmn) # jsonkmpnokmn + print("Keyboard: has kmp with kmp.inf", file=goodinfkmp) + print("Keyboard: has kmp with kmp.inf but it has no Keyboard", file=infnokeyboard) + print("Keyboard: has kmp but no kmp.json and no or broken kmp.inf", file=brokeninf) + print("Keyboard: does not have kmp so mobile/web only", file=nokmp) + print("Keyboard: has no data", file=nodata) + + + for kbid in keyboarddata: + kbdata = get_keyboard_data(kbid) + print(kbid) if kbdata: if 'packageFilename' in kbdata: - kmpfile = get_kmp_file(kbdata) + kmpfile = get_kmp_file(kbdata, True) with tempfile.TemporaryDirectory() as tmpdirname: extract_kmp(kmpfile, tmpdirname) - info, system, options, keyboards, files = get_metadata(tmpdirname) - if keyboards: - print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "with kmp.json", file=goodjsonkmp) - else: - info, system, options, keyboards, files = get_infdata(tmpdirname) + try: + info, system, options, keyboards, files = get_metadata(tmpdirname) if keyboards: - print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "with kmp.inf", file=goodinfkmp) - elif files: - print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "with kmp.inf but it has no Keyboard", file=infnokeyboard) + if 'sourcePath' in kbdata: + response = get_kmn(kbid, kbdata['sourcePath']) + if response.status_code == 200: + kmndownloadfile = os.path.join(tmpdirname, kbid + ".kmn") + with open(kmndownloadfile, 'wb') as f: + f.write(response.content) + subprocess.run(["kmflcomp", kmndownloadfile], stdout=subprocess.PIPE, stderr= subprocess.STDOUT) + kmfl_file = os.path.join(tmpdirname, kbid + ".kmfl") + if os.path.isfile(kmfl_file): + logging.debug("goodjsonkmpkmn") + print(kbid, file=goodjsonkmpkmn) # goodjsonkmpkmn + else: + logging.debug("jsonkmpbadkmn") + print(kbid, file=jsonkmpbadkmn) # jsonkmpbadkmn + else: + logging.debug("jsonkmpmissingkmn") + print(kbid, file=jsonkmpmissingkmn) # jsonkmpmissingkmn + else: + logging.debug("jsonkmpnokmn") + print(kbid, file=jsonkmpnokmn) # jsonkmpnokmn else: - print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "but no kmp.json and no or broken kmp.inf", file=brokeninf) + info, system, options, keyboards, files = get_infdata(tmpdirname) + if keyboards: + logging.debug("infnokeyboard") + print(kbid, file=goodinfkmp) + elif files: + logging.debug("goodinfkmp") + print(kbid, file=infnokeyboard) + else: + print(kbid, file=brokeninf) + except KeyError: + logging.debug("brokeninf") + print(kbid, file=brokeninf) else: - print("Keyboard:", kb['id'], "does not have kmp", file=nokmp) + logging.debug("nokmp") + print(kbid, file=nokmp) else: - print("Keyboard:", kb['id'], "has no data", file=nodata) + logging.debug("nodata") + print(kbid, file=nodata) if __name__ == "__main__": diff --git a/linux/keyman-config/keyman_config/get_kmp.py b/linux/keyman-config/keyman_config/get_kmp.py index 4c286d4e8e..68c607163d 100755 --- a/linux/keyman-config/keyman_config/get_kmp.py +++ b/linux/keyman-config/keyman_config/get_kmp.py @@ -54,7 +54,7 @@ def get_download_folder(): os.mkdir(km_cache) return km_cache -def get_kmp_file(kbdata): +def get_kmp_file(kbdata, cache=False): """ Get info from keyboard data to download kmp then download it. @@ -69,9 +69,9 @@ def get_kmp_file(kbdata): kmp_url = "https://downloads.keyman.com/keyboards/" + kbdata['id'] + "/" + kbdata['version'] + "/" + kbdata['packageFilename'] downloadfile = os.path.join(get_download_folder(), kbdata['packageFilename']) - return download_kmp_file(kmp_url, downloadfile) + return download_kmp_file(kmp_url, downloadfile, cache) -def download_kmp_file(url, kmpfile): +def download_kmp_file(url, kmpfile, cache=False): """ Download kmp file. @@ -85,7 +85,25 @@ def download_kmp_file(url, kmpfile): """ logging.info("Download URL: %s", url) downloadfile = None + + if cache: + home = str(Path.home()) + cache_dir = os.path.join(home, ".local/share/keyman") + current_dir = os.getcwd() + expire_after = datetime.timedelta(days=1) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + os.chdir(cache_dir) + requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after) + now = time.ctime(int(time.time())) + response = requests.get(url) #, stream=True) + + if cache: + logging.debug("Time: {0} / Used Cache: {1}".format(now, response.from_cache)) + os.chdir(current_dir) + requests_cache.core.uninstall_cache() + if response.status_code == 200: with open(kmpfile, 'wb') as f: f.write(response.content) From f3f14922edd71c95272c407a11166f541d8b6931 Mon Sep 17 00:00:00 2001 From: glasseyes Date: Thu, 20 Sep 2018 17:14:24 +0700 Subject: [PATCH 2/7] experiments: add check_kmp to see what kmp files have problems starting with what have files have type KM_UNKNOWN --- linux/.gitignore | 5 +- linux/keyman-config/experiments/check_kmp.py | 49 ++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100755 linux/keyman-config/experiments/check_kmp.py diff --git a/linux/.gitignore b/linux/.gitignore index f9199713f6..92f1ab9954 100644 --- a/linux/.gitignore +++ b/linux/.gitignore @@ -28,7 +28,7 @@ __pycache__ build-* deb-* builddebs -make_deb +keyman-config/make_deb tmp/ dist/ *~ @@ -54,4 +54,7 @@ ibus-kmfl/m4/visibility.m4 ibus-kmfl/m4/wchar_t.m4 ibus-kmfl/m4/wint_t.m4 ibus-kmfl/m4/xsize.m4 +keyman-config/build keyman-config/debian/man +keyman-config/experiments/*.txt +test.sh diff --git a/linux/keyman-config/experiments/check_kmp.py b/linux/keyman-config/experiments/check_kmp.py new file mode 100755 index 0000000000..d9ceb8dc5c --- /dev/null +++ b/linux/keyman-config/experiments/check_kmp.py @@ -0,0 +1,49 @@ +#!/usr/bin/python3 + +import datetime +import logging +import os +import requests +import requests_cache +import subprocess +import tempfile +import time +from dirlist import list_keyboards +#from keymankeyboards import get_api_keyboards + +#from keyman_config import get_kmp, install_kmp +#from keyman_config.get_kmp import get_keyboard_data, get_kmp_file +from keyman_config.install_kmp import get_metadata, extract_kmp +from keyman_config.kmpmetadata import KMFileTypes + +#TODO check if any files in files list in cached kmps are KM_UNKNOWN + +def main(): + logging.basicConfig(level=logging.DEBUG, format='%(levelname)s:%(message)s') + keyboarddata = list_keyboards() + home = os.path.expanduser("~") + cachebase = os.environ.get("XDG_CACHE_HOME", os.path.join(home, ".cache")) + km_cache=os.path.join(cachebase, "keyman") + if keyboarddata: + with open('./unknownfiles.txt', 'wt') as unknownfiles: + print("files found in cached kmps of type KM_UNKNOWN", file=unknownfiles) + for kbid in keyboarddata: + kmpfile = os.path.join(km_cache, kbid+".kmp") + if os.path.exists(kmpfile): + with tempfile.TemporaryDirectory() as tmpdirname: + extract_kmp(kmpfile, tmpdirname) + try: + info, system, options, keyboards, files = get_metadata(tmpdirname) + if files: + for kbfile in files: + if kbfile['type'] == KMFileTypes.KM_UNKNOWN: + print(kbfile['name'], file=unknownfiles) + except Exception as e: + print(type(e)) # the exception instance + print(e.args) # arguments stored in .args + print(e) # __str__ allows args to be printed directly, pass + pass + + +if __name__ == "__main__": + main() From 2d0815b69c4ce978200b4a2019b34bca3f047832 Mon Sep 17 00:00:00 2001 From: glasseyes Date: Fri, 21 Sep 2018 10:44:10 +0700 Subject: [PATCH 3/7] [linux] for #1189 handle more file types --- linux/keyman-config/keyman_config/kmpmetadata.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/linux/keyman-config/keyman_config/kmpmetadata.py b/linux/keyman-config/keyman_config/kmpmetadata.py index e7100b59fa..1c6365c5ac 100755 --- a/linux/keyman-config/keyman_config/kmpmetadata.py +++ b/linux/keyman-config/keyman_config/kmpmetadata.py @@ -108,7 +108,7 @@ def determine_filetype(filename): name, ext = os.path.splitext(filename) if ext.lower() == ".ico": return KMFileTypes.KM_ICON - elif ext.lower() == ".kmn": + elif ext.lower() == ".kmn" or ext.lower() == ".kvks": return KMFileTypes.KM_SOURCE elif ext.lower() == ".kmx": return KMFileTypes.KM_KMX @@ -116,11 +116,17 @@ def determine_filetype(filename): return KMFileTypes.KM_KVK elif ext.lower() == ".ttf" or ext.lower() == ".otf": return KMFileTypes.KM_FONT - elif ext.lower() == ".txt" or ext.lower() == ".pdf" or ext.lower() == ".htm" or ext.lower() == ".html": + elif ext.lower() == ".txt" or ext.lower() == ".pdf" or ext.lower() == ".htm" \ + or ext.lower() == ".html" or ext.lower() == ".doc" or ext.lower() == ".docx" \ + or ext.lower() == ".css" or ext.lower() == ".chm" or ext.lower() == "" \ + or ext.lower() == ".md" or ext.lower() == ".odt" or ext.lower() == ".rtf" \ + or ext.lower() == ".dot": return KMFileTypes.KM_DOC elif ext.lower() == ".inf" or ext.lower() == ".json": return KMFileTypes.KM_META - elif ext.lower() == ".png" or ext.lower() == ".jpeg" or ext.lower() == ".jpg" or ext.lower() == ".gif": + elif ext.lower() == ".png" or ext.lower() == ".jpeg" \ + or ext.lower() == ".jpg" or ext.lower() == ".gif" \ + or ext.lower() == ".bmp": return KMFileTypes.KM_IMAGE else: return KMFileTypes.KM_UNKNOWN From c2d124b15fe95d6c36b935671a04a65d26cdeea6 Mon Sep 17 00:00:00 2001 From: glasseyes Date: Fri, 21 Sep 2018 13:36:27 +0700 Subject: [PATCH 4/7] debug logging of paths not print --- linux/keyman-config/experiments/dirlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linux/keyman-config/experiments/dirlist.py b/linux/keyman-config/experiments/dirlist.py index 847ed904c6..f2a2e675e2 100755 --- a/linux/keyman-config/experiments/dirlist.py +++ b/linux/keyman-config/experiments/dirlist.py @@ -42,7 +42,7 @@ def get_dir_list(): def list_keyboards(): kblist = [] for file in get_dir_list(): - print(file) + logging.debug(file) kb = os.path.basename(os.path.dirname(file)) if kb != "keyboards": kblist.append(kb) From 8c2d186dc17fa98f885f8964afc77034dd617d65 Mon Sep 17 00:00:00 2001 From: glasseyes Date: Mon, 24 Sep 2018 12:01:00 +0700 Subject: [PATCH 5/7] [linux] options to cache more and for longer in experiments --- linux/keyman-config/experiments/check_kmp.py | 12 ++++-------- linux/keyman-config/experiments/dirlist.py | 7 +++---- linux/keyman-config/experiments/keymankeyboards.py | 3 ++- linux/keyman-config/experiments/list_kmp.py | 10 ++++------ linux/keyman-config/keyman_config/get_kmp.py | 14 ++++++++------ 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/linux/keyman-config/experiments/check_kmp.py b/linux/keyman-config/experiments/check_kmp.py index d9ceb8dc5c..767b462d4f 100755 --- a/linux/keyman-config/experiments/check_kmp.py +++ b/linux/keyman-config/experiments/check_kmp.py @@ -9,21 +9,17 @@ import subprocess import tempfile import time from dirlist import list_keyboards -#from keymankeyboards import get_api_keyboards - -#from keyman_config import get_kmp, install_kmp -#from keyman_config.get_kmp import get_keyboard_data, get_kmp_file from keyman_config.install_kmp import get_metadata, extract_kmp from keyman_config.kmpmetadata import KMFileTypes +from keyman_config.get_kmp import keyman_cache_dir #TODO check if any files in files list in cached kmps are KM_UNKNOWN def main(): - logging.basicConfig(level=logging.DEBUG, format='%(levelname)s:%(message)s') + logging.basicConfig(level=logging.INFO, format='%(levelname)s:%(message)s') + km_cache=keyman_cache_dir() + logging.info("looking for KM_UNKNOWN files in cached kmps") keyboarddata = list_keyboards() - home = os.path.expanduser("~") - cachebase = os.environ.get("XDG_CACHE_HOME", os.path.join(home, ".cache")) - km_cache=os.path.join(cachebase, "keyman") if keyboarddata: with open('./unknownfiles.txt', 'wt') as unknownfiles: print("files found in cached kmps of type KM_UNKNOWN", file=unknownfiles) diff --git a/linux/keyman-config/experiments/dirlist.py b/linux/keyman-config/experiments/dirlist.py index f2a2e675e2..ebdc5186af 100755 --- a/linux/keyman-config/experiments/dirlist.py +++ b/linux/keyman-config/experiments/dirlist.py @@ -7,15 +7,14 @@ import requests import requests_cache import os import time +from keyman_config.get_kmp import keyman_cache_dir def get_keyboard_dir_page(kb_url): logging.info("Getting keyboard list") logging.debug("At URL %s", kb_url) - home = os.path.expanduser("~") - datahome = os.environ.get("XDG_DATA_HOME", os.path.join(home, ".local", "share")) - cache_dir = os.path.join(datahome, "keyman") + cache_dir = keyman_cache_dir() current_dir = os.getcwd() - expire_after = datetime.timedelta(days=1) + expire_after = datetime.timedelta(days=7) if not os.path.isdir(cache_dir): os.makedirs(cache_dir) os.chdir(cache_dir) diff --git a/linux/keyman-config/experiments/keymankeyboards.py b/linux/keyman-config/experiments/keymankeyboards.py index d2163fdfb5..a07a2cb1bf 100755 --- a/linux/keyman-config/experiments/keymankeyboards.py +++ b/linux/keyman-config/experiments/keymankeyboards.py @@ -22,6 +22,7 @@ import requests import requests_cache import os from pathlib import Path +from keyman_config.get_kmp import keyman_cache_dir def get_api_keyboards(verbose=False): """ @@ -37,7 +38,7 @@ def get_api_keyboards(verbose=False): headers = {'Content-Type': 'application/json', 'Accept-Encoding': 'gzip, deflate, br'} home = str(Path.home()) - cache_dir = os.path.join(home, ".local/share/keyman") + cache_dir = keyman_cache_dir() current_dir = os.getcwd() expire_after = datetime.timedelta(days=1) if not os.path.isdir(cache_dir): diff --git a/linux/keyman-config/experiments/list_kmp.py b/linux/keyman-config/experiments/list_kmp.py index 38024b82d1..d67f635dab 100755 --- a/linux/keyman-config/experiments/list_kmp.py +++ b/linux/keyman-config/experiments/list_kmp.py @@ -12,7 +12,7 @@ from dirlist import list_keyboards #from keymankeyboards import get_api_keyboards #from keyman_config import get_kmp, install_kmp -from keyman_config.get_kmp import get_keyboard_data, get_kmp_file +from keyman_config.get_kmp import get_keyboard_data, get_kmp_file, keyman_cache_dir from keyman_config.install_kmp import get_metadata, get_infdata, extract_kmp #TODO check for kmn and check if it is compilable @@ -22,11 +22,9 @@ def get_kmn(kbid, sourcePath): base_url = "https://raw.github.com/keymanapp/keyboards/master/" + sourcePath kmn_url = base_url + "/source/" + kbid + ".kmn" - home = os.path.expanduser("~") - datahome = os.environ.get("XDG_DATA_HOME", os.path.join(home, ".local", "share")) - cache_dir = os.path.join(datahome, "keyman") + cache_dir = keyman_cache_dir() current_dir = os.getcwd() - expire_after = datetime.timedelta(days=1) + expire_after = datetime.timedelta(days=7) if not os.path.isdir(cache_dir): os.makedirs(cache_dir) os.chdir(cache_dir) @@ -65,7 +63,7 @@ def main(): for kbid in keyboarddata: - kbdata = get_keyboard_data(kbid) + kbdata = get_keyboard_data(kbid, True) print(kbid) if kbdata: if 'packageFilename' in kbdata: diff --git a/linux/keyman-config/keyman_config/get_kmp.py b/linux/keyman-config/keyman_config/get_kmp.py index 19aa2b88b5..a8d089421b 100755 --- a/linux/keyman-config/keyman_config/get_kmp.py +++ b/linux/keyman-config/keyman_config/get_kmp.py @@ -11,7 +11,7 @@ import os from pathlib import Path -def get_keyboard_data(keyboardid): +def get_keyboard_data(keyboardid, weekCache=False): """ Get Keyboard data from web api. @@ -26,7 +26,10 @@ def get_keyboard_data(keyboardid): home = str(Path.home()) cache_dir = keyman_cache_dir() current_dir = os.getcwd() - expire_after = datetime.timedelta(days=1) + if weekCache: + expire_after = datetime.timedelta(days=7) + else: + expire_after = datetime.timedelta(days=1) os.chdir(cache_dir) requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after) now = time.ctime(int(time.time())) @@ -110,14 +113,13 @@ def download_kmp_file(url, kmpfile, cache=False): downloadfile = None if cache: - home = str(Path.home()) - cache_dir = os.path.join(home, ".local/share/keyman") + cache_dir = keyman_cache_dir() current_dir = os.getcwd() - expire_after = datetime.timedelta(days=1) + expire_after = datetime.timedelta(days=7) if not os.path.isdir(cache_dir): os.makedirs(cache_dir) os.chdir(cache_dir) - requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after) + requests_cache.install_cache(cache_name='keyman_kmp_cache', backend='sqlite', expire_after=expire_after) now = time.ctime(int(time.time())) response = requests.get(url) #, stream=True) From 675c200c21901112c49ecca67ee74d63f27362f4 Mon Sep 17 00:00:00 2001 From: glasseyes Date: Wed, 3 Oct 2018 15:47:34 +0700 Subject: [PATCH 6/7] [linux] fix merge in kmpmetadata --- linux/keyman-config/keyman_config/kmpmetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linux/keyman-config/keyman_config/kmpmetadata.py b/linux/keyman-config/keyman_config/kmpmetadata.py index d4e410ce0e..192cf054ca 100755 --- a/linux/keyman-config/keyman_config/kmpmetadata.py +++ b/linux/keyman-config/keyman_config/kmpmetadata.py @@ -121,7 +121,7 @@ def determine_filetype(kblist, filename): name, ext = os.path.splitext(filename) if ext.lower() == ".ico": return KMFileTypes.KM_ICON - elif ext.lower() == ".kmn" or ext.lower() == ".kvks": + elif ext.lower() == ".kmn": return KMFileTypes.KM_SOURCE elif ext.lower() == ".kvks": return KMFileTypes.KM_OSK_SOURCE From c08cb2e2d4420472d474f37219fef452ad20a52c Mon Sep 17 00:00:00 2001 From: glasseyes Date: Thu, 4 Oct 2018 11:34:45 +0700 Subject: [PATCH 7/7] [linux] changes from review 1232 --- linux/keyman-config/experiments/findkmninkmp.sh | 10 ---------- linux/keyman-config/keyman_config/get_kmp.py | 3 +++ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/linux/keyman-config/experiments/findkmninkmp.sh b/linux/keyman-config/experiments/findkmninkmp.sh index ecdb1d473a..73df3af809 100755 --- a/linux/keyman-config/experiments/findkmninkmp.sh +++ b/linux/keyman-config/experiments/findkmninkmp.sh @@ -10,13 +10,3 @@ do # echo "no kmn in $file" fi done - -#for file in `find ~/.cache/keyman -name "*.kmp"` -#do -# unzip -q -t "$file" *.kmn 2>&1 -# if [ "$?" == "0" ]; then -# echo "$file" -## else -## echo "no kmn in $file" -# fi -#done diff --git a/linux/keyman-config/keyman_config/get_kmp.py b/linux/keyman-config/keyman_config/get_kmp.py index 8045b158c8..3d831a3a02 100755 --- a/linux/keyman-config/keyman_config/get_kmp.py +++ b/linux/keyman-config/keyman_config/get_kmp.py @@ -17,6 +17,7 @@ def get_keyboard_data(keyboardid, weekCache=False): Args: keyboardid (str): Keyboard ID + weekCache (bool) : cache data for 1 week, default is 1 day Returns: dict: Keyboard data """ @@ -87,6 +88,7 @@ def get_kmp_file(kbdata, cache=False): Args: kbdata (dict): Keyboard data + cache (bool): Whether to cache the kmp file web request Returns: str: path where kmp file has been downloaded """ @@ -107,6 +109,7 @@ def download_kmp_file(url, kmpfile, cache=False): kmpfile (str): Where to save the kmp file. currently it does no checks on this location assumes that is in users keyman cache dir + cache(bool): Whether to cache the kmp file web request for a week Returns: str: path where kmp file has been downloaded """