Merge pull request #1232 from keymanapp/linux-list-kmp

[linux] list kmp experiments

linux/.gitignore

@@ -28,7 +28,7 @@ __pycache__
 build-*
 deb-*
 builddebs
-make_deb
+keyman-config/make_deb
 tmp/
 dist/
 *~
@@ -54,5 +54,7 @@ ibus-kmfl/m4/visibility.m4
 ibus-kmfl/m4/wchar_t.m4
 ibus-kmfl/m4/wint_t.m4
 ibus-kmfl/m4/xsize.m4
-keyman-config/debian/man
 keyman-config/build
+keyman-config/debian/man
+keyman-config/experiments/*.txt
+test.sh

linux/keyman-config/experiments/check_kmp.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+
+import datetime
+import logging
+import os
+import requests
+import requests_cache
+import subprocess
+import tempfile
+import time
+from dirlist import list_keyboards
+from keyman_config.install_kmp import get_metadata, extract_kmp
+from keyman_config.kmpmetadata import KMFileTypes
+from keyman_config.get_kmp import keyman_cache_dir
+
+#TODO check if any files in files list in cached kmps are KM_UNKNOWN
+
+def main():
+	logging.basicConfig(level=logging.INFO, format='%(levelname)s:%(message)s')
+	km_cache=keyman_cache_dir()
+	logging.info("looking for KM_UNKNOWN files in cached kmps")
+	keyboarddata = list_keyboards()
+	if keyboarddata:
+		with open('./unknownfiles.txt', 'wt') as unknownfiles:
+			print("files found in cached kmps of type KM_UNKNOWN", file=unknownfiles)
+			for kbid in keyboarddata:
+				kmpfile = os.path.join(km_cache, kbid+".kmp")
+				if os.path.exists(kmpfile):
+					with tempfile.TemporaryDirectory() as tmpdirname:
+						extract_kmp(kmpfile, tmpdirname)
+						try:
+							info, system, options, keyboards, files = get_metadata(tmpdirname)
+							if files:
+								for kbfile in files:
+									if kbfile['type'] == KMFileTypes.KM_UNKNOWN:
+										print(kbfile['name'], file=unknownfiles)
+						except Exception as e:
+							print(type(e))    # the exception instance
+							print(e.args)     # arguments stored in .args
+							print(e)          # __str__ allows args to be printed directly,		pass
+							pass
+
+
+if __name__ == "__main__":
+	main()

linux/keyman-config/experiments/dirlist.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python3
+
+from bs4 import BeautifulSoup
+import datetime
+import logging
+import requests
+import requests_cache
+import os
+import time
+from keyman_config.get_kmp import keyman_cache_dir
+
+def get_keyboard_dir_page(kb_url):
+    logging.info("Getting keyboard list")
+    logging.debug("At URL %s", kb_url)
+    cache_dir = keyman_cache_dir()
+    current_dir = os.getcwd()
+    expire_after = datetime.timedelta(days=7)
+    if not os.path.isdir(cache_dir):
+        os.makedirs(cache_dir)
+    os.chdir(cache_dir)
+    requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after)
+    now = time.ctime(int(time.time()))
+    response = requests.get(kb_url)
+    logging.debug("Time: {0} / Used Cache: {1}".format(now, response.from_cache))
+    os.chdir(current_dir)
+    requests_cache.core.uninstall_cache()
+    if response.status_code == 200:
+        return response.text
+    else:
+        return None
+
+
+def get_dir_list():
+    url = "https://downloads.keyman.com/keyboards/"
+    page = get_keyboard_dir_page(url)
+#    print(page)
+    soup = BeautifulSoup(page, 'html.parser')
+#    return [url + '/' + node.get('href') for node in soup.find_all('a') if node.get('href').endswith(ext)]
+    return [url + node.get('href') for node in soup.find_all('a')]
+
+def list_keyboards():
+    kblist = []
+    for file in get_dir_list():
+        logging.debug(file)
+        kb = os.path.basename(os.path.dirname(file))
+        if kb != "keyboards":
+            kblist.append(kb)
+    return kblist

linux/keyman-config/experiments/findkmninkmp.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+echo "cached kmp with kmn files inside them:"
+find ~/.cache/keyman/ -print0| while read -d $'\0' file
+do
+	unzip -q -t "$file" *.kmn 2>&1 
+	if [ "$?" == "0" ]; then
+		basename "$file" .kmp
+#	else
+#		echo "no kmn in $file"
+	fi
+done

linux/keyman-config/experiments/keymankeyboards.py

@@ -22,6 +22,7 @@ import requests
 import requests_cache
 import os
 from pathlib import Path
+from keyman_config.get_kmp import keyman_cache_dir
 
 def get_api_keyboards(verbose=False):
 	"""
@@ -33,11 +34,11 @@ def get_api_keyboards(verbose=False):
 		dict: Keyboard data
 		None: if http request not successful
 	"""
-	api_url = "https://api.keyman.com/cloud/4.0/keyboards"
+	api_url = "https://api.keyman.com/cloud/4.0/keyboards?version=10.0"
 	headers = {'Content-Type': 'application/json',
 		'Accept-Encoding': 'gzip, deflate, br'}
 	home = str(Path.home())
-	cache_dir = os.path.join(home, ".local/share/keyman")
+	cache_dir = keyman_cache_dir()
 	current_dir = os.getcwd()
 	expire_after = datetime.timedelta(days=1)
 	if not os.path.isdir(cache_dir):

linux/keyman-config/experiments/list_kmp.py

@@ -1,42 +1,117 @@
 #!/usr/bin/python3
 
+import datetime
+import logging
+import os
+import requests
+import requests_cache
+import subprocess
 import tempfile
-from keymankeyboards import get_api_keyboards
-from get_kmp import get_keyboard_data, get_kmp_file
-from install_kmp import get_metadata, get_infdata, extract_kmp
+import time
+from dirlist import list_keyboards
+#from keymankeyboards import get_api_keyboards
+
+#from keyman_config import get_kmp, install_kmp
+from keyman_config.get_kmp import get_keyboard_data, get_kmp_file, keyman_cache_dir
+from keyman_config.install_kmp import get_metadata, get_infdata, extract_kmp
+
+#TODO check for kmn and check if it is compilable
+#TODO extra output files jsonkmpnokmn, jsonkmpbadkmn, goodjsonkmpkmn and for inf as well
+
+def get_kmn(kbid, sourcePath):
+	base_url = "https://raw.github.com/keymanapp/keyboards/master/" + sourcePath
+	kmn_url = base_url + "/source/" + kbid + ".kmn"
+
+	cache_dir = keyman_cache_dir()
+	current_dir = os.getcwd()
+	expire_after = datetime.timedelta(days=7)
+	if not os.path.isdir(cache_dir):
+		os.makedirs(cache_dir)
+	os.chdir(cache_dir)
+	requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after)
+	now = time.ctime(int(time.time()))
+	response = requests.get(kmn_url)
+	logging.debug("Time: {0} / Used Cache: {1}".format(now, response.from_cache))
+	os.chdir(current_dir)
+	requests_cache.core.uninstall_cache()
+
+	return requests.get(kmn_url)	
 
 def main():
-	keyboarddata = get_api_keyboards()
+	logging.basicConfig(level=logging.DEBUG, format='%(levelname)s:%(message)s')
+	keyboarddata = list_keyboards()
 	if keyboarddata:
 		with open('./nokmp.txt', 'wt') as nokmp, \
 			open('./infnokeyboard.txt', 'wt') as infnokeyboard, \
+			open('./goodjsonkmpkmn.txt', 'wt') as goodjsonkmpkmn, \
+			open('./jsonkmpnokmn.txt', 'wt') as jsonkmpnokmn, \
+			open('./jsonkmpbadkmn.txt', 'wt') as jsonkmpbadkmn, \
+			open('./jsonkmpmissingkmn.txt', 'wt') as jsonkmpmissingkmn, \
 			open('./brokeninf.txt', 'wt') as brokeninf, \
 			open('./nodata.txt', 'wt') as nodata, \
-			open('./goodjsonkmp.txt', 'wt') as goodjsonkmp, \
 			open('./goodinfkmp.txt', 'wt') as goodinfkmp:
-			for kb in keyboarddata['keyboard']:
-				kbdata = get_keyboard_data(kb['id'])
-				print(kb['id'])
+
+			print("Keyboard: will work in kmfl :)", file=goodjsonkmpkmn) # goodjsonkmpkmn
+			print("Keyboard: has uncompilable kmn", file=jsonkmpbadkmn) # jsonkmpbadkmn
+			print("Keyboard: has json in kmp but can't find the kmn on github", file=jsonkmpmissingkmn) # jsonkmpmissingkmn
+			print("Keyboard: has json in kmp but has no sourcePath to look for kmn on github", file=jsonkmpnokmn) # jsonkmpnokmn
+			print("Keyboard: has kmp with kmp.inf", file=goodinfkmp)
+			print("Keyboard: has kmp with kmp.inf but it has no Keyboard", file=infnokeyboard)
+			print("Keyboard: has kmp but no kmp.json and no or broken kmp.inf", file=brokeninf)
+			print("Keyboard: does not have kmp so mobile/web only", file=nokmp)
+			print("Keyboard: has no data", file=nodata)
+
+
+			for kbid in keyboarddata:
+				kbdata = get_keyboard_data(kbid, True)
+				print(kbid)
 				if kbdata:
 					if 'packageFilename' in kbdata:
-						kmpfile = get_kmp_file(kbdata)
+						kmpfile = get_kmp_file(kbdata, True)
 						with tempfile.TemporaryDirectory() as tmpdirname:
 							extract_kmp(kmpfile, tmpdirname)
-							info, system, options, keyboards, files = get_metadata(tmpdirname)
-							if keyboards:
-								print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "with kmp.json", file=goodjsonkmp)
-							else:
-								info, system, options, keyboards, files = get_infdata(tmpdirname)
+							try:
+								info, system, options, keyboards, files = get_metadata(tmpdirname)
 								if keyboards:
-									print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "with kmp.inf", file=goodinfkmp)
-								elif files:
-									print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "with kmp.inf but it has no Keyboard", file=infnokeyboard)
+									if 'sourcePath' in kbdata:
+										response = get_kmn(kbid, kbdata['sourcePath'])
+										if response.status_code == 200:
+											kmndownloadfile = os.path.join(tmpdirname, kbid + ".kmn")
+											with open(kmndownloadfile, 'wb') as f:
+												f.write(response.content)
+											subprocess.run(["kmflcomp", kmndownloadfile], stdout=subprocess.PIPE, stderr= subprocess.STDOUT)
+											kmfl_file = os.path.join(tmpdirname, kbid + ".kmfl")
+											if os.path.isfile(kmfl_file):
+												logging.debug("goodjsonkmpkmn")
+												print(kbid, file=goodjsonkmpkmn) # goodjsonkmpkmn
+											else:
+												logging.debug("jsonkmpbadkmn")
+												print(kbid, file=jsonkmpbadkmn) # jsonkmpbadkmn
+										else:
+											logging.debug("jsonkmpmissingkmn")
+											print(kbid, file=jsonkmpmissingkmn) # jsonkmpmissingkmn
+									else:
+										logging.debug("jsonkmpnokmn")
+										print(kbid, file=jsonkmpnokmn) # jsonkmpnokmn
 								else:
-									print("Keyboard:", kb['id'], "has kmp", kbdata['packageFilename'], "but no kmp.json and no or broken kmp.inf", file=brokeninf)
+									info, system, options, keyboards, files = get_infdata(tmpdirname)
+									if keyboards:
+										logging.debug("infnokeyboard")
+										print(kbid, file=goodinfkmp)
+									elif files:
+										logging.debug("goodinfkmp")
+										print(kbid, file=infnokeyboard)
+									else:
+										print(kbid, file=brokeninf)
+							except KeyError:
+								logging.debug("brokeninf")
+								print(kbid, file=brokeninf)
 					else:
-						print("Keyboard:", kb['id'], "does not have kmp", file=nokmp)
+						logging.debug("nokmp")
+						print(kbid, file=nokmp)
 				else:
-					print("Keyboard:", kb['id'], "has no data", file=nodata)
+					logging.debug("nodata")
+					print(kbid, file=nodata)
 
 
 if __name__ == "__main__":

linux/keyman-config/keyman_config/get_kmp.py

@@ -11,12 +11,13 @@ import os
 from pathlib import Path
 
 
-def get_keyboard_data(keyboardid):
+def get_keyboard_data(keyboardid, weekCache=False):
 	"""
 	Get Keyboard data from web api.
 
 	Args:
 		keyboardid (str): Keyboard ID
+		weekCache (bool) : cache data for 1 week, default is 1 day
 	Returns:
 		dict: Keyboard data
 	"""
@@ -26,7 +27,10 @@ def get_keyboard_data(keyboardid):
 	home = str(Path.home())
 	cache_dir = keyman_cache_dir()
 	current_dir = os.getcwd()
-	expire_after = datetime.timedelta(days=1)
+	if weekCache:
+		expire_after = datetime.timedelta(days=7)
+	else:
+		expire_after = datetime.timedelta(days=1)
 	os.chdir(cache_dir)
 	requests_cache.install_cache(cache_name='keyman_cache', backend='sqlite', expire_after=expire_after)
 	now = time.ctime(int(time.time()))
@@ -78,12 +82,13 @@ def user_keyboard_dir(keyboardid):
 	return os.path.join(user_keyman_dir(), keyboardid)
 
 
-def get_kmp_file(kbdata):
+def get_kmp_file(kbdata, cache=False):
 	"""
 	Get info from keyboard data to download kmp then download it.
 
 	Args:
 		kbdata (dict): Keyboard data
+		cache (bool): Whether to cache the kmp file web request
 	Returns:
 		str: path where kmp file has been downloaded
 	"""
@@ -93,9 +98,9 @@ def get_kmp_file(kbdata):
 
 	kmp_url = "https://downloads.keyman.com/keyboards/" + kbdata['id'] + "/" + kbdata['version'] + "/" + kbdata['packageFilename']
 	downloadfile = os.path.join(get_download_folder(), kbdata['packageFilename'])
-	return download_kmp_file(kmp_url, downloadfile)
+	return download_kmp_file(kmp_url, downloadfile, cache)
 
-def download_kmp_file(url, kmpfile):
+def download_kmp_file(url, kmpfile, cache=False):
 	"""
 	Download kmp file.
 
@@ -104,12 +109,30 @@ def download_kmp_file(url, kmpfile):
 		kmpfile (str): Where to save the kmp file.
 			currently it does no checks on this location
 			assumes that is in users keyman cache dir
+		cache(bool): Whether to cache the kmp file web request for a week
 	Returns:
 		str: path where kmp file has been downloaded
 	"""
 	logging.info("Download URL: %s", url)
 	downloadfile = None
+
+	if cache:
+		cache_dir = keyman_cache_dir()
+		current_dir = os.getcwd()
+		expire_after = datetime.timedelta(days=7)
+		if not os.path.isdir(cache_dir):
+			os.makedirs(cache_dir)
+		os.chdir(cache_dir)
+		requests_cache.install_cache(cache_name='keyman_kmp_cache', backend='sqlite', expire_after=expire_after)
+		now = time.ctime(int(time.time()))
+
 	response = requests.get(url) #, stream=True)
+
+	if cache:
+		logging.debug("Time: {0} / Used Cache: {1}".format(now, response.from_cache))
+		os.chdir(current_dir)
+		requests_cache.core.uninstall_cache()
+
 	if response.status_code == 200:
 		with open(kmpfile, 'wb') as f:
 			f.write(response.content)