info.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591
  1. """Functions that help us generate and use info.json files.
  2. """
  3. import json
  4. from collections.abc import Mapping
  5. from glob import glob
  6. from pathlib import Path
  7. import hjson
  8. import jsonschema
  9. from dotty_dict import dotty
  10. from milc import cli
  11. from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
  12. from qmk.c_parse import find_layouts
  13. from qmk.keyboard import config_h, rules_mk
  14. from qmk.keymap import list_keymaps
  15. from qmk.makefile import parse_rules_mk_file
  16. from qmk.math import compute
  17. true_values = ['1', 'on', 'yes']
  18. false_values = ['0', 'off', 'no']
  19. def info_json(keyboard):
  20. """Generate the info.json data for a specific keyboard.
  21. """
  22. cur_dir = Path('keyboards')
  23. rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk')
  24. if 'DEFAULT_FOLDER' in rules:
  25. keyboard = rules['DEFAULT_FOLDER']
  26. rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules)
  27. info_data = {
  28. 'keyboard_name': str(keyboard),
  29. 'keyboard_folder': str(keyboard),
  30. 'keymaps': {},
  31. 'layouts': {},
  32. 'parse_errors': [],
  33. 'parse_warnings': [],
  34. 'maintainer': 'qmk',
  35. }
  36. # Populate the list of JSON keymaps
  37. for keymap in list_keymaps(keyboard, c=False, fullpath=True):
  38. info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'}
  39. # Populate layout data
  40. layouts, aliases = _find_all_layouts(info_data, keyboard)
  41. if aliases:
  42. info_data['layout_aliases'] = aliases
  43. for layout_name, layout_json in layouts.items():
  44. if not layout_name.startswith('LAYOUT_kc'):
  45. layout_json['c_macro'] = True
  46. info_data['layouts'][layout_name] = layout_json
  47. # Merge in the data from info.json, config.h, and rules.mk
  48. info_data = merge_info_jsons(keyboard, info_data)
  49. info_data = _extract_config_h(info_data)
  50. info_data = _extract_rules_mk(info_data)
  51. # Validate against the jsonschema
  52. try:
  53. keyboard_api_validate(info_data)
  54. except jsonschema.ValidationError as e:
  55. json_path = '.'.join([str(p) for p in e.absolute_path])
  56. cli.log.error('Invalid API data: %s: %s: %s', keyboard, json_path, e.message)
  57. exit()
  58. # Make sure we have at least one layout
  59. if not info_data.get('layouts'):
  60. _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
  61. # Make sure we supply layout macros for the community layouts we claim to support
  62. for layout in info_data.get('community_layouts', []):
  63. layout_name = 'LAYOUT_' + layout
  64. if layout_name not in info_data.get('layouts', {}) and layout_name not in info_data.get('layout_aliases', {}):
  65. _log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
  66. return info_data
  67. def _json_load(json_file):
  68. """Load a json file from disk.
  69. Note: file must be a Path object.
  70. """
  71. try:
  72. return hjson.load(json_file.open(encoding='utf-8'))
  73. except json.decoder.JSONDecodeError as e:
  74. cli.log.error('Invalid JSON encountered attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e)
  75. exit(1)
  76. def _jsonschema(schema_name):
  77. """Read a jsonschema file from disk.
  78. FIXME(skullydazed/anyone): Refactor to make this a public function.
  79. """
  80. schema_path = Path(f'data/schemas/{schema_name}.jsonschema')
  81. if not schema_path.exists():
  82. schema_path = Path('data/schemas/false.jsonschema')
  83. return _json_load(schema_path)
  84. def keyboard_validate(data):
  85. """Validates data against the keyboard jsonschema.
  86. """
  87. schema = _jsonschema('keyboard')
  88. validator = jsonschema.Draft7Validator(schema).validate
  89. return validator(data)
  90. def keyboard_api_validate(data):
  91. """Validates data against the api_keyboard jsonschema.
  92. """
  93. base = _jsonschema('keyboard')
  94. relative = _jsonschema('api_keyboard')
  95. resolver = jsonschema.RefResolver.from_schema(base)
  96. validator = jsonschema.Draft7Validator(relative, resolver=resolver).validate
  97. return validator(data)
  98. def _extract_features(info_data, rules):
  99. """Find all the features enabled in rules.mk.
  100. """
  101. # Special handling for bootmagic which also supports a "lite" mode.
  102. if rules.get('BOOTMAGIC_ENABLE') == 'lite':
  103. rules['BOOTMAGIC_LITE_ENABLE'] = 'on'
  104. del rules['BOOTMAGIC_ENABLE']
  105. if rules.get('BOOTMAGIC_ENABLE') == 'full':
  106. rules['BOOTMAGIC_ENABLE'] = 'on'
  107. # Skip non-boolean features we haven't implemented special handling for
  108. for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE':
  109. if rules.get(feature):
  110. del rules[feature]
  111. # Process the rest of the rules as booleans
  112. for key, value in rules.items():
  113. if key.endswith('_ENABLE'):
  114. key = '_'.join(key.split('_')[:-1]).lower()
  115. value = True if value.lower() in true_values else False if value.lower() in false_values else value
  116. if 'config_h_features' not in info_data:
  117. info_data['config_h_features'] = {}
  118. if 'features' not in info_data:
  119. info_data['features'] = {}
  120. if key in info_data['features']:
  121. _log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,))
  122. info_data['features'][key] = value
  123. info_data['config_h_features'][key] = value
  124. return info_data
  125. def _pin_name(pin):
  126. """Returns the proper representation for a pin.
  127. """
  128. pin = pin.strip()
  129. if not pin:
  130. return None
  131. elif pin.isdigit():
  132. return int(pin)
  133. elif pin == 'NO_PIN':
  134. return None
  135. elif pin[0] in 'ABCDEFGHIJK' and pin[1].isdigit():
  136. return pin
  137. raise ValueError(f'Invalid pin: {pin}')
  138. def _extract_pins(pins):
  139. """Returns a list of pins from a comma separated string of pins.
  140. """
  141. return [_pin_name(pin) for pin in pins.split(',')]
  142. def _extract_direct_matrix(info_data, direct_pins):
  143. """
  144. """
  145. info_data['matrix_pins'] = {}
  146. direct_pin_array = []
  147. while direct_pins[-1] != '}':
  148. direct_pins = direct_pins[:-1]
  149. for row in direct_pins.split('},{'):
  150. if row.startswith('{'):
  151. row = row[1:]
  152. if row.endswith('}'):
  153. row = row[:-1]
  154. direct_pin_array.append([])
  155. for pin in row.split(','):
  156. if pin == 'NO_PIN':
  157. pin = None
  158. direct_pin_array[-1].append(pin)
  159. return direct_pin_array
  160. def _extract_matrix_info(info_data, config_c):
  161. """Populate the matrix information.
  162. """
  163. row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip()
  164. col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip()
  165. direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1]
  166. if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c:
  167. if 'matrix_size' in info_data:
  168. _log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.')
  169. info_data['matrix_size'] = {
  170. 'cols': compute(config_c.get('MATRIX_COLS', '0')),
  171. 'rows': compute(config_c.get('MATRIX_ROWS', '0')),
  172. }
  173. if row_pins and col_pins:
  174. if 'matrix_pins' in info_data:
  175. _log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.')
  176. info_data['matrix_pins'] = {
  177. 'cols': _extract_pins(col_pins),
  178. 'rows': _extract_pins(row_pins),
  179. }
  180. if direct_pins:
  181. if 'matrix_pins' in info_data:
  182. _log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.')
  183. info_data['matrix_pins']['direct'] = _extract_direct_matrix(info_data, direct_pins)
  184. return info_data
  185. def _extract_config_h(info_data):
  186. """Pull some keyboard information from existing config.h files
  187. """
  188. config_c = config_h(info_data['keyboard_folder'])
  189. # Pull in data from the json map
  190. dotty_info = dotty(info_data)
  191. info_config_map = _json_load(Path('data/mappings/info_config.json'))
  192. for config_key, info_dict in info_config_map.items():
  193. info_key = info_dict['info_key']
  194. key_type = info_dict.get('value_type', 'str')
  195. try:
  196. if config_key in config_c and info_dict.get('to_json', True):
  197. if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
  198. _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
  199. if key_type.startswith('array'):
  200. if '.' in key_type:
  201. key_type, array_type = key_type.split('.', 1)
  202. else:
  203. array_type = None
  204. config_value = config_c[config_key].replace('{', '').replace('}', '').strip()
  205. if array_type == 'int':
  206. dotty_info[info_key] = list(map(int, config_value.split(',')))
  207. else:
  208. dotty_info[info_key] = config_value.split(',')
  209. elif key_type == 'bool':
  210. dotty_info[info_key] = config_c[config_key] in true_values
  211. elif key_type == 'hex':
  212. dotty_info[info_key] = '0x' + config_c[config_key][2:].upper()
  213. elif key_type == 'list':
  214. dotty_info[info_key] = config_c[config_key].split()
  215. elif key_type == 'int':
  216. dotty_info[info_key] = int(config_c[config_key])
  217. else:
  218. dotty_info[info_key] = config_c[config_key]
  219. except Exception as e:
  220. _log_warning(info_data, f'{config_key}->{info_key}: {e}')
  221. info_data.update(dotty_info)
  222. # Pull data that easily can't be mapped in json
  223. _extract_matrix_info(info_data, config_c)
  224. return info_data
  225. def _extract_rules_mk(info_data):
  226. """Pull some keyboard information from existing rules.mk files
  227. """
  228. rules = rules_mk(info_data['keyboard_folder'])
  229. info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4'))
  230. if info_data['processor'] in CHIBIOS_PROCESSORS:
  231. arm_processor_rules(info_data, rules)
  232. elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS:
  233. avr_processor_rules(info_data, rules)
  234. else:
  235. cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor']))
  236. unknown_processor_rules(info_data, rules)
  237. # Pull in data from the json map
  238. dotty_info = dotty(info_data)
  239. info_rules_map = _json_load(Path('data/mappings/info_rules.json'))
  240. for rules_key, info_dict in info_rules_map.items():
  241. info_key = info_dict['info_key']
  242. key_type = info_dict.get('value_type', 'str')
  243. try:
  244. if rules_key in rules and info_dict.get('to_json', True):
  245. if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
  246. _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
  247. if key_type.startswith('array'):
  248. if '.' in key_type:
  249. key_type, array_type = key_type.split('.', 1)
  250. else:
  251. array_type = None
  252. rules_value = rules[rules_key].replace('{', '').replace('}', '').strip()
  253. if array_type == 'int':
  254. dotty_info[info_key] = list(map(int, rules_value.split(',')))
  255. else:
  256. dotty_info[info_key] = rules_value.split(',')
  257. elif key_type == 'list':
  258. dotty_info[info_key] = rules[rules_key].split()
  259. elif key_type == 'bool':
  260. dotty_info[info_key] = rules[rules_key] in true_values
  261. elif key_type == 'hex':
  262. dotty_info[info_key] = '0x' + rules[rules_key][2:].upper()
  263. elif key_type == 'int':
  264. dotty_info[info_key] = int(rules[rules_key])
  265. else:
  266. dotty_info[info_key] = rules[rules_key]
  267. except Exception as e:
  268. _log_warning(info_data, f'{rules_key}->{info_key}: {e}')
  269. info_data.update(dotty_info)
  270. # Merge in config values that can't be easily mapped
  271. _extract_features(info_data, rules)
  272. return info_data
  273. def _merge_layouts(info_data, new_info_data):
  274. """Merge new_info_data into info_data in an intelligent way.
  275. """
  276. for layout_name, layout_json in new_info_data['layouts'].items():
  277. if layout_name in info_data['layouts']:
  278. # Pull in layouts we have a macro for
  279. if len(info_data['layouts'][layout_name]['layout']) != len(layout_json['layout']):
  280. msg = '%s: %s: Number of elements in info.json does not match! info.json:%s != %s:%s'
  281. _log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout_json['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
  282. else:
  283. for i, key in enumerate(info_data['layouts'][layout_name]['layout']):
  284. key.update(layout_json['layout'][i])
  285. else:
  286. # Pull in layouts that have matrix data
  287. missing_matrix = False
  288. for key in layout_json.get('layout', {}):
  289. if 'matrix' not in key:
  290. missing_matrix = True
  291. if not missing_matrix:
  292. if layout_name in info_data['layouts']:
  293. # Update an existing layout with new data
  294. for i, key in enumerate(info_data['layouts'][layout_name]['layout']):
  295. key.update(layout_json['layout'][i])
  296. else:
  297. # Copy in the new layout wholesale
  298. layout_json['c_macro'] = False
  299. info_data['layouts'][layout_name] = layout_json
  300. return info_data
  301. def _search_keyboard_h(path):
  302. current_path = Path('keyboards/')
  303. aliases = {}
  304. layouts = {}
  305. for directory in path.parts:
  306. current_path = current_path / directory
  307. keyboard_h = '%s.h' % (directory,)
  308. keyboard_h_path = current_path / keyboard_h
  309. if keyboard_h_path.exists():
  310. new_layouts, new_aliases = find_layouts(keyboard_h_path)
  311. layouts.update(new_layouts)
  312. for alias, alias_text in new_aliases.items():
  313. if alias_text in layouts:
  314. aliases[alias] = alias_text
  315. return layouts, aliases
  316. def _find_all_layouts(info_data, keyboard):
  317. """Looks for layout macros associated with this keyboard.
  318. """
  319. layouts, aliases = _search_keyboard_h(Path(keyboard))
  320. if not layouts:
  321. # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above.
  322. info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard))
  323. for file in glob('keyboards/%s/*.h' % keyboard):
  324. if file.endswith('.h'):
  325. these_layouts, these_aliases = find_layouts(file)
  326. if these_layouts:
  327. layouts.update(these_layouts)
  328. for alias, alias_text in these_aliases.items():
  329. if alias_text in layouts:
  330. aliases[alias] = alias_text
  331. return layouts, aliases
  332. def _log_error(info_data, message):
  333. """Send an error message to both JSON and the log.
  334. """
  335. info_data['parse_errors'].append(message)
  336. cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
  337. def _log_warning(info_data, message):
  338. """Send a warning message to both JSON and the log.
  339. """
  340. info_data['parse_warnings'].append(message)
  341. cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
  342. def arm_processor_rules(info_data, rules):
  343. """Setup the default info for an ARM board.
  344. """
  345. info_data['processor_type'] = 'arm'
  346. info_data['protocol'] = 'ChibiOS'
  347. if 'bootloader' not in info_data:
  348. if 'STM32' in info_data['processor']:
  349. info_data['bootloader'] = 'stm32-dfu'
  350. else:
  351. info_data['bootloader'] = 'unknown'
  352. if 'STM32' in info_data['processor']:
  353. info_data['platform'] = 'STM32'
  354. elif 'MCU_SERIES' in rules:
  355. info_data['platform'] = rules['MCU_SERIES']
  356. elif 'ARM_ATSAM' in rules:
  357. info_data['platform'] = 'ARM_ATSAM'
  358. return info_data
  359. def avr_processor_rules(info_data, rules):
  360. """Setup the default info for an AVR board.
  361. """
  362. info_data['processor_type'] = 'avr'
  363. info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
  364. info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA'
  365. if 'bootloader' not in info_data:
  366. info_data['bootloader'] = 'atmel-dfu'
  367. # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
  368. # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA'
  369. return info_data
  370. def unknown_processor_rules(info_data, rules):
  371. """Setup the default keyboard info for unknown boards.
  372. """
  373. info_data['bootloader'] = 'unknown'
  374. info_data['platform'] = 'unknown'
  375. info_data['processor'] = 'unknown'
  376. info_data['processor_type'] = 'unknown'
  377. info_data['protocol'] = 'unknown'
  378. return info_data
  379. def deep_update(origdict, newdict):
  380. """Update a dictionary in place, recursing to do a deep copy.
  381. """
  382. for key, value in newdict.items():
  383. if isinstance(value, Mapping):
  384. origdict[key] = deep_update(origdict.get(key, {}), value)
  385. else:
  386. origdict[key] = value
  387. return origdict
  388. def merge_info_jsons(keyboard, info_data):
  389. """Return a merged copy of all the info.json files for a keyboard.
  390. """
  391. for info_file in find_info_json(keyboard):
  392. # Load and validate the JSON data
  393. new_info_data = _json_load(info_file)
  394. if not isinstance(new_info_data, dict):
  395. _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
  396. continue
  397. try:
  398. keyboard_validate(new_info_data)
  399. except jsonschema.ValidationError as e:
  400. json_path = '.'.join([str(p) for p in e.absolute_path])
  401. cli.log.error('Not including data from file: %s', info_file)
  402. cli.log.error('\t%s: %s', json_path, e.message)
  403. continue
  404. # Merge layout data in
  405. for layout_name, layout in new_info_data.get('layouts', {}).items():
  406. if layout_name in info_data['layouts']:
  407. for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):
  408. existing_key.update(new_key)
  409. else:
  410. layout['c_macro'] = False
  411. info_data['layouts'][layout_name] = layout
  412. # Update info_data with the new data
  413. if 'layouts' in new_info_data:
  414. del (new_info_data['layouts'])
  415. deep_update(info_data, new_info_data)
  416. return info_data
  417. def find_info_json(keyboard):
  418. """Finds all the info.json files associated with a keyboard.
  419. """
  420. # Find the most specific first
  421. base_path = Path('keyboards')
  422. keyboard_path = base_path / keyboard
  423. keyboard_parent = keyboard_path.parent
  424. info_jsons = [keyboard_path / 'info.json']
  425. # Add DEFAULT_FOLDER before parents, if present
  426. rules = rules_mk(keyboard)
  427. if 'DEFAULT_FOLDER' in rules:
  428. info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json')
  429. # Add in parent folders for least specific
  430. for _ in range(5):
  431. info_jsons.append(keyboard_parent / 'info.json')
  432. if keyboard_parent.parent == base_path:
  433. break
  434. keyboard_parent = keyboard_parent.parent
  435. # Return a list of the info.json files that actually exist
  436. return [info_json for info_json in info_jsons if info_json.exists()]