info.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548
  1. """Functions that help us generate and use info.json files.
  2. """
  3. from glob import glob
  4. from pathlib import Path
  5. import jsonschema
  6. from dotty_dict import dotty
  7. from milc import cli
  8. from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
  9. from qmk.c_parse import find_layouts
  10. from qmk.json_schema import deep_update, json_load, validate
  11. from qmk.keyboard import config_h, rules_mk
  12. from qmk.keymap import list_keymaps
  13. from qmk.makefile import parse_rules_mk_file
  14. from qmk.math import compute
  15. true_values = ['1', 'on', 'yes']
  16. false_values = ['0', 'off', 'no']
  17. def _valid_community_layout(layout):
  18. """Validate that a declared community list exists
  19. """
  20. return (Path('layouts/default') / layout).exists()
  21. def info_json(keyboard):
  22. """Generate the info.json data for a specific keyboard.
  23. """
  24. cur_dir = Path('keyboards')
  25. rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk')
  26. if 'DEFAULT_FOLDER' in rules:
  27. keyboard = rules['DEFAULT_FOLDER']
  28. rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules)
  29. info_data = {
  30. 'keyboard_name': str(keyboard),
  31. 'keyboard_folder': str(keyboard),
  32. 'keymaps': {},
  33. 'layouts': {},
  34. 'parse_errors': [],
  35. 'parse_warnings': [],
  36. 'maintainer': 'qmk',
  37. }
  38. # Populate the list of JSON keymaps
  39. for keymap in list_keymaps(keyboard, c=False, fullpath=True):
  40. info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'}
  41. # Populate layout data
  42. layouts, aliases = _find_all_layouts(info_data, keyboard)
  43. if aliases:
  44. info_data['layout_aliases'] = aliases
  45. for layout_name, layout_json in layouts.items():
  46. if not layout_name.startswith('LAYOUT_kc'):
  47. layout_json['c_macro'] = True
  48. info_data['layouts'][layout_name] = layout_json
  49. # Merge in the data from info.json, config.h, and rules.mk
  50. info_data = merge_info_jsons(keyboard, info_data)
  51. info_data = _extract_config_h(info_data)
  52. info_data = _extract_rules_mk(info_data)
  53. # Validate against the jsonschema
  54. try:
  55. validate(info_data, 'qmk.api.keyboard.v1')
  56. except jsonschema.ValidationError as e:
  57. json_path = '.'.join([str(p) for p in e.absolute_path])
  58. cli.log.error('Invalid API data: %s: %s: %s', keyboard, json_path, e.message)
  59. exit()
  60. # Make sure we have at least one layout
  61. if not info_data.get('layouts'):
  62. _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
  63. # Filter out any non-existing community layouts
  64. for layout in info_data.get('community_layouts', []):
  65. if not _valid_community_layout(layout):
  66. # Ignore layout from future checks
  67. info_data['community_layouts'].remove(layout)
  68. _log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout))
  69. # Make sure we supply layout macros for the community layouts we claim to support
  70. for layout in info_data.get('community_layouts', []):
  71. layout_name = 'LAYOUT_' + layout
  72. if layout_name not in info_data.get('layouts', {}) and layout_name not in info_data.get('layout_aliases', {}):
  73. _log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
  74. return info_data
  75. def _extract_features(info_data, rules):
  76. """Find all the features enabled in rules.mk.
  77. """
  78. # Special handling for bootmagic which also supports a "lite" mode.
  79. if rules.get('BOOTMAGIC_ENABLE') == 'lite':
  80. rules['BOOTMAGIC_LITE_ENABLE'] = 'on'
  81. del rules['BOOTMAGIC_ENABLE']
  82. if rules.get('BOOTMAGIC_ENABLE') == 'full':
  83. rules['BOOTMAGIC_ENABLE'] = 'on'
  84. # Skip non-boolean features we haven't implemented special handling for
  85. for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE':
  86. if rules.get(feature):
  87. del rules[feature]
  88. # Process the rest of the rules as booleans
  89. for key, value in rules.items():
  90. if key.endswith('_ENABLE'):
  91. key = '_'.join(key.split('_')[:-1]).lower()
  92. value = True if value.lower() in true_values else False if value.lower() in false_values else value
  93. if 'config_h_features' not in info_data:
  94. info_data['config_h_features'] = {}
  95. if 'features' not in info_data:
  96. info_data['features'] = {}
  97. if key in info_data['features']:
  98. _log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,))
  99. info_data['features'][key] = value
  100. info_data['config_h_features'][key] = value
  101. return info_data
  102. def _pin_name(pin):
  103. """Returns the proper representation for a pin.
  104. """
  105. pin = pin.strip()
  106. if not pin:
  107. return None
  108. elif pin.isdigit():
  109. return int(pin)
  110. elif pin == 'NO_PIN':
  111. return None
  112. return pin
  113. def _extract_pins(pins):
  114. """Returns a list of pins from a comma separated string of pins.
  115. """
  116. return [_pin_name(pin) for pin in pins.split(',')]
  117. def _extract_direct_matrix(info_data, direct_pins):
  118. """
  119. """
  120. info_data['matrix_pins'] = {}
  121. direct_pin_array = []
  122. while direct_pins[-1] != '}':
  123. direct_pins = direct_pins[:-1]
  124. for row in direct_pins.split('},{'):
  125. if row.startswith('{'):
  126. row = row[1:]
  127. if row.endswith('}'):
  128. row = row[:-1]
  129. direct_pin_array.append([])
  130. for pin in row.split(','):
  131. if pin == 'NO_PIN':
  132. pin = None
  133. direct_pin_array[-1].append(pin)
  134. return direct_pin_array
  135. def _extract_matrix_info(info_data, config_c):
  136. """Populate the matrix information.
  137. """
  138. row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip()
  139. col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip()
  140. direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1]
  141. if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c:
  142. if 'matrix_size' in info_data:
  143. _log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.')
  144. info_data['matrix_size'] = {
  145. 'cols': compute(config_c.get('MATRIX_COLS', '0')),
  146. 'rows': compute(config_c.get('MATRIX_ROWS', '0')),
  147. }
  148. if row_pins and col_pins:
  149. if 'matrix_pins' in info_data:
  150. _log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.')
  151. info_data['matrix_pins'] = {
  152. 'cols': _extract_pins(col_pins),
  153. 'rows': _extract_pins(row_pins),
  154. }
  155. if direct_pins:
  156. if 'matrix_pins' in info_data:
  157. _log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.')
  158. info_data['matrix_pins']['direct'] = _extract_direct_matrix(info_data, direct_pins)
  159. return info_data
  160. def _extract_config_h(info_data):
  161. """Pull some keyboard information from existing config.h files
  162. """
  163. config_c = config_h(info_data['keyboard_folder'])
  164. # Pull in data from the json map
  165. dotty_info = dotty(info_data)
  166. info_config_map = json_load(Path('data/mappings/info_config.json'))
  167. for config_key, info_dict in info_config_map.items():
  168. info_key = info_dict['info_key']
  169. key_type = info_dict.get('value_type', 'str')
  170. try:
  171. if config_key in config_c and info_dict.get('to_json', True):
  172. if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
  173. _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
  174. if key_type.startswith('array'):
  175. if '.' in key_type:
  176. key_type, array_type = key_type.split('.', 1)
  177. else:
  178. array_type = None
  179. config_value = config_c[config_key].replace('{', '').replace('}', '').strip()
  180. if array_type == 'int':
  181. dotty_info[info_key] = list(map(int, config_value.split(',')))
  182. else:
  183. dotty_info[info_key] = config_value.split(',')
  184. elif key_type == 'bool':
  185. dotty_info[info_key] = config_c[config_key] in true_values
  186. elif key_type == 'hex':
  187. dotty_info[info_key] = '0x' + config_c[config_key][2:].upper()
  188. elif key_type == 'list':
  189. dotty_info[info_key] = config_c[config_key].split()
  190. elif key_type == 'int':
  191. dotty_info[info_key] = int(config_c[config_key])
  192. else:
  193. dotty_info[info_key] = config_c[config_key]
  194. except Exception as e:
  195. _log_warning(info_data, f'{config_key}->{info_key}: {e}')
  196. info_data.update(dotty_info)
  197. # Pull data that easily can't be mapped in json
  198. _extract_matrix_info(info_data, config_c)
  199. return info_data
  200. def _extract_rules_mk(info_data):
  201. """Pull some keyboard information from existing rules.mk files
  202. """
  203. rules = rules_mk(info_data['keyboard_folder'])
  204. info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4'))
  205. if info_data['processor'] in CHIBIOS_PROCESSORS:
  206. arm_processor_rules(info_data, rules)
  207. elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS:
  208. avr_processor_rules(info_data, rules)
  209. else:
  210. cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor']))
  211. unknown_processor_rules(info_data, rules)
  212. # Pull in data from the json map
  213. dotty_info = dotty(info_data)
  214. info_rules_map = json_load(Path('data/mappings/info_rules.json'))
  215. for rules_key, info_dict in info_rules_map.items():
  216. info_key = info_dict['info_key']
  217. key_type = info_dict.get('value_type', 'str')
  218. try:
  219. if rules_key in rules and info_dict.get('to_json', True):
  220. if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
  221. _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
  222. if key_type.startswith('array'):
  223. if '.' in key_type:
  224. key_type, array_type = key_type.split('.', 1)
  225. else:
  226. array_type = None
  227. rules_value = rules[rules_key].replace('{', '').replace('}', '').strip()
  228. if array_type == 'int':
  229. dotty_info[info_key] = list(map(int, rules_value.split(',')))
  230. else:
  231. dotty_info[info_key] = rules_value.split(',')
  232. elif key_type == 'list':
  233. dotty_info[info_key] = rules[rules_key].split()
  234. elif key_type == 'bool':
  235. dotty_info[info_key] = rules[rules_key] in true_values
  236. elif key_type == 'hex':
  237. dotty_info[info_key] = '0x' + rules[rules_key][2:].upper()
  238. elif key_type == 'int':
  239. dotty_info[info_key] = int(rules[rules_key])
  240. else:
  241. dotty_info[info_key] = rules[rules_key]
  242. except Exception as e:
  243. _log_warning(info_data, f'{rules_key}->{info_key}: {e}')
  244. info_data.update(dotty_info)
  245. # Merge in config values that can't be easily mapped
  246. _extract_features(info_data, rules)
  247. return info_data
  248. def _merge_layouts(info_data, new_info_data):
  249. """Merge new_info_data into info_data in an intelligent way.
  250. """
  251. for layout_name, layout_json in new_info_data['layouts'].items():
  252. if layout_name in info_data['layouts']:
  253. # Pull in layouts we have a macro for
  254. if len(info_data['layouts'][layout_name]['layout']) != len(layout_json['layout']):
  255. msg = '%s: %s: Number of elements in info.json does not match! info.json:%s != %s:%s'
  256. _log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout_json['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
  257. else:
  258. for i, key in enumerate(info_data['layouts'][layout_name]['layout']):
  259. key.update(layout_json['layout'][i])
  260. else:
  261. # Pull in layouts that have matrix data
  262. missing_matrix = False
  263. for key in layout_json.get('layout', {}):
  264. if 'matrix' not in key:
  265. missing_matrix = True
  266. if not missing_matrix:
  267. if layout_name in info_data['layouts']:
  268. # Update an existing layout with new data
  269. for i, key in enumerate(info_data['layouts'][layout_name]['layout']):
  270. key.update(layout_json['layout'][i])
  271. else:
  272. # Copy in the new layout wholesale
  273. layout_json['c_macro'] = False
  274. info_data['layouts'][layout_name] = layout_json
  275. return info_data
  276. def _search_keyboard_h(path):
  277. current_path = Path('keyboards/')
  278. aliases = {}
  279. layouts = {}
  280. for directory in path.parts:
  281. current_path = current_path / directory
  282. keyboard_h = '%s.h' % (directory,)
  283. keyboard_h_path = current_path / keyboard_h
  284. if keyboard_h_path.exists():
  285. new_layouts, new_aliases = find_layouts(keyboard_h_path)
  286. layouts.update(new_layouts)
  287. for alias, alias_text in new_aliases.items():
  288. if alias_text in layouts:
  289. aliases[alias] = alias_text
  290. return layouts, aliases
  291. def _find_all_layouts(info_data, keyboard):
  292. """Looks for layout macros associated with this keyboard.
  293. """
  294. layouts, aliases = _search_keyboard_h(Path(keyboard))
  295. if not layouts:
  296. # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above.
  297. info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard))
  298. for file in glob('keyboards/%s/*.h' % keyboard):
  299. if file.endswith('.h'):
  300. these_layouts, these_aliases = find_layouts(file)
  301. if these_layouts:
  302. layouts.update(these_layouts)
  303. for alias, alias_text in these_aliases.items():
  304. if alias_text in layouts:
  305. aliases[alias] = alias_text
  306. return layouts, aliases
  307. def _log_error(info_data, message):
  308. """Send an error message to both JSON and the log.
  309. """
  310. info_data['parse_errors'].append(message)
  311. cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
  312. def _log_warning(info_data, message):
  313. """Send a warning message to both JSON and the log.
  314. """
  315. info_data['parse_warnings'].append(message)
  316. cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
  317. def arm_processor_rules(info_data, rules):
  318. """Setup the default info for an ARM board.
  319. """
  320. info_data['processor_type'] = 'arm'
  321. info_data['protocol'] = 'ChibiOS'
  322. if 'bootloader' not in info_data:
  323. if 'STM32' in info_data['processor']:
  324. info_data['bootloader'] = 'stm32-dfu'
  325. else:
  326. info_data['bootloader'] = 'unknown'
  327. if 'STM32' in info_data['processor']:
  328. info_data['platform'] = 'STM32'
  329. elif 'MCU_SERIES' in rules:
  330. info_data['platform'] = rules['MCU_SERIES']
  331. elif 'ARM_ATSAM' in rules:
  332. info_data['platform'] = 'ARM_ATSAM'
  333. return info_data
  334. def avr_processor_rules(info_data, rules):
  335. """Setup the default info for an AVR board.
  336. """
  337. info_data['processor_type'] = 'avr'
  338. info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
  339. info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA'
  340. if 'bootloader' not in info_data:
  341. info_data['bootloader'] = 'atmel-dfu'
  342. # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
  343. # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA'
  344. return info_data
  345. def unknown_processor_rules(info_data, rules):
  346. """Setup the default keyboard info for unknown boards.
  347. """
  348. info_data['bootloader'] = 'unknown'
  349. info_data['platform'] = 'unknown'
  350. info_data['processor'] = 'unknown'
  351. info_data['processor_type'] = 'unknown'
  352. info_data['protocol'] = 'unknown'
  353. return info_data
  354. def merge_info_jsons(keyboard, info_data):
  355. """Return a merged copy of all the info.json files for a keyboard.
  356. """
  357. for info_file in find_info_json(keyboard):
  358. # Load and validate the JSON data
  359. new_info_data = json_load(info_file)
  360. if not isinstance(new_info_data, dict):
  361. _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
  362. continue
  363. try:
  364. validate(new_info_data, 'qmk.keyboard.v1')
  365. except jsonschema.ValidationError as e:
  366. json_path = '.'.join([str(p) for p in e.absolute_path])
  367. cli.log.error('Not including data from file: %s', info_file)
  368. cli.log.error('\t%s: %s', json_path, e.message)
  369. continue
  370. # Merge layout data in
  371. if 'layout_aliases' in new_info_data:
  372. info_data['layout_aliases'] = {**info_data.get('layout_aliases', {}), **new_info_data['layout_aliases']}
  373. del new_info_data['layout_aliases']
  374. for layout_name, layout in new_info_data.get('layouts', {}).items():
  375. if layout_name in info_data.get('layout_aliases', {}):
  376. _log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}")
  377. layout_name = info_data['layout_aliases'][layout_name]
  378. if layout_name in info_data['layouts']:
  379. for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):
  380. existing_key.update(new_key)
  381. else:
  382. layout['c_macro'] = False
  383. info_data['layouts'][layout_name] = layout
  384. # Update info_data with the new data
  385. if 'layouts' in new_info_data:
  386. del new_info_data['layouts']
  387. deep_update(info_data, new_info_data)
  388. return info_data
  389. def find_info_json(keyboard):
  390. """Finds all the info.json files associated with a keyboard.
  391. """
  392. # Find the most specific first
  393. base_path = Path('keyboards')
  394. keyboard_path = base_path / keyboard
  395. keyboard_parent = keyboard_path.parent
  396. info_jsons = [keyboard_path / 'info.json']
  397. # Add DEFAULT_FOLDER before parents, if present
  398. rules = rules_mk(keyboard)
  399. if 'DEFAULT_FOLDER' in rules:
  400. info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json')
  401. # Add in parent folders for least specific
  402. for _ in range(5):
  403. info_jsons.append(keyboard_parent / 'info.json')
  404. if keyboard_parent.parent == base_path:
  405. break
  406. keyboard_parent = keyboard_parent.parent
  407. # Return a list of the info.json files that actually exist
  408. return [info_json for info_json in info_jsons if info_json.exists()]