Optimize our jsonschema by using refs (#13271)

* fix some broken info.json files

* optimize our jsonschema using refs

* fix formatting after vscode broke it

* make flake8 happy

* cleanup

* make our schema validation more compact and flexible

data/schemas/api_keyboard.jsonschema

@@ -1,34 +1,22 @@
 {
+    "$id": "qmk.api.keyboard.v1",
     "allOf": [
-        { "$ref": "qmk.keyboard.v1" },
+        {"$ref": "qmk.keyboard.v1"},
         {
-            "$id": "qmk.api.keyboard.v1",
-            "keymaps": {
-                "type": "string"
-            },
-            "parse_errors": {
-                "type": "array",
-                "items": {
-                    "type": "string"
-                }
-            },
-            "parse_warnings": {
-                "type": "array",
-                "items": {
-                    "type": "string"
-                }
-            },
-            "processor_type": {
-                "type": "string"
-            },
-            "protocol": {
-                "type": "string"
-            },
-            "keyboard_folder": {
-                "type": "string"
-            },
-            "platform": {
-                "type": "string"
+            "properties": {
+                "keymaps": {
+                    "type": "object",
+                    "properties": {
+                        "url": {"type": "string"}
+                    }
+
+                },
+                "parse_errors": {"$ref": "qmk.definitions.v1#/string_array"},
+                "parse_warnings": {"$ref": "qmk.definitions.v1#/string_array"},
+                "processor_type": {"type": "string"},
+                "protocol": {"type": "string"},
+                "keyboard_folder": {"type": "string"},
+                "platform": {"type": "string"}
             }
         }
     ]

data/schemas/definitions.jsonschema

@@ -0,0 +1,107 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "qmk.definitions.v1",
+    "title": "Common definitions used across QMK's jsonschemas.",
+    "type": "object",
+    "boolean_array": {
+        "type": "object",
+        "additionalProperties": {"type": "boolean"}
+    },
+    "filename": {
+        "type": "string",
+        "minLength": 1,
+        "pattern": "^[0-9a-z_]*$"
+    },
+    "hex_number_2d": {
+        "type": "string",
+        "pattern": "^0x[0-9A-F]{2}$"
+    },
+    "hex_number_4d": {
+        "type": "string",
+        "pattern": "^0x[0-9A-F]{4}$"
+    },
+    "text_identifier": {
+        "type": "string",
+        "minLength": 1,
+        "maxLength": 250
+    },
+    "layout_macro": {
+        "oneOf": [
+            {
+                "type": "string",
+                "enum": ["LAYOUT", "LAYOUT_planck_1x2uC"]
+            },
+            {
+                "type": "string",
+                "pattern": "^LAYOUT_[0-9a-z_]*$"
+            }
+        ]
+    },
+    "key_unit": {
+        "type": "number",
+        "min": 0.25
+    },
+    "mcu_pin_array": {
+        "type": "array",
+        "items": {"$ref": "#/mcu_pin"}
+    },
+    "mcu_pin": {
+        "oneOf": [
+            {
+                "type": "string",
+                "pattern": "^[A-K]\\d{1,2}$"
+            },
+            {
+                "type": "string",
+                "pattern": "^LINE_PIN\\d{1,2}$"
+            },
+            {
+                "type": "number",
+                "multipleOf": 1
+            },
+            {
+                "type": "null"
+            }
+        ]
+    },
+    "signed_decimal": {
+        "type": "number"
+    },
+    "signed_int": {
+        "type": "number",
+        "multipleOf": 1
+    }
+    "signed_int_8": {
+        "type": "number",
+        "min": -127,
+        "max": 127,
+        "multipleOf": 1
+    }
+    "string_array": {
+        "type": "array",
+        "items": {
+            "type": "string"
+        }
+    },
+    "string_object": {
+        "type": "object",
+        "additionalProperties": {
+            "type": "string"
+        }
+    },
+    "unsigned_decimal": {
+        "type": "number",
+        "min": 0
+    },
+    "unsigned_int": {
+        "type": "number",
+        "min": 0,
+        "multipleOf": 1
+    }
+    "unsigned_int_8": {
+        "type": "number",
+        "min": 0,
+        "max": 255,
+        "multipleOf": 1
+    }
+}

data/schemas/keyboard.jsonschema

@@ -1,24 +1,12 @@
 {
-    "$schema": "http://json-schema.org/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "$id": "qmk.keyboard.v1",
     "title": "Keyboard Information",
     "type": "object",
     "properties": {
-        "keyboard_name": {
-            "type": "string",
-            "minLength": 2,
-            "maxLength": 250
-        },
-        "maintainer": {
-            "type": "string",
-            "minLength": 2,
-            "maxLength": 250
-        },
-        "manufacturer": {
-            "type": "string",
-            "minLength": 2,
-            "maxLength": 250
-        },
+        "keyboard_name": {"$ref": "qmk.definitions.v1#/text_identifier"},
+        "maintainer": {"$ref": "qmk.definitions.v1#/text_identifier"},
+        "manufacturer": {"$ref": "qmk.definitions.v1#/text_identifier"},
         "url": {
             "type": "string",
             "format": "uri"
@@ -40,62 +28,25 @@
             "type": "string",
             "enum": ["COL2ROW", "ROW2COL"]
         },
-        "debounce": {
-            "type": "number",
-            "min": 0,
-            "multipleOf": 1
-        },
-        "height": {
-            "type": "number",
-            "min": 0.25
-        },
-        "width": {
-            "type": "number",
-            "min": 0.25
-        },
+        "debounce": {"$ref": "qmk.definitions.v1#/unsigned_int"},
+        "height": {"$ref": "qmk.definitions.v1#/key_unit"},
+        "width": {"$ref": "qmk.definitions.v1#/key_unit"},
         "community_layouts": {
             "type": "array",
-            "items": {
-                "type": "string",
-                "minLength": 2,
-                "pattern": "^[0-9a-z_]*$"
-            }
-        },
-        "features": {
-            "type": "object",
-            "additionalProperties": {"type": "boolean"}
+            "items": {"$ref": "qmk.definitions.v1#/filename"}
         },
+        "features": {"$ref": "qmk.definitions.v1#/boolean_array"},
         "indicators": {
             "type": "object",
             "properties": {
-                "caps_lock": {
-                    "type": "string",
-                    "pattern": "^[A-K]\\d{1,2}$"
-                },
-                "num_lock": {
-                    "type": "string",
-                    "pattern": "^[A-K]\\d{1,2}$"
-                },
-                "scroll_lock": {
-                    "type": "string",
-                    "pattern": "^[A-K]\\d{1,2}$"
-                }
+                "caps_lock": {"$ref": "qmk.definitions.v1#/mcu_pin"},
+                "num_lock": {"$ref": "qmk.definitions.v1#/mcu_pin"},
+                "scroll_lock": {"$ref": "qmk.definitions.v1#/mcu_pin"}
             }
         },
         "layout_aliases": {
             "type": "object",
-            "additionalProperties": {
-                "oneOf": [
-                    {
-                        "type": "string",
-                        "enum": ["LAYOUT", "LAYOUT_planck_1x2uC"]
-                    },
-                    {
-                        "type": "string",
-                        "pattern": "^LAYOUT_[0-9a-z_]*$"
-                    }
-                ]
-            }
+            "additionalProperties": {"$ref": "qmk.definitions.v1#/layout_macro"}
         },
         "layouts": {
             "type": "object",
@@ -109,11 +60,7 @@
                     "c_macro": {
                         "type": "boolean"
                     },
-                    "key_count": {
-                        "type": "number",
-                        "min": 0,
-                        "multipleOf": 1
-                    },
+                    "key_count": {"$ref": "qmk.definitions.v1#/key_unit"},
                     "layout": {
                         "type": "array",
                         "items": {
@@ -131,34 +78,14 @@
                                         "multipleOf": 1
                                     }
                                 },
-                                "h": {
-                                    "type": "number",
-                                    "min": 0.25
-                                },
-                                "r": {
-                                    "type": "number",
-                                    "min": 0
-                                },
-                                "rx": {
-                                    "type": "number",
-                                    "min": 0
-                                },
-                                "ry": {
-                                    "type": "number",
-                                    "min": 0
-                                },
-                                "w": {
-                                    "type": "number",
-                                    "min": 0.25
-                                },
-                                "x": {
-                                    "type": "number",
-                                    "min": 0
-                                },
-                                "y": {
-                                    "type": "number",
-                                    "min": 0
-                                }
+                                "key_count": {"$ref": "qmk.definitions.v1#/key_unit"},
+                                "r": {"$ref": "qmk.definitions.v1#/unsigned_decimal"},
+                                "rx": {"$ref": "qmk.definitions.v1#/unsigned_decimal"},
+                                "ry": {"$ref": "qmk.definitions.v1#/unsigned_decimal"},
+                                "h": {"$ref": "qmk.definitions.v1#/key_unit"},
+                                "w": {"$ref": "qmk.definitions.v1#/key_unit"},
+                                "x": {"$ref": "qmk.definitions.v1#/key_unit"},
+                                "y": {"$ref": "qmk.definitions.v1#/key_unit"}
                             }
                         }
                     }
@@ -171,73 +98,10 @@
             "properties": {
                 "direct": {
                     "type": "array",
-                    "items": {
-                        "type": "array",
-                        "items": {
-                            "oneOf": [
-                                {
-                                    "type": "string",
-                                    "pattern": "^[A-K]\\d{1,2}$"
-                                },
-                                {
-                                    "type": "string",
-                                    "pattern": "^LINE_PIN\\d{1,2}$"
-                                },
-                                {
-                                    "type": "number",
-                                    "multipleOf": 1
-                                },
-                                {
-                                    "type": "null"
-                                }
-                            ]
-                        }
-                    }
+                    "items": {$ref": "qmk.definitions.v1#/mcu_pin_array"}
                 },
-                "cols": {
-                    "type": "array",
-                    "items": {
-                        "oneOf": [
-                            {
-                                "type": "string",
-                                "pattern": "^[A-K]\\d{1,2}$"
-                            },
-                            {
-                                "type": "string",
-                                "pattern": "^LINE_PIN\\d{1,2}$"
-                            },
-                            {
-                                "type": "number",
-                                "multipleOf": 1
-                            },
-                            {
-                                "type": "null"
-                            }
-                        ]
-                    }
-                },
-                "rows": {
-                    "type": "array",
-                    "items": {
-                        "oneOf": [
-                            {
-                                "type": "string",
-                                "pattern": "^[A-K]\\d{1,2}$"
-                            },
-                            {
-                                "type": "string",
-                                "pattern": "^LINE_PIN\\d{1,2}$"
-                            },
-                            {
-                                "type": "number",
-                                "multipleOf": 1
-                            },
-                            {
-                                "type": "null"
-                            }
-                        ]
-                    }
-                }
+                "cols": {"$ref": "qmk.definitions.v1#/mcu_pin_array"},
+                "rows": {"$ref": "qmk.definitions.v1#/mcu_pin_array"}
             }
         },
         "rgblight": {
@@ -250,47 +114,19 @@
                         "type": "boolean"
                     }
                 },
-                "brightness_steps": {
-                    "type": "number",
-                    "min": 0,
-                    "multipleOf": 1
-                },
-                "hue_steps": {
-                    "type": "number",
-                    "min": 0,
-                    "multipleOf": 1
-                },
-                "led_count": {
-                    "type": "number",
-                    "min": 0,
-                    "multipleOf": 1
-                },
-                "max_brightness": {
-                    "type": "number",
-                    "min": 0,
-                    "max": 255,
-                    "multipleOf": 1
-                },
-                "pin": {
-                    "type": "string",
-                    "pattern": "^([A-K]\\d{1,2}|LINE_PIN\\d{1,2})$"
-                },
-                "saturation_steps": {
-                    "type": "number",
-                    "min": 0,
-                    "multipleOf": 1
-                },
+                "brightness_steps": {"$ref": "qmk.definitions.v1#/unsigned_int"},
+                "hue_steps": {"$ref": "qmk.definitions.v1#/unsigned_int"},
+                "led_count": {"$ref": "qmk.definitions.v1#/unsigned_int"},
+                "max_brightness": {"$ref": "qmk.definitions.v1#/unsigned_int_8"},
+                "pin": {"$ref": "qmk.definitions.v1#/mcu_pin"},
+                "saturation_steps": {"$ref": "qmk.definitions.v1#/unsigned_int"},
                 "sleep": {"type": "boolean"},
                 "split": {"type": "boolean"},
                 "split_count": {
                     "type": "array",
                     "minLength": 2,
                     "maxLength": 2,
-                    "items": {
-                        "type": "number",
-                        "min": 0,
-                        "multipleOf": 1
-                    }
+                    "items": {"$ref": "qmk.definitions.v1#/unsigned_int"}
                 }
             }
         },
@@ -298,40 +134,19 @@
             "type": "object",
             "additionalProperties": false,
             "properties": {
-                "device_ver": {
-                    "type": "string",
-                    "pattern": "^[0-9A-F]x[0-9A-F][0-9A-F][0-9A-F][0-9A-F]"
-                },
-                "pid": {
-                    "type": "string",
-                    "pattern": "^[0-9A-F]x[0-9A-F][0-9A-F][0-9A-F][0-9A-F]"
-                },
-                "vid": {
-                    "type": "string",
-                    "pattern": "^[0-9A-F]x[0-9A-F][0-9A-F][0-9A-F][0-9A-F]"
-                }
+                "device_ver": {"$ref": "qmk.definitions.v1#/hex_number_4d"},
+                "pid": {"$ref": "qmk.definitions.v1#/hex_number_4d"},
+                "vid": {"$ref": "qmk.definitions.v1#/hex_number_4d"}
             }
         },
         "qmk_lufa_bootloader": {
             "type": "object",
             "additionalProperties": false,
             "properties": {
-                "esc_output": {
-                    "type": "string",
-                    "pattern": "^[A-K]\\d{1,2}$"
-                },
-                "esc_input": {
-                    "type": "string",
-                    "pattern": "^[A-K]\\d{1,2}$"
-                },
-                "led": {
-                    "type": "string",
-                    "pattern": "^[A-K]\\d{1,2}$"
-                },
-                "speaker": {
-                    "type": "string",
-                    "pattern": "^[A-K]\\d{1,2}$"
-                }
+                "esc_output": {"$ref": "qmk.definitions.v1#/mcu_pin"},
+                "esc_input": {"$ref": "qmk.definitions.v1#/mcu_pin"},
+                "led": {"$ref": "qmk.definitions.v1#/mcu_pin"},
+                "speaker": {"$ref": "qmk.definitions.v1#/mcu_pin"}
             }
         }
     }

keyboards/ramonimbao/chevron/info.json

@@ -1,7 +1,5 @@
 {
     "keyboard_name": "Chevron",
-    "url": "",
-    "maintainer": "",
     "width": 14.5,
     "height": 5,
     "layouts": {

keyboards/sendyyeah/75pixels/info.json

@@ -7,7 +7,7 @@
     "layouts": {
         "LAYOUT_ortho_5x15": {
             "layout": [
-                {"label": "Esc", "X": 0, "y": 0},
+                {"label": "Esc", "x": 0, "y": 0},
                 {"label": "1", "x": 1, "y": 0},
                 {"label": "2", "x": 2, "y": 0},
                 {"label": "3", "x": 3, "y": 0},
@@ -22,21 +22,21 @@
                 {"label": "NumLock", "x": 12, "y": 0},
                 {"label": "/", "x": 13, "y": 0},
                 {"label": "*", "x": 14, "y": 0},
-                {"label": "Tab", "X": 0, "y": 1},
-                {"label": "Q", "X": 1, "y": 1},
-                {"label": "W", "X": 2, "y": 1},
-                {"label": "E", "X": 3, "y": 1},
-                {"label": "R", "X": 4, "y": 1},
-                {"label": "T", "X": 5, "y": 1},
-                {"label": "Y", "X": 6, "y": 1},
-                {"label": "U", "X": 7, "y": 1},
-                {"label": "I", "X": 8, "y": 1},
-                {"label": "O", "X": 9, "y": 1},
-                {"label": "P", "X": 10, "y": 1},
-                {"label": "|\n\\", "X": 11, "y": 1},
-                {"label": "7\nHome", "X": 12, "y": 1},
-                {"label": "8\nUp", "X": 13, "y": 1},
-                {"label": "9\nPgUp", "X": 14, "y": 1},
+                {"label": "Tab", "x": 0, "y": 1},
+                {"label": "Q", "x": 1, "y": 1},
+                {"label": "W", "x": 2, "y": 1},
+                {"label": "E", "x": 3, "y": 1},
+                {"label": "R", "x": 4, "y": 1},
+                {"label": "T", "x": 5, "y": 1},
+                {"label": "Y", "x": 6, "y": 1},
+                {"label": "U", "x": 7, "y": 1},
+                {"label": "I", "x": 8, "y": 1},
+                {"label": "O", "x": 9, "y": 1},
+                {"label": "P", "x": 10, "y": 1},
+                {"label": "|\n\\", "x": 11, "y": 1},
+                {"label": "7\nHome", "x": 12, "y": 1},
+                {"label": "8\nUp", "x": 13, "y": 1},
+                {"label": "9\nPgUp", "x": 14, "y": 1},
                 {"label": "Caps", "x": 0, "y": 2},
                 {"label": "A", "x": 1, "y": 2},
                 {"label": "S", "x": 2, "y": 2},
@@ -67,21 +67,21 @@
                 {"label": "1\nEnd", "x": 12, "y": 3},
                 {"label": "2\nDown", "x": 13, "y": 3},
                 {"label": "3\nPgDn", "x": 14, "y": 3},
-                {"label": "Ctrl", "X": 0, "y": 4},
-                {"label": "Win", "X": 1, "y": 4},
-                {"label": "Alt", "X": 2, "y": 4},
-                {"label": "Fn", "X": 3, "y": 4},
-                {"label": "Lower", "X": 4, "y": 4},
-                {"label": "Space", "X": 5, "y": 4},
-                {"label": "Space", "X": 6, "y": 4},
-                {"label": "Raise", "X": 7, "y": 4},
-                {"label": "Alt", "X": 8, "y": 4},
-                {"label": "Win", "X": 9, "y": 4},
-                {"label": "Menu", "X": 10, "y": 4},
-                {"label": "Ctrl", "X": 11, "y": 4},
-                {"label": "0\nIns", "X": 12, "y": 4},
-                {"label": ".\nDel", "X": 13, "y": 4},
-                {"label": "Enter", "X": 14, "y": 4}
+                {"label": "Ctrl", "x": 0, "y": 4},
+                {"label": "Win", "x": 1, "y": 4},
+                {"label": "Alt", "x": 2, "y": 4},
+                {"label": "Fn", "x": 3, "y": 4},
+                {"label": "Lower", "x": 4, "y": 4},
+                {"label": "Space", "x": 5, "y": 4},
+                {"label": "Space", "x": 6, "y": 4},
+                {"label": "Raise", "x": 7, "y": 4},
+                {"label": "Alt", "x": 8, "y": 4},
+                {"label": "Win", "x": 9, "y": 4},
+                {"label": "Menu", "x": 10, "y": 4},
+                {"label": "Ctrl", "x": 11, "y": 4},
+                {"label": "0\nIns", "x": 12, "y": 4},
+                {"label": ".\nDel", "x": 13, "y": 4},
+                {"label": "Enter", "x": 14, "y": 4}
             ]
         }
     }

lib/python/qmk/cli/format/json.py

@@ -8,7 +8,7 @@ from jsonschema import ValidationError
 from milc import cli
 
 from qmk.info import info_json
-from qmk.json_schema import json_load, keyboard_validate
+from qmk.json_schema import json_load, validate
 from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder
 from qmk.path import normpath
 
@@ -23,14 +23,13 @@ def format_json(cli):
 
     if cli.args.format == 'auto':
         try:
-            keyboard_validate(json_file)
+            validate(json_file, 'qmk.keyboard.v1')
             json_encoder = InfoJSONEncoder
 
         except ValidationError as e:
             cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e)
             cli.log.info('Treating %s as a keymap file.', cli.args.json_file)
             json_encoder = KeymapJSONEncoder
-
     elif cli.args.format == 'keyboard':
         json_encoder = InfoJSONEncoder
     elif cli.args.format == 'keymap':

lib/python/qmk/info.py

@@ -9,7 +9,7 @@ from milc import cli
 
 from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
 from qmk.c_parse import find_layouts
-from qmk.json_schema import deep_update, json_load, keyboard_validate, keyboard_api_validate
+from qmk.json_schema import deep_update, json_load, validate
 from qmk.keyboard import config_h, rules_mk
 from qmk.keymap import list_keymaps
 from qmk.makefile import parse_rules_mk_file
@@ -66,7 +66,7 @@ def info_json(keyboard):
 
     # Validate against the jsonschema
     try:
-        keyboard_api_validate(info_data)
+        validate(info_data, 'qmk.api.keyboard.v1')
 
     except jsonschema.ValidationError as e:
         json_path = '.'.join([str(p) for p in e.absolute_path])
@@ -490,7 +490,7 @@ def merge_info_jsons(keyboard, info_data):
             continue
 
         try:
-            keyboard_validate(new_info_data)
+            validate(new_info_data, 'qmk.keyboard.v1')
         except jsonschema.ValidationError as e:
             json_path = '.'.join([str(p) for p in e.absolute_path])
             cli.log.error('Not including data from file: %s', info_file)

lib/python/qmk/json_schema.py

@@ -24,9 +24,10 @@ def json_load(json_file):
 
 def load_jsonschema(schema_name):
     """Read a jsonschema file from disk.
-
-    FIXME(skullydazed/anyone): Refactor to make this a public function.
     """
+    if Path(schema_name).exists():
+        return json_load(schema_name)
+
     schema_path = Path(f'data/schemas/{schema_name}.jsonschema')
 
     if not schema_path.exists():
@@ -35,28 +36,33 @@ def load_jsonschema(schema_name):
     return json_load(schema_path)
 
 
-def keyboard_validate(data):
-    """Validates data against the keyboard jsonschema.
+def create_validator(schema):
+    """Creates a validator for the given schema id.
     """
-    schema = load_jsonschema('keyboard')
-    validator = jsonschema.Draft7Validator(schema).validate
+    schema_store = {}
 
-    return validator(data)
+    for schema_file in Path('data/schemas').glob('*.jsonschema'):
+        schema_data = load_jsonschema(schema_file)
+        if not isinstance(schema_data, dict):
+            cli.log.debug('Skipping schema file %s', schema_file)
+            continue
+        schema_store[schema_data['$id']] = schema_data
+
+    resolver = jsonschema.RefResolver.from_schema(schema_store['qmk.keyboard.v1'], store=schema_store)
+
+    return jsonschema.Draft7Validator(schema_store[schema], resolver=resolver).validate
 
 
-def keyboard_api_validate(data):
-    """Validates data against the api_keyboard jsonschema.
+def validate(data, schema):
+    """Validates data against a schema.
     """
-    base = load_jsonschema('keyboard')
-    relative = load_jsonschema('api_keyboard')
-    resolver = jsonschema.RefResolver.from_schema(base)
-    validator = jsonschema.Draft7Validator(relative, resolver=resolver).validate
+    validator = create_validator(schema)
 
     return validator(data)
 
 
 def deep_update(origdict, newdict):
-    """Update a dictionary in place, recursing to do a deep copy.
+    """Update a dictionary in place, recursing to do a depth-first deep copy.
     """
     for key, value in newdict.items():
         if isinstance(value, Mapping):