From 106795069740bede91cb5fbc20cd506a48776653 Mon Sep 17 00:00:00 2001 From: Scott Lahteine Date: Tue, 17 Dec 2024 12:45:58 -0600 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A8=20Fix=20and=20improve=20schema=20e?= =?UTF-8?q?xports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- buildroot/share/PlatformIO/scripts/schema.py | 41 +++++++++++---- .../share/PlatformIO/scripts/signature.py | 50 +++++++++++++------ 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/buildroot/share/PlatformIO/scripts/schema.py b/buildroot/share/PlatformIO/scripts/schema.py index d5179d43b5..d23e5c4716 100755 --- a/buildroot/share/PlatformIO/scripts/schema.py +++ b/buildroot/share/PlatformIO/scripts/schema.py @@ -183,18 +183,28 @@ def extract_files(filekey): # - The line starts with '======' so just skip it. # def use_comment(c, opt, sec, bufref): - if c.startswith(':'): # If the comment starts with : then it has magic JSON - d = c[1:].strip() # Strip the leading : - cbr = c.rindex('}') if d.startswith('{') else c.rindex(']') if d.startswith('[') else 0 + ''' + c - The comment line to parse + opt - Options JSON string to return (if not updated) + sec - Section to return (if not updated) + bufref - The comment buffer to add to + ''' + sc = c.strip() # Strip for special patterns + if sc.startswith(':'): # If the comment starts with : then it has magic JSON + d = sc[1:].strip() # Strip the leading : and spaces + # Look for a JSON container + cbr = sc.rindex('}') if d.startswith('{') else sc.rindex(']') if d.startswith('[') else 0 if cbr: - opt, cmt = c[1:cbr+1].strip(), c[cbr+1:].strip() + opt, cmt = sc[1:cbr+1].strip(), sc[cbr+1:].strip() if cmt != '': bufref.append(cmt) else: - opt = c[1:].strip() - elif c.startswith('@section'): # Start a new section - sec = c[8:].strip() - elif not c.startswith('========'): - bufref.append(c) + opt = sc[1:].strip() # Some literal value not in a JSON container? + else: + m = re.match(r'@section\s*(.+)', sc) # Start a new section? + if m: + sec = m[1] + elif not sc.startswith('========'): + bufref.append(c) # Anything else is part of the comment return opt, sec # For slash comments, capture consecutive slash comments. @@ -225,7 +235,7 @@ def extract_files(filekey): # Collect temperature sensors if state == Parse.GET_SENSORS: - sens = re.match(r'^(-?\d+)\s*:\s*(.+)$', cline) + sens = re.match(r'^\s*(-?\d+)\s*:\s*(.+)$', cline) if sens: s2 = sens[2].replace("'", "''") options_json += f"{sens[1]}:'{sens[1]} - {s2}', " @@ -433,6 +443,17 @@ def dump_json(schema:dict, jpath:Path): def dump_yaml(schema:dict, ypath:Path): import yaml + + # Custom representer for all multi-line strings + def str_literal_representer(dumper, data): + if '\n' in data: # Check for multi-line strings + # Add a newline to trigger '|+' + if not data.endswith('\n'): data += '\n' + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + + yaml.add_representer(str, str_literal_representer) + with ypath.open('w', encoding='utf-8') as yfile: yaml.dump(schema, yfile, default_flow_style=False, width=120, indent=2) diff --git a/buildroot/share/PlatformIO/scripts/signature.py b/buildroot/share/PlatformIO/scripts/signature.py index efb8527869..f47d509bab 100755 --- a/buildroot/share/PlatformIO/scripts/signature.py +++ b/buildroot/share/PlatformIO/scripts/signature.py @@ -206,15 +206,21 @@ def compute_build_signature(env): print(red + "Error: " + str(exc)) conf_schema = None + optorder = ('MOTHERBOARD','SERIAL_PORT','BAUDRATE','USE_WATCHDOG','THERMAL_PROTECTION_HOTENDS','THERMAL_PROTECTION_HYSTERESIS','THERMAL_PROTECTION_PERIOD','BUFSIZE','BLOCK_BUFFER_SIZE','MAX_CMD_SIZE','EXTRUDERS','TEMP_SENSOR_0','TEMP_HYSTERESIS','HEATER_0_MINTEMP','HEATER_0_MAXTEMP','PREHEAT_1_TEMP_HOTEND','BANG_MAX','PIDTEMP','PID_K1','PID_MAX','PID_FUNCTIONAL_RANGE','DEFAULT_KP','DEFAULT_KI','DEFAULT_KD','X_DRIVER_TYPE','Y_DRIVER_TYPE','Z_DRIVER_TYPE','E0_DRIVER_TYPE','X_BED_SIZE','X_MIN_POS','X_MAX_POS','Y_BED_SIZE','Y_MIN_POS','Y_MAX_POS','Z_MIN_POS','Z_MAX_POS','X_HOME_DIR','Y_HOME_DIR','Z_HOME_DIR','X_MIN_ENDSTOP_HIT_STATE','Y_MIN_ENDSTOP_HIT_STATE','Z_MIN_ENDSTOP_HIT_STATE','DEFAULT_AXIS_STEPS_PER_UNIT','AXIS_RELATIVE_MODES','DEFAULT_MAX_FEEDRATE','DEFAULT_MAX_ACCELERATION','HOMING_FEEDRATE_MM_M','HOMING_BUMP_DIVISOR','X_ENABLE_ON','Y_ENABLE_ON','Z_ENABLE_ON','E_ENABLE_ON','INVERT_X_DIR','INVERT_Y_DIR','INVERT_Z_DIR','INVERT_E0_DIR','STEP_STATE_E','STEP_STATE_X','STEP_STATE_Y','STEP_STATE_Z','DISABLE_X','DISABLE_Y','DISABLE_Z','DISABLE_E','PROPORTIONAL_FONT_RATIO','DEFAULT_NOMINAL_FILAMENT_DIA','JUNCTION_DEVIATION_MM','DEFAULT_ACCELERATION','DEFAULT_TRAVEL_ACCELERATION','DEFAULT_RETRACT_ACCELERATION','DEFAULT_MINIMUMFEEDRATE','DEFAULT_MINTRAVELFEEDRATE','MINIMUM_PLANNER_SPEED','MIN_STEPS_PER_SEGMENT','DEFAULT_MINSEGMENTTIME','BED_OVERSHOOT','BUSY_WHILE_HEATING','DEFAULT_EJERK','DEFAULT_KEEPALIVE_INTERVAL','DEFAULT_LEVELING_FADE_HEIGHT','DISABLE_OTHER_EXTRUDERS','DISPLAY_CHARSET_HD44780','EEPROM_BOOT_SILENT','EEPROM_CHITCHAT','ENDSTOPPULLUPS','EXTRUDE_MAXLENGTH','EXTRUDE_MINTEMP','HOST_KEEPALIVE_FEATURE','HOTEND_OVERSHOOT','JD_HANDLE_SMALL_SEGMENTS','LCD_INFO_SCREEN_STYLE','LCD_LANGUAGE','MAX_BED_POWER','MESH_INSET','MIN_SOFTWARE_ENDSTOPS','MAX_SOFTWARE_ENDSTOPS','MIN_SOFTWARE_ENDSTOP_X','MIN_SOFTWARE_ENDSTOP_Y','MIN_SOFTWARE_ENDSTOP_Z','MAX_SOFTWARE_ENDSTOP_X','MAX_SOFTWARE_ENDSTOP_Y','MAX_SOFTWARE_ENDSTOP_Z','PREHEAT_1_FAN_SPEED','PREHEAT_1_LABEL','PREHEAT_1_TEMP_BED','PREVENT_COLD_EXTRUSION','PREVENT_LENGTHY_EXTRUDE','PRINTJOB_TIMER_AUTOSTART','PROBING_MARGIN','SHOW_BOOTSCREEN','SOFT_PWM_SCALE','STRING_CONFIG_H_AUTHOR','TEMP_BED_HYSTERESIS','TEMP_BED_RESIDENCY_TIME','TEMP_BED_WINDOW','TEMP_RESIDENCY_TIME','TEMP_WINDOW','VALIDATE_HOMING_ENDSTOPS','XY_PROBE_FEEDRATE','Z_CLEARANCE_BETWEEN_PROBES','Z_CLEARANCE_DEPLOY_PROBE','Z_CLEARANCE_MULTI_PROBE','ARC_SUPPORT','AUTO_REPORT_TEMPERATURES','AUTOTEMP','AUTOTEMP_OLDWEIGHT','BED_CHECK_INTERVAL','DEFAULT_STEPPER_TIMEOUT_SEC','DEFAULT_VOLUMETRIC_EXTRUDER_LIMIT','DISABLE_IDLE_X','DISABLE_IDLE_Y','DISABLE_IDLE_Z','DISABLE_IDLE_E','E0_AUTO_FAN_PIN','ENCODER_100X_STEPS_PER_SEC','ENCODER_10X_STEPS_PER_SEC','ENCODER_RATE_MULTIPLIER','EXTENDED_CAPABILITIES_REPORT','EXTRUDER_AUTO_FAN_SPEED','EXTRUDER_AUTO_FAN_TEMPERATURE','FANMUX0_PIN','FANMUX1_PIN','FANMUX2_PIN','FASTER_GCODE_PARSER','HOMING_BUMP_MM','MAX_ARC_SEGMENT_MM','MIN_ARC_SEGMENT_MM','MIN_CIRCLE_SEGMENTS','N_ARC_CORRECTION','SERIAL_OVERRUN_PROTECTION','SLOWDOWN','SLOWDOWN_DIVISOR','TEMP_SENSOR_BED','THERMAL_PROTECTION_BED_HYSTERESIS','THERMOCOUPLE_MAX_ERRORS','TX_BUFFER_SIZE','WATCH_BED_TEMP_INCREASE','WATCH_BED_TEMP_PERIOD','WATCH_TEMP_INCREASE','WATCH_TEMP_PERIOD') + + def optsort(x, optorder): + return optorder.index(x) if x in optorder else float('inf') + # - # CONFIG_EXPORT 2 = config.ini, 5 = Config.h + # CONFIG_EXPORT 102 = config.ini, 105 = Config.h # Get sections using the schema class # if extended_dump and config_dump in (2, 5): if not conf_schema: exit(1) # Start with a preferred @section ordering - preorder = ('info','user','machine','extruder','bed temp','fans','stepper drivers','geometry','homing','endstops','probes','lcd','interface','host','reporting') + preorder = ('test','custom','info','machine','eeprom','stepper drivers','multi stepper','idex','extruder','geometry','homing','kinematics','motion','motion control','endstops','filament runout sensors','probe type','probes','bltouch','leveling','temperature','hotend temp','mpctemp','pid temp','mpc temp','bed temp','chamber temp','fans','tool change','advanced pause','calibrate','calibration','media','lcd','lights','caselight','interface','custom main menu','custom config menu','custom buttons','develop','debug matrix','delta','scara','tpara','polar','polargraph','cnc','nozzle park','nozzle clean','gcode','serial','host','filament width','i2c encoders','i2cbus','joystick','multi-material','nanodlp','network','photo','power','psu control','reporting','safety','security','servos','stats','tmc/config','tmc/hybrid','tmc/serial','tmc/smart','tmc/spi','tmc/stallguard','tmc/status','tmc/stealthchop','tmc/tmc26x','units','volumetrics','extras') + sections = { key:{} for key in preorder } # Group options by schema @section @@ -229,7 +235,7 @@ def compute_build_signature(env): sections[sect][name] = ddict # - # CONFIG_EXPORT 2 = config.ini + # CONFIG_EXPORT 2 or 102 = config.ini # if config_dump == 2: print(yellow + "Generating config.ini ...") @@ -337,7 +343,9 @@ f'''# sani = re.sub(r'[- ]+', '_', skey).lower() outfile.write(f"\n[config:{sani}]\n") opts = sections[skey] - for name in sorted(opts): + opts_keys = sorted(opts.keys(), key=lambda x: optsort(x, optorder)) + for name in opts_keys: + if name in ignore: continue val = opts[name]['value'] if val == '': val = 'on' #print(f" {name} = {val}") @@ -348,14 +356,16 @@ f'''# # Standard export just dumps config:basic and config:advanced sections for header in real_config: outfile.write(f'\n[{filegrp[header]}]\n') - for name in sorted(real_config[header]): + opts = real_config[header] + opts_keys = sorted(opts.keys(), key=lambda x: optsort(x, optorder)) + for name in opts_keys: if name in ignore: continue - val = real_config[header][name]['value'] + val = opts[name]['value'] if val == '': val = 'on' outfile.write(ini_fmt.format(name.lower(), val) + '\n') # - # CONFIG_EXPORT 5 = Config.h + # CONFIG_EXPORT 5 or 105 = Config.h # if config_dump == 5: print(yellow + "Generating Config-export.h ...") @@ -382,7 +392,8 @@ f'''# #print(f" skey: {skey}") opts = sections[skey] headed = False - for name in sorted(opts): + opts_keys = sorted(opts.keys(), key=lambda x: optsort(x, optorder)) + for name in opts_keys: if name in ignore: continue val = opts[name]['value'] if not headed: @@ -395,17 +406,19 @@ f'''# # Dump config options in just two sections, by file for header in real_config: out_text += f'\n/**\n * Overrides for {header}\n */\n' - for name in sorted(real_config[header]): + opts = real_config[header] + opts_keys = sorted(opts.keys(), key=lambda x: optsort(x, optorder)) + for name in opts_keys: if name in ignore: continue - val = real_config[header][name]['value'] + val = opts[name]['value'] out_text += define_fmt.format(name, val).strip() + '\n' outfile.write(out_text) # - # CONFIG_EXPORT 3 = schema.json, 4 = schema.yml + # CONFIG_EXPORT 3 = schema.json, 13 = schema_grouped.json, 4 = schema.yml # - if config_dump in (3, 4): + if config_dump in (3, 4, 13): if conf_schema: # @@ -434,7 +447,7 @@ f'''# schema.dump_yaml(conf_schema, build_path / 'schema.yml') # - # Produce a JSON file for CONFIGURATION_EMBEDDING or CONFIG_EXPORT == 1 + # Produce a JSON file for CONFIGURATION_EMBEDDING or CONFIG_EXPORT == 1 or 101 # Skip if an identical JSON file was already present. # if not same_hash and (config_dump == 1 or 'CONFIGURATION_EMBEDDING' in build_defines): @@ -502,5 +515,12 @@ f'''# if __name__ == "__main__": # Build required. From command line just explain usage. - print("Use schema.py to export JSON and YAML from the command-line.") - print("Build Marlin with CONFIG_EXPORT 2 to export 'config.ini'.") + print("*** THIS SCRIPT USED BY common-dependencies.py ***\n\n" + + "Current options for config and schema export:\n" + + " - marlin_config.json : Build Marlin with CONFIG_EXPORT 1 or 101. (Use CONFIGURATION_EMBEDDING for 'mc.zip')\n" + + " - config.ini : Build Marlin with CONFIG_EXPORT 2 or 102.\n" + + " - schema.json : Run 'schema.py json' (CONFIG_EXPORT 3).\n" + + " - schema_grouped.json : Run 'schema.py group' (CONFIG_EXPORT 13).\n" + + " - schema.yml : Run 'schema.py yml' (CONFIG_EXPORT 4).\n" + + " - Config-export.h : Build Marlin with CONFIG_EXPORT 5 or 105.\n" + )