3D-printing/Marlin
1
0
Fork 0
mirror of https://github.com/MarlinFirmware/Marlin.git synced 2025-07-11 00:37:48 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

🔨 Update config/schema scripts (#26483)

Browse source
This commit is contained in:
Scott Lahteine 2023-11-29 14:24:20 -06:00 • committed by GitHub
parent b962014087
commit d62ee95d28
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 160 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

40
Marlin/config.ini
View file

@ -3,10 +3,50 @@
# config.ini - Options to apply before the build
#
[config:base]
#
# ini_use_config - A comma-separated list of actions to apply to the Configuration files.
# The actions will be applied in the listed order.
# - none
# Ignore this file and don't apply any configuration options
#
# - base
# Just apply the options in config:base to the configuration
#
# - minimal
# Just apply the options in config:minimal to the configuration
#
# - all
# Apply all 'config:*' sections in this file to the configuration
#
# - another.ini
# Load another INI file with a path relative to this config.ini file (i.e., within Marlin/)
#
# - https://me.myserver.com/path/to/configs
# Fetch configurations from any URL.
#
# - example/Creality/Ender-5 Plus @ bugfix-2.1.x
# Fetch example configuration files from the MarlinFirmware/Configurations repository
# https://raw.githubusercontent.com/MarlinFirmware/Configurations/bugfix-2.1.x/config/examples/Creality/Ender-5%20Plus/
#
# - example/default @ release-2.0.9.7
# Fetch default configuration files from the MarlinFirmware/Configurations repository
# https://raw.githubusercontent.com/MarlinFirmware/Configurations/release-2.0.9.7/config/default/
#
# - [disable]
# Comment out all #defines in both Configuration.h and Configuration_adv.h. This is useful
# to start with a clean slate before applying any config: options, so only the options explicitly
# set in config.ini will be enabled in the configuration.
#
# - [flatten] (Not yet implemented)
# Produce a flattened set of Configuration.h and Configuration_adv.h files with only the enabled
# #defines and no comments. A clean look, but context-free.
#
ini_use_config = none
# Load all config: sections in this file
;ini_use_config = all
# Disable everything and apply subsequent config:base options
;ini_use_config = [disable], base
# Load config file relative to Marlin/
;ini_use_config = another.ini
# Download configurations from GitHub

49
buildroot/share/PlatformIO/scripts/configuration.py Normal file → Executable file
View file

@ -1,8 +1,9 @@
#!/usr/bin/env python3
#
# configuration.py
# Apply options from config.ini to the existing Configuration headers
#
import re, shutil, configparser
import re, shutil, configparser, datetime
from pathlib import Path
verbose = 0
@ -43,6 +44,7 @@ def apply_opt(name, val, conf=None):
if val in ("on", "", None):
newline = re.sub(r'^(\s*)//+\s*(#define)(\s{1,3})?(\s*)', r'\1\2 \4', line)
elif val == "off":
# TODO: Comment more lines in a multi-line define with \ continuation
newline = re.sub(r'^(\s*)(#define)(\s{1,3})?(\s*)', r'\1//\2 \4', line)
else:
# For options with values, enable and set the value
@ -88,9 +90,38 @@ def apply_opt(name, val, conf=None):
elif not isdef:
break
linenum += 1
lines.insert(linenum, f"{prefix}#define {added:30} // Added by config.ini\n")
currtime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
lines.insert(linenum, f"{prefix}#define {added:30} // Added by config.ini {currtime}\n")
fullpath.write_text(''.join(lines), encoding='utf-8')
# Disable all (most) defined options in the configuration files.
# Everything in the named sections. Section hint for exceptions may be added.
def disable_all_options():
# Create a regex to match the option and capture parts of the line
regex = re.compile(r'^(\s*)(#define\s+)([A-Z0-9_]+\b)(\s?)(\s*)(.*?)(\s*)(//.*)?$', re.IGNORECASE)
# Disable all enabled options in both Config files
for file in ("Configuration.h", "Configuration_adv.h"):
fullpath = config_path(file)
lines = fullpath.read_text(encoding='utf-8').split('\n')
found = False
for i in range(len(lines)):
line = lines[i]
match = regex.match(line)
if match:
name = match[3].upper()
if name in ('CONFIGURATION_H_VERSION', 'CONFIGURATION_ADV_H_VERSION'): continue
if name.startswith('_'): continue
found = True
# Comment out the define
# TODO: Comment more lines in a multi-line define with \ continuation
lines[i] = re.sub(r'^(\s*)(#define)(\s{1,3})?(\s*)', r'\1//\2 \4', line)
blab(f"Disable {name}")
# If the option was found, write the modified lines
if found:
fullpath.write_text('\n'.join(lines), encoding='utf-8')
# Fetch configuration files from GitHub given the path.
# Return True if any files were fetched.
def fetch_example(url):
@ -130,7 +161,7 @@ def fetch_example(url):
def section_items(cp, sectkey):
return cp.items(sectkey) if sectkey in cp.sections() else []
# Apply all items from a config section
# Apply all items from a config section. Ignore ini_ items outside of config:base and config:root.
def apply_ini_by_name(cp, sect):
iniok = True
if sect in ('config:base', 'config:root'):
@ -206,7 +237,17 @@ def apply_config_ini(cp):
fetch_example(ckey)
ckey = 'base'
if ckey == 'all':
#
# [flatten] Write out Configuration.h and Configuration_adv.h files with
# just the enabled options and all other content removed.
#
#if ckey == '[flatten]':
# write_flat_configs()
if ckey == '[disable]':
disable_all_options()
elif ckey == 'all':
apply_sections(cp)
else:

79
buildroot/share/PlatformIO/scripts/schema.py
View file

@ -2,8 +2,14 @@
#
# schema.py
#
# Used by signature.py via common-dependencies.py to generate a schema file during the PlatformIO build.
# This script can also be run standalone from within the Marlin repo to generate all schema files.
# Used by signature.py via common-dependencies.py to generate a schema file during the PlatformIO build
# when CONFIG_EXPORT is defined in the configuration.
#
# This script can also be run standalone from within the Marlin repo to generate JSON and YAML schema files.
#
# This script is a companion to abm/js/schema.js in the MarlinFirmware/AutoBuildMarlin project, which has
# been extended to evaluate conditions and can determine what options are actually enabled, not just which
# options are uncommented. That will be migrated to this script for standalone migration.
#
import re,json
from pathlib import Path
@ -95,6 +101,8 @@ def extract():
sch_out = { 'basic':{}, 'advanced':{} }
# Regex for #define NAME [VALUE] [COMMENT] with sanitized line
defgrep = re.compile(r'^(//)?\s*(#define)\s+([A-Za-z0-9_]+)\s*(.*?)\s*(//.+)?$')
# Pattern to match a float value
flt = r'[-+]?\s*(\d+\.|\d*\.\d+)([eE][-+]?\d+)?[fF]?'
# Defines to ignore
ignore = ('CONFIGURATION_H_VERSION', 'CONFIGURATION_ADV_H_VERSION', 'CONFIG_EXAMPLES_DIR', 'CONFIG_EXPORT')
# Start with unknown state
@ -314,26 +322,27 @@ def extract():
}
# Type is based on the value
if val == '':
value_type = 'switch'
elif re.match(r'^(true|false)$', val):
value_type = 'bool'
val = val == 'true'
elif re.match(r'^[-+]?\s*\d+$', val):
value_type = 'int'
val = int(val)
elif re.match(r'[-+]?\s*(\d+\.|\d*\.\d+)([eE][-+]?\d+)?[fF]?', val):
value_type = 'float'
val = float(val.replace('f',''))
else:
value_type = 'string' if val[0] == '"' \
else 'char' if val[0] == "'" \
else 'state' if re.match(r'^(LOW|HIGH)$', val) \
else 'enum' if re.match(r'^[A-Za-z0-9_]{3,}$', val) \
else 'int[]' if re.match(r'^{(\s*[-+]?\s*\d+\s*(,\s*)?)+}$', val) \
else 'float[]' if re.match(r'^{(\s*[-+]?\s*(\d+\.|\d*\.\d+)([eE][-+]?\d+)?[fF]?\s*(,\s*)?)+}$', val) \
else 'array' if val[0] == '{' \
else ''
value_type = \
'switch' if val == '' \
else 'bool' if re.match(r'^(true|false)$', val) \
else 'int' if re.match(r'^[-+]?\s*\d+$', val) \
else 'ints' if re.match(r'^([-+]?\s*\d+)(\s*,\s*[-+]?\s*\d+)+$', val) \
else 'floats' if re.match(rf'({flt}(\s*,\s*{flt})+)', val) \
else 'float' if re.match(f'^({flt})$', val) \
else 'string' if val[0] == '"' \
else 'char' if val[0] == "'" \
else 'state' if re.match(r'^(LOW|HIGH)$', val) \
else 'enum' if re.match(r'^[A-Za-z0-9_]{3,}$', val) \
else 'int[]' if re.match(r'^{\s*[-+]?\s*\d+(\s*,\s*[-+]?\s*\d+)*\s*}$', val) \
else 'float[]' if re.match(r'^{{\s*{flt}(\s*,\s*{flt})*\s*}}$', val) \
else 'array' if val[0] == '{' \
else ''
val = (val == 'true') if value_type == 'bool' \
else int(val) if value_type == 'int' \
else val.replace('f','') if value_type == 'floats' \
else float(val.replace('f','')) if value_type == 'float' \
else val
if val != '': define_info['value'] = val
if value_type != '': define_info['type'] = value_type
@ -402,25 +411,35 @@ def main():
if schema:
# Get the first command line argument
# Get the command line arguments after the script name
import sys
if len(sys.argv) > 1:
arg = sys.argv[1]
else:
arg = 'some'
args = sys.argv[1:]
if len(args) == 0: args = ['some']
# Does the given array intersect at all with args?
def inargs(c): return len(set(args) & set(c)) > 0
# Help / Unknown option
unk = not inargs(['some','json','jsons','group','yml','yaml'])
if (unk): print(f"Unknown option: '{args[0]}'")
if inargs(['-h', '--help']) or unk:
print("Usage: schema.py [some|json|jsons|group|yml|yaml]...")
print(" some = json + yml")
print(" jsons = json + group")
return
# JSON schema
if arg in ['some', 'json', 'jsons']:
if inargs(['some', 'json', 'jsons']):
print("Generating JSON ...")
dump_json(schema, Path('schema.json'))
# JSON schema (wildcard names)
if arg in ['group', 'jsons']:
if inargs(['group', 'jsons']):
group_options(schema)
dump_json(schema, Path('schema_grouped.json'))
# YAML
if arg in ['some', 'yml', 'yaml']:
if inargs(['some', 'yml', 'yaml']):
try:
import yaml
except ImportError:

47
buildroot/share/PlatformIO/scripts/signature.py Normal file → Executable file
View file

@ -1,3 +1,4 @@
#!/usr/bin/env python3
#
# signature.py
#
@ -44,35 +45,35 @@ def compress_file(filepath, storedname, outpath):
zipf.write(filepath, arcname=storedname, compress_type=zipfile.ZIP_BZIP2, compresslevel=9)
#
# Compute the build signature. The idea is to extract all defines in the configuration headers
# to build a unique reversible signature from this build so it can be included in the binary
# We can reverse the signature to get a 1:1 equivalent configuration file
# Compute the build signature by extracting all configuration settings and
# building a unique reversible signature that can be included in the binary.
# The signature can be reversed to get a 1:1 equivalent configuration file.
#
def compute_build_signature(env):
if 'BUILD_SIGNATURE' in env:
return
build_path = Path(env['PROJECT_BUILD_DIR'], env['PIOENV'])
marlin_json = build_path / 'marlin_config.json'
marlin_zip = build_path / 'mc.zip'
# Definitions from these files will be kept
files_to_keep = [ 'Marlin/Configuration.h', 'Marlin/Configuration_adv.h' ]
build_path = Path(env['PROJECT_BUILD_DIR'], env['PIOENV'])
# Check if we can skip processing
hashes = ''
for header in files_to_keep:
hashes += get_file_sha256sum(header)[0:10]
marlin_json = build_path / 'marlin_config.json'
marlin_zip = build_path / 'mc.zip'
# Read existing config file
# Read a previously exported JSON file
# Same configuration, skip recomputing the build signature
same_hash = False
try:
with marlin_json.open() as infile:
conf = json.load(infile)
if conf['__INITIAL_HASH'] == hashes:
# Same configuration, skip recomputing the building signature
same_hash = conf['__INITIAL_HASH'] == hashes
if same_hash:
compress_file(marlin_json, 'marlin_config.json', marlin_zip)
return
except:
pass
@ -125,9 +126,6 @@ def compute_build_signature(env):
# Remove all boards now
if key.startswith("BOARD_") and key != "BOARD_INFO_NAME":
continue
# Remove all keys ending by "_NAME" as it does not make a difference to the configuration
if key.endswith("_NAME") and key != "CUSTOM_MACHINE_NAME":
continue
# Remove all keys ending by "_T_DECLARED" as it's a copy of extraneous system stuff
if key.endswith("_T_DECLARED"):
continue
@ -196,7 +194,7 @@ def compute_build_signature(env):
outfile.write(ini_fmt.format(key.lower(), ' = ' + val))
#
# Produce a schema.json file if CONFIG_EXPORT == 3
# CONFIG_EXPORT 3 = schema.json, 4 = schema.yml
#
if config_dump >= 3:
try:
@ -207,7 +205,7 @@ def compute_build_signature(env):
if conf_schema:
#
# Produce a schema.json file if CONFIG_EXPORT == 3
# 3 = schema.json
#
if config_dump in (3, 13):
print("Generating schema.json ...")
@ -217,7 +215,7 @@ def compute_build_signature(env):
schema.dump_json(conf_schema, build_path / 'schema_grouped.json')
#
# Produce a schema.yml file if CONFIG_EXPORT == 4
# 4 = schema.yml
#
elif config_dump == 4:
print("Generating schema.yml ...")
@ -243,8 +241,9 @@ def compute_build_signature(env):
#
# Produce a JSON file for CONFIGURATION_EMBEDDING or CONFIG_EXPORT == 1
# Skip if an identical JSON file was already present.
#
if config_dump == 1 or 'CONFIGURATION_EMBEDDING' in defines:
if not same_hash and (config_dump == 1 or 'CONFIGURATION_EMBEDDING' in defines):
with marlin_json.open('w') as outfile:
json.dump(data, outfile, separators=(',', ':'))
@ -255,9 +254,10 @@ def compute_build_signature(env):
return
# Compress the JSON file as much as we can
compress_file(marlin_json, 'marlin_config.json', marlin_zip)
if not same_hash:
compress_file(marlin_json, 'marlin_config.json', marlin_zip)
# Generate a C source file for storing this array
# Generate a C source file containing the entire ZIP file as an array
with open('Marlin/src/mczip.h','wb') as result_file:
result_file.write(
b'#ifndef NO_CONFIGURATION_EMBEDDING_WARNING\n'
@ -274,3 +274,8 @@ def compute_build_signature(env):
if count % 16:
result_file.write(b'\n')
result_file.write(b'};\n')
if __name__ == "__main__":
# Build required. From command line just explain usage.
print("Use schema.py to export JSON and YAML from the command-line.")
print("Build Marlin with CONFIG_EXPORT 2 to export 'config.ini'.")