Fix handling of RECORD objects with variable subobjects#83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sveinse merged 2 commits into from
Aug 20, 2026
Merged

Fix handling of RECORD objects with variable subobjects #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4f46e40
Fix handling of objects with repeated subobjects, "each"
sveinse Aug 19, 2026
8353500
Add stop-gap for linting errors
sveinse Aug 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,41 @@ line-length = 120
exclude = [
"tests/od/**",
]
lint.ignore = [
# Stop gap for getting ruff checks to pass. These should be fixed and removed in the future.
"B015", # Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it.
"B017", # Do not assert blind exception: `Exception`
"BLE001", # Do not catch blind exception: `Exception`
"C400", # Unnecessary generator (rewrite as a list comprehension)
"C401", # Unnecessary generator (rewrite as a set comprehension)
"C408", # Unnecessary `tuple()` call (rewrite as a literal)
"C413", # Unnecessary `list()` call around `sorted()`
"FURB122", # Use of `fo.write` in a for loop
"FURB167", # Use of regular expression alias `re.M`
"I001", # Import block is un-sorted or un-formatted
"PIE810", # Call `startswith` once with a `tuple`
"PLR1711", # Useless `return` statement at end of function
"PLR1736", # List index lookup in `enumerate()` loop
"PLW0127", # Self-assignment of variable
"PLW1510", # `subprocess.run` without explicit `check` argument
"PYI034", # `__iadd__`/`__imod__` methods usually return `self` at runtime
"PYI041", # Use `complex` instead of `int | float | complex`
"RET501", # Do not explicitly `return None` in function if it is the only possible return value
"RUF010", # Use explicit conversion flag
"RUF012", # Mutable default value for class attribute
"RUF059", # Unpacked variable `editors` is never used
"RUF100", # Unused `noqa` directive
"S102", # Use of `exec` detected
"SIM102", # Use a single `if` statement instead of nested `if` statements
"SIM114", # Combine `if` branches using logical `or` operator
"SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements
"SIM202", # Use `x == y` instead of `not x != y`
"TRY004", # Prefer `TypeError` exception for invalid type
"UP031", # Use format specifiers instead of percent format
"UP034", # Avoid extraneous parentheses
"UP035", # Import from `collections.abc` instead
"UP037", # Remove quotes from type annotation
]

[tool.ruff.lint.per-file-ignores]
"tests/*" = [
Expand Down
78 changes: 44 additions & 34 deletions src/objdictgen/jsonod.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -120,6 +120,7 @@ class ValidationError(Exception):
# node.ParamDictionary[index] = { N: { ..dict..}, ..dict.. }
FIELDS_PARAMS = {'comment', 'save', 'buffer_size'}
FIELDS_PARAMS_PROMOTE = {'callback'}
FIELDS_PARAMS_EACH = {"start_index"}

# Fields representing the dictionary value
FIELDS_VALUE = {'value'}
Expand DownExpand Up@@ -421,7 +422,7 @@ def generate_node(contents: str|TODJson, validate: bool = True) -> Node:
if validate:
validate_fromdict(jd, objtypes_i2s, objtypes_s2i)

return node_fromdict(jd, objtypes_s2i)
return node_fromdict(jd, objtypes_s2i, validate=validate)


def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True) -> TODJson:
Expand DownExpand Up@@ -484,7 +485,6 @@ def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True
finally:
# Add in a fancyer index (do it here after index is finished being used)
if rich:
index = obj["index"]
obj["index"] = f'@@"0x{index:04X}" // {index}@@'

dictionary.append(obj)
Expand DownExpand Up@@ -598,9 +598,12 @@ def indexentry_to_jsondict(ientry: TIndexEntry) -> TODObjJson:
obj['struct'] = struct
obj['sub'] = obj.pop('values', []) # type: ignore[typeddict-item] # values is about to be renamed

# Move subindex[1] to 'each' on objecs that contain 'nbmax'
if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][1]:
obj['each'] = obj['sub'].pop(1) # type: ignore[typeddict-item]
# Move subindex[-1] to 'each' on objecs that contain 'nbmax'
if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][-1]:
obj['each'] = obj['sub'].pop(-1) # type: ignore[typeddict-item]
start_index = len(obj["sub"])
if start_index > 1:
obj["each"]["start_index"] = start_index

# Baseobj should have been emptied
if odobj != {}:
Expand DownExpand Up@@ -892,20 +895,18 @@ def validate_indexentry(ientry: TIndexEntry):
nbmaxok = True

elif struct in (OD.ARRAY, OD.NARRAY):
if len(nbmax) == 2:
if len(nbmax) == 2: # Array only have length + repeat
lenok = True
if sum(nbmax) == 1 and nbmax[1]:
if sum(nbmax) == 1 and nbmax[-1]:
nbmaxok = True

elif struct in (OD.RECORD, OD.NRECORD):
if sum(nbmax) and len(nbmax) > 1 and nbmax[1]:
if len(nbmax) >= 2: # Record can have more than one item before repeat
lenok = True
if sum(nbmax) == 1 and nbmax[-1]:
nbmaxok = True
if len(nbmax) == 2:
lenok = True
elif sum(nbmax) == 0:
if sum(nbmax) == 0:
nbmaxok = True
if len(nbmax) > 1:
lenok = True
else:
raise ValidationError(f"Unknown struct '{struct}'")

Expand All@@ -915,7 +916,7 @@ def validate_indexentry(ientry: TIndexEntry):
raise ValidationError(f"Unexpexted count of subindexes in mapping object, found {len(nbmax)}")


def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int], validate: bool = True) -> Node:
""" Convert a dict jd into a Node """

# Create the node and fill the most basic data
Expand DownExpand Up@@ -983,15 +984,16 @@ def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
elif 'built-in' in groups:
refobj = maps.MAPPING_DICTIONARY.get(index)

diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
if diff:
log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
for line in diff.pretty().splitlines():
log.debug(' %s', line)
raise ValidationError(
f"Built-in object index 0x{index:04x} ({index}) "
"does not match against system parameters"
)
if validate:
diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
if diff:
log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
for line in diff.pretty().splitlines():
log.debug(' %s', line)
raise ValidationError(
f"Built-in object index 0x{index:04x} ({index}) "
"does not match against system parameters"
)

return node

Expand DownExpand Up@@ -1093,7 +1095,9 @@ def rearrange_for_node(obj: TODObjJson, objtypes_s2i: dict[str, int]) -> TIndexE

# Move back the each object
if 'each' in obj:
subitems.append(obj.pop('each')) # type: ignore[arg-type]
each = obj.pop('each')
each.pop("start_index", None) # Remove the start_index if present
subitems.append(each) # type: ignore[arg-type]

# Check if the object is a repeat object
repeat = obj.pop('repeat', False)
Expand DownExpand Up@@ -1173,7 +1177,7 @@ def validate_fromdict(jsonobj: TODJson, objtypes_i2s: dict[int, str], objtypes_s
# Verify that we have the expected members
member_compare(jsonobj.keys(), must=FIELDS_DATA_MUST, optional=FIELDS_DATA_OPT)

def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False, is_index_each=False):

# Validated: (See FIELDS_MAPVAPS_*, FIELDS_PARAMS and FIELDS_VALUE)
# ----------
Expand DownExpand Up@@ -1209,7 +1213,7 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):

# Set what parameters should be present, optional or not present
if idx == -1: # Checking "each" section. No object or value
params = 'no'
params = "each"

elif is_repeat: # Object repeat = defined elsewhere. No definition needed.
defs = 'no'
Expand All@@ -1219,10 +1223,9 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
elif is_var: # VAR type, guaranteed idx==0 here
value = 'opt'

elif is_each: # Param have "each". Should never have any defs in idx > 0
if idx > 0:
defs = 'no'
value = 'must'
elif is_each and idx >= is_index_each: # Param have "each" and the index is in the "each" items
defs = 'no'
value = 'must'

else: # All other (not each, not repeat, not VAR)
if idx > 0:
Expand All@@ -1240,6 +1243,8 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
# must |= FIELDS_PARAMS
if params == 'opt':
opts |= FIELDS_PARAMS
if params == 'each':
opts |= FIELDS_PARAMS_EACH
if value == 'must':
must |= FIELDS_VALUE
if value == 'opt':
Expand DownExpand Up@@ -1335,7 +1340,10 @@ def _validate_dictionary(index, obj):
for idx, sub in enumerate(subitems):
try:
is_var = struct in (OD.VAR, OD.NVAR)
_validate_sub(sub, idx, is_var=is_var, is_repeat=is_repeat, is_each='each' in obj)
_validate_sub(
sub, idx, is_var=is_var, is_repeat=is_repeat, is_each="each" in obj,
is_index_each=obj.get("each", {}).get("start_index", 1),
)
except Exception as exc:
exc_amend(exc, f"sub[{idx}]: ")
raise
Expand All@@ -1347,9 +1355,11 @@ def _validate_dictionary(index, obj):
if struct in (OD.VAR, OD.NVAR):
raise ValidationError("Unexpected 'each' use in VAR/NVAR object")

# Having 'each' requires use of only one sub item with 'name' in it
if not (sum(has_name) == 1 and has_name[0]):
raise ValidationError("Unexpected subitems. Subitem 0 must contain name")
# When each is present, the other items must have a name field in them
if sum(has_name) != sub.get("start_index", 1):
raise ValidationError(
f"Unexpected subitems. Subitems 0..{sub['start_index'] - 1} must contain name"
)

try:
_validate_sub(sub, idx=-1)
Expand Down
8 changes: 7 additions & 1 deletion src/objdictgen/schema/od.schema.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -187,6 +187,7 @@
"pdo": { "$ref": "#pdo" },
"nbmin": { "$ref": "#nbmin" },
"nbmax": { "$ref": "#nbmax" },
"start_index": { "$ref": "#index" },
"default": { "$ref": "#value" }
},
"additionalProperties": false,
Expand DownExpand Up@@ -305,6 +306,11 @@
"type": "integer"
},

"start_index": {
"$id": "#start_index",
"type": "integer"
},

"pdo": {
"$id": "#pdo",
"description": "Set if object is usable in a PDO",
Expand DownExpand Up@@ -411,4 +417,4 @@
}

}
}
}
1 change: 1 addition & 0 deletions src/objdictgen/typing.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -198,6 +198,7 @@ def GetDefaultStringSize(self) -> int:
"pdo": bool,
"nbmin": int,
"nbmax": int,
"start_index": int,
"default": TODValue,
})
"""JSON object dictionary "each" type definition."""
Expand Down
2 changes: 1 addition & 1 deletion tests/conftest.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -100,7 +100,7 @@ class Fn:
def diff(a, b, predicate=None, postprocess=None, **kw):
""" Diff two files """
if predicate is None:
predicate = lambda x: True # noqa: E731
predicate = lambda x: True
with open(a, 'r', encoding="utf-8") as f:
da = [n.rstrip() for n in f if predicate(n)]
with open(b, 'r', encoding="utf-8") as f:
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Fix handling of RECORD objects with variable subobjects#83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

sveinse
merged 2 commits into

from

Aug 20, 2026
 Merged


Fix handling of RECORD objects with variable subobjects
#83
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
4f46e40

Fix handling of objects with repeated subobjects, "each"


sveinse Aug 19, 2026
8353500

Add stop-gap for linting errors


sveinse Aug 19, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


35 changes: 35 additions & 0 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,41 @@ line-length = 120
exclude = [
 "tests/od/**",
]
lint.ignore = [
 # Stop gap for getting ruff checks to pass. These should be fixed and removed in the future.
 "B015", # Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it.
 "B017", # Do not assert blind exception: `Exception`
 "BLE001", # Do not catch blind exception: `Exception`
 "C400", # Unnecessary generator (rewrite as a list comprehension)
 "C401", # Unnecessary generator (rewrite as a set comprehension)
 "C408", # Unnecessary `tuple()` call (rewrite as a literal)
 "C413", # Unnecessary `list()` call around `sorted()`
 "FURB122", # Use of `fo.write` in a for loop
 "FURB167", # Use of regular expression alias `re.M`
 "I001", # Import block is un-sorted or un-formatted
 "PIE810", # Call `startswith` once with a `tuple`
 "PLR1711", # Useless `return` statement at end of function
 "PLR1736", # List index lookup in `enumerate()` loop
 "PLW0127", # Self-assignment of variable
 "PLW1510", # `subprocess.run` without explicit `check` argument
 "PYI034", # `__iadd__`/`__imod__` methods usually return `self` at runtime
 "PYI041", # Use `complex` instead of `int | float | complex`
 "RET501", # Do not explicitly `return None` in function if it is the only possible return value
 "RUF010", # Use explicit conversion flag
 "RUF012", # Mutable default value for class attribute
 "RUF059", # Unpacked variable `editors` is never used
 "RUF100", # Unused `noqa` directive
 "S102", # Use of `exec` detected
 "SIM102", # Use a single `if` statement instead of nested `if` statements
 "SIM114", # Combine `if` branches using logical `or` operator
 "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements
 "SIM202", # Use `x == y` instead of `not x != y`
 "TRY004", # Prefer `TypeError` exception for invalid type
 "UP031", # Use format specifiers instead of percent format
 "UP034", # Avoid extraneous parentheses
 "UP035", # Import from `collections.abc` instead
 "UP037", # Remove quotes from type annotation
]

[tool.ruff.lint.per-file-ignores]
"tests/*" = [
Expand Down

78 changes: 44 additions & 34 deletions
src/objdictgen/jsonod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,6 +120,7 @@ class ValidationError(Exception):
# node.ParamDictionary[index] = { N: { ..dict..}, ..dict.. }
FIELDS_PARAMS = {'comment', 'save', 'buffer_size'}
FIELDS_PARAMS_PROMOTE = {'callback'}
FIELDS_PARAMS_EACH = {"start_index"}

# Fields representing the dictionary value
FIELDS_VALUE = {'value'}
Expand DownExpand Up@@ -421,7 +422,7 @@ def generate_node(contents: str|TODJson, validate: bool = True) -> Node:
 if validate:
 validate_fromdict(jd, objtypes_i2s, objtypes_s2i)

 return node_fromdict(jd, objtypes_s2i)
 return node_fromdict(jd, objtypes_s2i, validate=validate)


def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True) -> TODJson:
Expand DownExpand Up@@ -484,7 +485,6 @@ def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True
 finally:
 # Add in a fancyer index (do it here after index is finished being used)
 if rich:
 index = obj["index"]
 obj["index"] = f'@@"0x{index:04X}" // {index}@@'

 dictionary.append(obj)
Expand DownExpand Up@@ -598,9 +598,12 @@ def indexentry_to_jsondict(ientry: TIndexEntry) -> TODObjJson:
 obj['struct'] = struct
 obj['sub'] = obj.pop('values', []) # type: ignore[typeddict-item] # values is about to be renamed

 # Move subindex[1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][1]:
 obj['each'] = obj['sub'].pop(1) # type: ignore[typeddict-item]
 # Move subindex[-1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][-1]:
 obj['each'] = obj['sub'].pop(-1) # type: ignore[typeddict-item]
 start_index = len(obj["sub"])
 if start_index > 1:
 obj["each"]["start_index"] = start_index

 # Baseobj should have been emptied
 if odobj != {}:
Expand DownExpand Up@@ -892,20 +895,18 @@ def validate_indexentry(ientry: TIndexEntry):
 nbmaxok = True

 elif struct in (OD.ARRAY, OD.NARRAY):
 if len(nbmax) == 2:
 if len(nbmax) == 2: # Array only have length + repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[1]:
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True

 elif struct in (OD.RECORD, OD.NRECORD):
 if sum(nbmax) and len(nbmax) > 1 and nbmax[1]:
 if len(nbmax) >= 2: # Record can have more than one item before repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True
 if len(nbmax) == 2:
 lenok = True
 elif sum(nbmax) == 0:
 if sum(nbmax) == 0:
 nbmaxok = True
 if len(nbmax) > 1:
 lenok = True
 else:
 raise ValidationError(f"Unknown struct '{struct}'")

Expand All@@ -915,7 +916,7 @@ def validate_indexentry(ientry: TIndexEntry):
 raise ValidationError(f"Unexpexted count of subindexes in mapping object, found {len(nbmax)}")


def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int], validate: bool = True) -> Node:
 """ Convert a dict jd into a Node """

 # Create the node and fill the most basic data
Expand DownExpand Up@@ -983,15 +984,16 @@ def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
 elif 'built-in' in groups:
 refobj = maps.MAPPING_DICTIONARY.get(index)

 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )
 if validate:
 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )

 return node

Expand DownExpand Up@@ -1093,7 +1095,9 @@ def rearrange_for_node(obj: TODObjJson, objtypes_s2i: dict[str, int]) -> TIndexE

 # Move back the each object
 if 'each' in obj:
 subitems.append(obj.pop('each')) # type: ignore[arg-type]
 each = obj.pop('each')
 each.pop("start_index", None) # Remove the start_index if present
 subitems.append(each) # type: ignore[arg-type]

 # Check if the object is a repeat object
 repeat = obj.pop('repeat', False)
Expand DownExpand Up@@ -1173,7 +1177,7 @@ def validate_fromdict(jsonobj: TODJson, objtypes_i2s: dict[int, str], objtypes_s
 # Verify that we have the expected members
 member_compare(jsonobj.keys(), must=FIELDS_DATA_MUST, optional=FIELDS_DATA_OPT)

 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False, is_index_each=False):

 # Validated: (See FIELDS_MAPVAPS_*, FIELDS_PARAMS and FIELDS_VALUE)
 # ----------
Expand DownExpand Up@@ -1209,7 +1213,7 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):

 # Set what parameters should be present, optional or not present
 if idx == -1: # Checking "each" section. No object or value
 params = 'no'
 params = "each"

 elif is_repeat: # Object repeat = defined elsewhere. No definition needed.
 defs = 'no'
Expand All@@ -1219,10 +1223,9 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 elif is_var: # VAR type, guaranteed idx==0 here
 value = 'opt'

 elif is_each: # Param have "each". Should never have any defs in idx > 0
 if idx > 0:
 defs = 'no'
 value = 'must'
 elif is_each and idx >= is_index_each: # Param have "each" and the index is in the "each" items
 defs = 'no'
 value = 'must'

 else: # All other (not each, not repeat, not VAR)
 if idx > 0:
Expand All@@ -1240,6 +1243,8 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 # must |= FIELDS_PARAMS
 if params == 'opt':
 opts |= FIELDS_PARAMS
 if params == 'each':
 opts |= FIELDS_PARAMS_EACH
 if value == 'must':
 must |= FIELDS_VALUE
 if value == 'opt':
Expand DownExpand Up@@ -1335,7 +1340,10 @@ def _validate_dictionary(index, obj):
 for idx, sub in enumerate(subitems):
 try:
 is_var = struct in (OD.VAR, OD.NVAR)
 _validate_sub(sub, idx, is_var=is_var, is_repeat=is_repeat, is_each='each' in obj)
 _validate_sub(
 sub, idx, is_var=is_var, is_repeat=is_repeat, is_each="each" in obj,
 is_index_each=obj.get("each", {}).get("start_index", 1),
 )
 except Exception as exc:
 exc_amend(exc, f"sub[{idx}]: ")
 raise
Expand All@@ -1347,9 +1355,11 @@ def _validate_dictionary(index, obj):
 if struct in (OD.VAR, OD.NVAR):
 raise ValidationError("Unexpected 'each' use in VAR/NVAR object")

 # Having 'each' requires use of only one sub item with 'name' in it
 if not (sum(has_name) == 1 and has_name[0]):
 raise ValidationError("Unexpected subitems. Subitem 0 must contain name")
 # When each is present, the other items must have a name field in them
 if sum(has_name) != sub.get("start_index", 1):
 raise ValidationError(
 f"Unexpected subitems. Subitems 0..{sub['start_index'] - 1} must contain name"
 )

 try:
 _validate_sub(sub, idx=-1)
Expand Down

8 changes: 7 additions & 1 deletion
src/objdictgen/schema/od.schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -187,6 +187,7 @@
 "pdo": { "$ref": "#pdo" },
 "nbmin": { "$ref": "#nbmin" },
 "nbmax": { "$ref": "#nbmax" },
 "start_index": { "$ref": "#index" },
 "default": { "$ref": "#value" }
 },
 "additionalProperties": false,
Expand DownExpand Up@@ -305,6 +306,11 @@
 "type": "integer"
 },

 "start_index": {
 "$id": "#start_index",
 "type": "integer"
 },

 "pdo": {
 "$id": "#pdo",
 "description": "Set if object is usable in a PDO",
Expand DownExpand Up@@ -411,4 +417,4 @@
 }

 }
}
}

1 change: 1 addition & 0 deletions
src/objdictgen/typing.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -198,6 +198,7 @@ def GetDefaultStringSize(self) -> int:
 "pdo": bool,
 "nbmin": int,
 "nbmax": int,
 "start_index": int,
 "default": TODValue,
})
"""JSON object dictionary "each" type definition."""
Expand Down

2 changes: 1 addition & 1 deletion
tests/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,7 +100,7 @@ class Fn:
 def diff(a, b, predicate=None, postprocess=None, **kw):
 """ Diff two files """
 if predicate is None:
 predicate = lambda x: True # noqa: E731
 predicate = lambda x: True
 with open(a, 'r', encoding="utf-8") as f:
 da = [n.rstrip() for n in f if predicate(n)]
 with open(b, 'r', encoding="utf-8") as f:
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix handling of RECORD objects with variable subobjects#83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

sveinse
merged 2 commits into

from

Aug 20, 2026
 Merged


Fix handling of RECORD objects with variable subobjects
#83
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
4f46e40

Fix handling of objects with repeated subobjects, "each"


sveinse Aug 19, 2026
8353500

Add stop-gap for linting errors


sveinse Aug 19, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


35 changes: 35 additions & 0 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,41 @@ line-length = 120
exclude = [
 "tests/od/**",
]
lint.ignore = [
 # Stop gap for getting ruff checks to pass. These should be fixed and removed in the future.
 "B015", # Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it.
 "B017", # Do not assert blind exception: `Exception`
 "BLE001", # Do not catch blind exception: `Exception`
 "C400", # Unnecessary generator (rewrite as a list comprehension)
 "C401", # Unnecessary generator (rewrite as a set comprehension)
 "C408", # Unnecessary `tuple()` call (rewrite as a literal)
 "C413", # Unnecessary `list()` call around `sorted()`
 "FURB122", # Use of `fo.write` in a for loop
 "FURB167", # Use of regular expression alias `re.M`
 "I001", # Import block is un-sorted or un-formatted
 "PIE810", # Call `startswith` once with a `tuple`
 "PLR1711", # Useless `return` statement at end of function
 "PLR1736", # List index lookup in `enumerate()` loop
 "PLW0127", # Self-assignment of variable
 "PLW1510", # `subprocess.run` without explicit `check` argument
 "PYI034", # `__iadd__`/`__imod__` methods usually return `self` at runtime
 "PYI041", # Use `complex` instead of `int | float | complex`
 "RET501", # Do not explicitly `return None` in function if it is the only possible return value
 "RUF010", # Use explicit conversion flag
 "RUF012", # Mutable default value for class attribute
 "RUF059", # Unpacked variable `editors` is never used
 "RUF100", # Unused `noqa` directive
 "S102", # Use of `exec` detected
 "SIM102", # Use a single `if` statement instead of nested `if` statements
 "SIM114", # Combine `if` branches using logical `or` operator
 "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements
 "SIM202", # Use `x == y` instead of `not x != y`
 "TRY004", # Prefer `TypeError` exception for invalid type
 "UP031", # Use format specifiers instead of percent format
 "UP034", # Avoid extraneous parentheses
 "UP035", # Import from `collections.abc` instead
 "UP037", # Remove quotes from type annotation
]

[tool.ruff.lint.per-file-ignores]
"tests/*" = [
Expand Down

78 changes: 44 additions & 34 deletions
src/objdictgen/jsonod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,6 +120,7 @@ class ValidationError(Exception):
# node.ParamDictionary[index] = { N: { ..dict..}, ..dict.. }
FIELDS_PARAMS = {'comment', 'save', 'buffer_size'}
FIELDS_PARAMS_PROMOTE = {'callback'}
FIELDS_PARAMS_EACH = {"start_index"}

# Fields representing the dictionary value
FIELDS_VALUE = {'value'}
Expand DownExpand Up@@ -421,7 +422,7 @@ def generate_node(contents: str|TODJson, validate: bool = True) -> Node:
 if validate:
 validate_fromdict(jd, objtypes_i2s, objtypes_s2i)

 return node_fromdict(jd, objtypes_s2i)
 return node_fromdict(jd, objtypes_s2i, validate=validate)


def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True) -> TODJson:
Expand DownExpand Up@@ -484,7 +485,6 @@ def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True
 finally:
 # Add in a fancyer index (do it here after index is finished being used)
 if rich:
 index = obj["index"]
 obj["index"] = f'@@"0x{index:04X}" // {index}@@'

 dictionary.append(obj)
Expand DownExpand Up@@ -598,9 +598,12 @@ def indexentry_to_jsondict(ientry: TIndexEntry) -> TODObjJson:
 obj['struct'] = struct
 obj['sub'] = obj.pop('values', []) # type: ignore[typeddict-item] # values is about to be renamed

 # Move subindex[1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][1]:
 obj['each'] = obj['sub'].pop(1) # type: ignore[typeddict-item]
 # Move subindex[-1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][-1]:
 obj['each'] = obj['sub'].pop(-1) # type: ignore[typeddict-item]
 start_index = len(obj["sub"])
 if start_index > 1:
 obj["each"]["start_index"] = start_index

 # Baseobj should have been emptied
 if odobj != {}:
Expand DownExpand Up@@ -892,20 +895,18 @@ def validate_indexentry(ientry: TIndexEntry):
 nbmaxok = True

 elif struct in (OD.ARRAY, OD.NARRAY):
 if len(nbmax) == 2:
 if len(nbmax) == 2: # Array only have length + repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[1]:
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True

 elif struct in (OD.RECORD, OD.NRECORD):
 if sum(nbmax) and len(nbmax) > 1 and nbmax[1]:
 if len(nbmax) >= 2: # Record can have more than one item before repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True
 if len(nbmax) == 2:
 lenok = True
 elif sum(nbmax) == 0:
 if sum(nbmax) == 0:
 nbmaxok = True
 if len(nbmax) > 1:
 lenok = True
 else:
 raise ValidationError(f"Unknown struct '{struct}'")

Expand All@@ -915,7 +916,7 @@ def validate_indexentry(ientry: TIndexEntry):
 raise ValidationError(f"Unexpexted count of subindexes in mapping object, found {len(nbmax)}")


def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int], validate: bool = True) -> Node:
 """ Convert a dict jd into a Node """

 # Create the node and fill the most basic data
Expand DownExpand Up@@ -983,15 +984,16 @@ def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
 elif 'built-in' in groups:
 refobj = maps.MAPPING_DICTIONARY.get(index)

 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )
 if validate:
 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )

 return node

Expand DownExpand Up@@ -1093,7 +1095,9 @@ def rearrange_for_node(obj: TODObjJson, objtypes_s2i: dict[str, int]) -> TIndexE

 # Move back the each object
 if 'each' in obj:
 subitems.append(obj.pop('each')) # type: ignore[arg-type]
 each = obj.pop('each')
 each.pop("start_index", None) # Remove the start_index if present
 subitems.append(each) # type: ignore[arg-type]

 # Check if the object is a repeat object
 repeat = obj.pop('repeat', False)
Expand DownExpand Up@@ -1173,7 +1177,7 @@ def validate_fromdict(jsonobj: TODJson, objtypes_i2s: dict[int, str], objtypes_s
 # Verify that we have the expected members
 member_compare(jsonobj.keys(), must=FIELDS_DATA_MUST, optional=FIELDS_DATA_OPT)

 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False, is_index_each=False):

 # Validated: (See FIELDS_MAPVAPS_*, FIELDS_PARAMS and FIELDS_VALUE)
 # ----------
Expand DownExpand Up@@ -1209,7 +1213,7 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):

 # Set what parameters should be present, optional or not present
 if idx == -1: # Checking "each" section. No object or value
 params = 'no'
 params = "each"

 elif is_repeat: # Object repeat = defined elsewhere. No definition needed.
 defs = 'no'
Expand All@@ -1219,10 +1223,9 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 elif is_var: # VAR type, guaranteed idx==0 here
 value = 'opt'

 elif is_each: # Param have "each". Should never have any defs in idx > 0
 if idx > 0:
 defs = 'no'
 value = 'must'
 elif is_each and idx >= is_index_each: # Param have "each" and the index is in the "each" items
 defs = 'no'
 value = 'must'

 else: # All other (not each, not repeat, not VAR)
 if idx > 0:
Expand All@@ -1240,6 +1243,8 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 # must |= FIELDS_PARAMS
 if params == 'opt':
 opts |= FIELDS_PARAMS
 if params == 'each':
 opts |= FIELDS_PARAMS_EACH
 if value == 'must':
 must |= FIELDS_VALUE
 if value == 'opt':
Expand DownExpand Up@@ -1335,7 +1340,10 @@ def _validate_dictionary(index, obj):
 for idx, sub in enumerate(subitems):
 try:
 is_var = struct in (OD.VAR, OD.NVAR)
 _validate_sub(sub, idx, is_var=is_var, is_repeat=is_repeat, is_each='each' in obj)
 _validate_sub(
 sub, idx, is_var=is_var, is_repeat=is_repeat, is_each="each" in obj,
 is_index_each=obj.get("each", {}).get("start_index", 1),
 )
 except Exception as exc:
 exc_amend(exc, f"sub[{idx}]: ")
 raise
Expand All@@ -1347,9 +1355,11 @@ def _validate_dictionary(index, obj):
 if struct in (OD.VAR, OD.NVAR):
 raise ValidationError("Unexpected 'each' use in VAR/NVAR object")

 # Having 'each' requires use of only one sub item with 'name' in it
 if not (sum(has_name) == 1 and has_name[0]):
 raise ValidationError("Unexpected subitems. Subitem 0 must contain name")
 # When each is present, the other items must have a name field in them
 if sum(has_name) != sub.get("start_index", 1):
 raise ValidationError(
 f"Unexpected subitems. Subitems 0..{sub['start_index'] - 1} must contain name"
 )

 try:
 _validate_sub(sub, idx=-1)
Expand Down

8 changes: 7 additions & 1 deletion
src/objdictgen/schema/od.schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -187,6 +187,7 @@
 "pdo": { "$ref": "#pdo" },
 "nbmin": { "$ref": "#nbmin" },
 "nbmax": { "$ref": "#nbmax" },
 "start_index": { "$ref": "#index" },
 "default": { "$ref": "#value" }
 },
 "additionalProperties": false,
Expand DownExpand Up@@ -305,6 +306,11 @@
 "type": "integer"
 },

 "start_index": {
 "$id": "#start_index",
 "type": "integer"
 },

 "pdo": {
 "$id": "#pdo",
 "description": "Set if object is usable in a PDO",
Expand DownExpand Up@@ -411,4 +417,4 @@
 }

 }
}
}

1 change: 1 addition & 0 deletions
src/objdictgen/typing.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -198,6 +198,7 @@ def GetDefaultStringSize(self) -> int:
 "pdo": bool,
 "nbmin": int,
 "nbmax": int,
 "start_index": int,
 "default": TODValue,
})
"""JSON object dictionary "each" type definition."""
Expand Down

2 changes: 1 addition & 1 deletion
tests/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,7 +100,7 @@ class Fn:
 def diff(a, b, predicate=None, postprocess=None, **kw):
 """ Diff two files """
 if predicate is None:
 predicate = lambda x: True # noqa: E731
 predicate = lambda x: True
 with open(a, 'r', encoding="utf-8") as f:
 da = [n.rstrip() for n in f if predicate(n)]
 with open(b, 'r', encoding="utf-8") as f:
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix handling of RECORD objects with variable subobjects#83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

sveinse
merged 2 commits into

from

Aug 20, 2026
 Merged


Fix handling of RECORD objects with variable subobjects
#83
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
4f46e40

Fix handling of objects with repeated subobjects, "each"


sveinse Aug 19, 2026
8353500

Add stop-gap for linting errors


sveinse Aug 19, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


35 changes: 35 additions & 0 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,41 @@ line-length = 120
exclude = [
 "tests/od/**",
]
lint.ignore = [
 # Stop gap for getting ruff checks to pass. These should be fixed and removed in the future.
 "B015", # Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it.
 "B017", # Do not assert blind exception: `Exception`
 "BLE001", # Do not catch blind exception: `Exception`
 "C400", # Unnecessary generator (rewrite as a list comprehension)
 "C401", # Unnecessary generator (rewrite as a set comprehension)
 "C408", # Unnecessary `tuple()` call (rewrite as a literal)
 "C413", # Unnecessary `list()` call around `sorted()`
 "FURB122", # Use of `fo.write` in a for loop
 "FURB167", # Use of regular expression alias `re.M`
 "I001", # Import block is un-sorted or un-formatted
 "PIE810", # Call `startswith` once with a `tuple`
 "PLR1711", # Useless `return` statement at end of function
 "PLR1736", # List index lookup in `enumerate()` loop
 "PLW0127", # Self-assignment of variable
 "PLW1510", # `subprocess.run` without explicit `check` argument
 "PYI034", # `__iadd__`/`__imod__` methods usually return `self` at runtime
 "PYI041", # Use `complex` instead of `int | float | complex`
 "RET501", # Do not explicitly `return None` in function if it is the only possible return value
 "RUF010", # Use explicit conversion flag
 "RUF012", # Mutable default value for class attribute
 "RUF059", # Unpacked variable `editors` is never used
 "RUF100", # Unused `noqa` directive
 "S102", # Use of `exec` detected
 "SIM102", # Use a single `if` statement instead of nested `if` statements
 "SIM114", # Combine `if` branches using logical `or` operator
 "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements
 "SIM202", # Use `x == y` instead of `not x != y`
 "TRY004", # Prefer `TypeError` exception for invalid type
 "UP031", # Use format specifiers instead of percent format
 "UP034", # Avoid extraneous parentheses
 "UP035", # Import from `collections.abc` instead
 "UP037", # Remove quotes from type annotation
]

[tool.ruff.lint.per-file-ignores]
"tests/*" = [
Expand Down

78 changes: 44 additions & 34 deletions
src/objdictgen/jsonod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,6 +120,7 @@ class ValidationError(Exception):
# node.ParamDictionary[index] = { N: { ..dict..}, ..dict.. }
FIELDS_PARAMS = {'comment', 'save', 'buffer_size'}
FIELDS_PARAMS_PROMOTE = {'callback'}
FIELDS_PARAMS_EACH = {"start_index"}

# Fields representing the dictionary value
FIELDS_VALUE = {'value'}
Expand DownExpand Up@@ -421,7 +422,7 @@ def generate_node(contents: str|TODJson, validate: bool = True) -> Node:
 if validate:
 validate_fromdict(jd, objtypes_i2s, objtypes_s2i)

 return node_fromdict(jd, objtypes_s2i)
 return node_fromdict(jd, objtypes_s2i, validate=validate)


def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True) -> TODJson:
Expand DownExpand Up@@ -484,7 +485,6 @@ def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True
 finally:
 # Add in a fancyer index (do it here after index is finished being used)
 if rich:
 index = obj["index"]
 obj["index"] = f'@@"0x{index:04X}" // {index}@@'

 dictionary.append(obj)
Expand DownExpand Up@@ -598,9 +598,12 @@ def indexentry_to_jsondict(ientry: TIndexEntry) -> TODObjJson:
 obj['struct'] = struct
 obj['sub'] = obj.pop('values', []) # type: ignore[typeddict-item] # values is about to be renamed

 # Move subindex[1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][1]:
 obj['each'] = obj['sub'].pop(1) # type: ignore[typeddict-item]
 # Move subindex[-1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][-1]:
 obj['each'] = obj['sub'].pop(-1) # type: ignore[typeddict-item]
 start_index = len(obj["sub"])
 if start_index > 1:
 obj["each"]["start_index"] = start_index

 # Baseobj should have been emptied
 if odobj != {}:
Expand DownExpand Up@@ -892,20 +895,18 @@ def validate_indexentry(ientry: TIndexEntry):
 nbmaxok = True

 elif struct in (OD.ARRAY, OD.NARRAY):
 if len(nbmax) == 2:
 if len(nbmax) == 2: # Array only have length + repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[1]:
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True

 elif struct in (OD.RECORD, OD.NRECORD):
 if sum(nbmax) and len(nbmax) > 1 and nbmax[1]:
 if len(nbmax) >= 2: # Record can have more than one item before repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True
 if len(nbmax) == 2:
 lenok = True
 elif sum(nbmax) == 0:
 if sum(nbmax) == 0:
 nbmaxok = True
 if len(nbmax) > 1:
 lenok = True
 else:
 raise ValidationError(f"Unknown struct '{struct}'")

Expand All@@ -915,7 +916,7 @@ def validate_indexentry(ientry: TIndexEntry):
 raise ValidationError(f"Unexpexted count of subindexes in mapping object, found {len(nbmax)}")


def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int], validate: bool = True) -> Node:
 """ Convert a dict jd into a Node """

 # Create the node and fill the most basic data
Expand DownExpand Up@@ -983,15 +984,16 @@ def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
 elif 'built-in' in groups:
 refobj = maps.MAPPING_DICTIONARY.get(index)

 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )
 if validate:
 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )

 return node

Expand DownExpand Up@@ -1093,7 +1095,9 @@ def rearrange_for_node(obj: TODObjJson, objtypes_s2i: dict[str, int]) -> TIndexE

 # Move back the each object
 if 'each' in obj:
 subitems.append(obj.pop('each')) # type: ignore[arg-type]
 each = obj.pop('each')
 each.pop("start_index", None) # Remove the start_index if present
 subitems.append(each) # type: ignore[arg-type]

 # Check if the object is a repeat object
 repeat = obj.pop('repeat', False)
Expand DownExpand Up@@ -1173,7 +1177,7 @@ def validate_fromdict(jsonobj: TODJson, objtypes_i2s: dict[int, str], objtypes_s
 # Verify that we have the expected members
 member_compare(jsonobj.keys(), must=FIELDS_DATA_MUST, optional=FIELDS_DATA_OPT)

 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False, is_index_each=False):

 # Validated: (See FIELDS_MAPVAPS_*, FIELDS_PARAMS and FIELDS_VALUE)
 # ----------
Expand DownExpand Up@@ -1209,7 +1213,7 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):

 # Set what parameters should be present, optional or not present
 if idx == -1: # Checking "each" section. No object or value
 params = 'no'
 params = "each"

 elif is_repeat: # Object repeat = defined elsewhere. No definition needed.
 defs = 'no'
Expand All@@ -1219,10 +1223,9 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 elif is_var: # VAR type, guaranteed idx==0 here
 value = 'opt'

 elif is_each: # Param have "each". Should never have any defs in idx > 0
 if idx > 0:
 defs = 'no'
 value = 'must'
 elif is_each and idx >= is_index_each: # Param have "each" and the index is in the "each" items
 defs = 'no'
 value = 'must'

 else: # All other (not each, not repeat, not VAR)
 if idx > 0:
Expand All@@ -1240,6 +1243,8 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 # must |= FIELDS_PARAMS
 if params == 'opt':
 opts |= FIELDS_PARAMS
 if params == 'each':
 opts |= FIELDS_PARAMS_EACH
 if value == 'must':
 must |= FIELDS_VALUE
 if value == 'opt':
Expand DownExpand Up@@ -1335,7 +1340,10 @@ def _validate_dictionary(index, obj):
 for idx, sub in enumerate(subitems):
 try:
 is_var = struct in (OD.VAR, OD.NVAR)
 _validate_sub(sub, idx, is_var=is_var, is_repeat=is_repeat, is_each='each' in obj)
 _validate_sub(
 sub, idx, is_var=is_var, is_repeat=is_repeat, is_each="each" in obj,
 is_index_each=obj.get("each", {}).get("start_index", 1),
 )
 except Exception as exc:
 exc_amend(exc, f"sub[{idx}]: ")
 raise
Expand All@@ -1347,9 +1355,11 @@ def _validate_dictionary(index, obj):
 if struct in (OD.VAR, OD.NVAR):
 raise ValidationError("Unexpected 'each' use in VAR/NVAR object")

 # Having 'each' requires use of only one sub item with 'name' in it
 if not (sum(has_name) == 1 and has_name[0]):
 raise ValidationError("Unexpected subitems. Subitem 0 must contain name")
 # When each is present, the other items must have a name field in them
 if sum(has_name) != sub.get("start_index", 1):
 raise ValidationError(
 f"Unexpected subitems. Subitems 0..{sub['start_index'] - 1} must contain name"
 )

 try:
 _validate_sub(sub, idx=-1)
Expand Down

8 changes: 7 additions & 1 deletion
src/objdictgen/schema/od.schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -187,6 +187,7 @@
 "pdo": { "$ref": "#pdo" },
 "nbmin": { "$ref": "#nbmin" },
 "nbmax": { "$ref": "#nbmax" },
 "start_index": { "$ref": "#index" },
 "default": { "$ref": "#value" }
 },
 "additionalProperties": false,
Expand DownExpand Up@@ -305,6 +306,11 @@
 "type": "integer"
 },

 "start_index": {
 "$id": "#start_index",
 "type": "integer"
 },

 "pdo": {
 "$id": "#pdo",
 "description": "Set if object is usable in a PDO",
Expand DownExpand Up@@ -411,4 +417,4 @@
 }

 }
}
}

1 change: 1 addition & 0 deletions
src/objdictgen/typing.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -198,6 +198,7 @@ def GetDefaultStringSize(self) -> int:
 "pdo": bool,
 "nbmin": int,
 "nbmax": int,
 "start_index": int,
 "default": TODValue,
})
"""JSON object dictionary "each" type definition."""
Expand Down

2 changes: 1 addition & 1 deletion
tests/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,7 +100,7 @@ class Fn:
 def diff(a, b, predicate=None, postprocess=None, **kw):
 """ Diff two files """
 if predicate is None:
 predicate = lambda x: True # noqa: E731
 predicate = lambda x: True
 with open(a, 'r', encoding="utf-8") as f:
 da = [n.rstrip() for n in f if predicate(n)]
 with open(b, 'r', encoding="utf-8") as f:
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Fix handling of RECORD objects with variable subobjects#83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

sveinse
merged 2 commits into

from

Aug 20, 2026
 Merged


Fix handling of RECORD objects with variable subobjects
#83
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
4f46e40

Fix handling of objects with repeated subobjects, "each"


sveinse Aug 19, 2026
8353500

Add stop-gap for linting errors


sveinse Aug 19, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


35 changes: 35 additions & 0 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,41 @@ line-length = 120
exclude = [
 "tests/od/**",
]
lint.ignore = [
 # Stop gap for getting ruff checks to pass. These should be fixed and removed in the future.
 "B015", # Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it.
 "B017", # Do not assert blind exception: `Exception`
 "BLE001", # Do not catch blind exception: `Exception`
 "C400", # Unnecessary generator (rewrite as a list comprehension)
 "C401", # Unnecessary generator (rewrite as a set comprehension)
 "C408", # Unnecessary `tuple()` call (rewrite as a literal)
 "C413", # Unnecessary `list()` call around `sorted()`
 "FURB122", # Use of `fo.write` in a for loop
 "FURB167", # Use of regular expression alias `re.M`
 "I001", # Import block is un-sorted or un-formatted
 "PIE810", # Call `startswith` once with a `tuple`
 "PLR1711", # Useless `return` statement at end of function
 "PLR1736", # List index lookup in `enumerate()` loop
 "PLW0127", # Self-assignment of variable
 "PLW1510", # `subprocess.run` without explicit `check` argument
 "PYI034", # `__iadd__`/`__imod__` methods usually return `self` at runtime
 "PYI041", # Use `complex` instead of `int | float | complex`
 "RET501", # Do not explicitly `return None` in function if it is the only possible return value
 "RUF010", # Use explicit conversion flag
 "RUF012", # Mutable default value for class attribute
 "RUF059", # Unpacked variable `editors` is never used
 "RUF100", # Unused `noqa` directive
 "S102", # Use of `exec` detected
 "SIM102", # Use a single `if` statement instead of nested `if` statements
 "SIM114", # Combine `if` branches using logical `or` operator
 "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements
 "SIM202", # Use `x == y` instead of `not x != y`
 "TRY004", # Prefer `TypeError` exception for invalid type
 "UP031", # Use format specifiers instead of percent format
 "UP034", # Avoid extraneous parentheses
 "UP035", # Import from `collections.abc` instead
 "UP037", # Remove quotes from type annotation
]

[tool.ruff.lint.per-file-ignores]
"tests/*" = [
Expand Down

78 changes: 44 additions & 34 deletions
src/objdictgen/jsonod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,6 +120,7 @@ class ValidationError(Exception):
# node.ParamDictionary[index] = { N: { ..dict..}, ..dict.. }
FIELDS_PARAMS = {'comment', 'save', 'buffer_size'}
FIELDS_PARAMS_PROMOTE = {'callback'}
FIELDS_PARAMS_EACH = {"start_index"}

# Fields representing the dictionary value
FIELDS_VALUE = {'value'}
Expand DownExpand Up@@ -421,7 +422,7 @@ def generate_node(contents: str|TODJson, validate: bool = True) -> Node:
 if validate:
 validate_fromdict(jd, objtypes_i2s, objtypes_s2i)

 return node_fromdict(jd, objtypes_s2i)
 return node_fromdict(jd, objtypes_s2i, validate=validate)


def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True) -> TODJson:
Expand DownExpand Up@@ -484,7 +485,6 @@ def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True
 finally:
 # Add in a fancyer index (do it here after index is finished being used)
 if rich:
 index = obj["index"]
 obj["index"] = f'@@"0x{index:04X}" // {index}@@'

 dictionary.append(obj)
Expand DownExpand Up@@ -598,9 +598,12 @@ def indexentry_to_jsondict(ientry: TIndexEntry) -> TODObjJson:
 obj['struct'] = struct
 obj['sub'] = obj.pop('values', []) # type: ignore[typeddict-item] # values is about to be renamed

 # Move subindex[1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][1]:
 obj['each'] = obj['sub'].pop(1) # type: ignore[typeddict-item]
 # Move subindex[-1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][-1]:
 obj['each'] = obj['sub'].pop(-1) # type: ignore[typeddict-item]
 start_index = len(obj["sub"])
 if start_index > 1:
 obj["each"]["start_index"] = start_index

 # Baseobj should have been emptied
 if odobj != {}:
Expand DownExpand Up@@ -892,20 +895,18 @@ def validate_indexentry(ientry: TIndexEntry):
 nbmaxok = True

 elif struct in (OD.ARRAY, OD.NARRAY):
 if len(nbmax) == 2:
 if len(nbmax) == 2: # Array only have length + repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[1]:
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True

 elif struct in (OD.RECORD, OD.NRECORD):
 if sum(nbmax) and len(nbmax) > 1 and nbmax[1]:
 if len(nbmax) >= 2: # Record can have more than one item before repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True
 if len(nbmax) == 2:
 lenok = True
 elif sum(nbmax) == 0:
 if sum(nbmax) == 0:
 nbmaxok = True
 if len(nbmax) > 1:
 lenok = True
 else:
 raise ValidationError(f"Unknown struct '{struct}'")

Expand All@@ -915,7 +916,7 @@ def validate_indexentry(ientry: TIndexEntry):
 raise ValidationError(f"Unexpexted count of subindexes in mapping object, found {len(nbmax)}")


def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int], validate: bool = True) -> Node:
 """ Convert a dict jd into a Node """

 # Create the node and fill the most basic data
Expand DownExpand Up@@ -983,15 +984,16 @@ def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
 elif 'built-in' in groups:
 refobj = maps.MAPPING_DICTIONARY.get(index)

 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )
 if validate:
 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )

 return node

Expand DownExpand Up@@ -1093,7 +1095,9 @@ def rearrange_for_node(obj: TODObjJson, objtypes_s2i: dict[str, int]) -> TIndexE

 # Move back the each object
 if 'each' in obj:
 subitems.append(obj.pop('each')) # type: ignore[arg-type]
 each = obj.pop('each')
 each.pop("start_index", None) # Remove the start_index if present
 subitems.append(each) # type: ignore[arg-type]

 # Check if the object is a repeat object
 repeat = obj.pop('repeat', False)
Expand DownExpand Up@@ -1173,7 +1177,7 @@ def validate_fromdict(jsonobj: TODJson, objtypes_i2s: dict[int, str], objtypes_s
 # Verify that we have the expected members
 member_compare(jsonobj.keys(), must=FIELDS_DATA_MUST, optional=FIELDS_DATA_OPT)

 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False, is_index_each=False):

 # Validated: (See FIELDS_MAPVAPS_*, FIELDS_PARAMS and FIELDS_VALUE)
 # ----------
Expand DownExpand Up@@ -1209,7 +1213,7 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):

 # Set what parameters should be present, optional or not present
 if idx == -1: # Checking "each" section. No object or value
 params = 'no'
 params = "each"

 elif is_repeat: # Object repeat = defined elsewhere. No definition needed.
 defs = 'no'
Expand All@@ -1219,10 +1223,9 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 elif is_var: # VAR type, guaranteed idx==0 here
 value = 'opt'

 elif is_each: # Param have "each". Should never have any defs in idx > 0
 if idx > 0:
 defs = 'no'
 value = 'must'
 elif is_each and idx >= is_index_each: # Param have "each" and the index is in the "each" items
 defs = 'no'
 value = 'must'

 else: # All other (not each, not repeat, not VAR)
 if idx > 0:
Expand All@@ -1240,6 +1243,8 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 # must |= FIELDS_PARAMS
 if params == 'opt':
 opts |= FIELDS_PARAMS
 if params == 'each':
 opts |= FIELDS_PARAMS_EACH
 if value == 'must':
 must |= FIELDS_VALUE
 if value == 'opt':
Expand DownExpand Up@@ -1335,7 +1340,10 @@ def _validate_dictionary(index, obj):
 for idx, sub in enumerate(subitems):
 try:
 is_var = struct in (OD.VAR, OD.NVAR)
 _validate_sub(sub, idx, is_var=is_var, is_repeat=is_repeat, is_each='each' in obj)
 _validate_sub(
 sub, idx, is_var=is_var, is_repeat=is_repeat, is_each="each" in obj,
 is_index_each=obj.get("each", {}).get("start_index", 1),
 )
 except Exception as exc:
 exc_amend(exc, f"sub[{idx}]: ")
 raise
Expand All@@ -1347,9 +1355,11 @@ def _validate_dictionary(index, obj):
 if struct in (OD.VAR, OD.NVAR):
 raise ValidationError("Unexpected 'each' use in VAR/NVAR object")

 # Having 'each' requires use of only one sub item with 'name' in it
 if not (sum(has_name) == 1 and has_name[0]):
 raise ValidationError("Unexpected subitems. Subitem 0 must contain name")
 # When each is present, the other items must have a name field in them
 if sum(has_name) != sub.get("start_index", 1):
 raise ValidationError(
 f"Unexpected subitems. Subitems 0..{sub['start_index'] - 1} must contain name"
 )

 try:
 _validate_sub(sub, idx=-1)
Expand Down

8 changes: 7 additions & 1 deletion
src/objdictgen/schema/od.schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -187,6 +187,7 @@
 "pdo": { "$ref": "#pdo" },
 "nbmin": { "$ref": "#nbmin" },
 "nbmax": { "$ref": "#nbmax" },
 "start_index": { "$ref": "#index" },
 "default": { "$ref": "#value" }
 },
 "additionalProperties": false,
Expand DownExpand Up@@ -305,6 +306,11 @@
 "type": "integer"
 },

 "start_index": {
 "$id": "#start_index",
 "type": "integer"
 },

 "pdo": {
 "$id": "#pdo",
 "description": "Set if object is usable in a PDO",
Expand DownExpand Up@@ -411,4 +417,4 @@
 }

 }
}
}

1 change: 1 addition & 0 deletions
src/objdictgen/typing.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -198,6 +198,7 @@ def GetDefaultStringSize(self) -> int:
 "pdo": bool,
 "nbmin": int,
 "nbmax": int,
 "start_index": int,
 "default": TODValue,
})
"""JSON object dictionary "each" type definition."""
Expand Down

2 changes: 1 addition & 1 deletion
tests/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,7 +100,7 @@ class Fn:
 def diff(a, b, predicate=None, postprocess=None, **kw):
 """ Diff two files """
 if predicate is None:
 predicate = lambda x: True # noqa: E731
 predicate = lambda x: True
 with open(a, 'r', encoding="utf-8") as f:
 da = [n.rstrip() for n in f if predicate(n)]
 with open(b, 'r', encoding="utf-8") as f:
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix handling of RECORD objects with variable subobjects#83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

sveinse
merged 2 commits into

from

Aug 20, 2026
 Merged


Fix handling of RECORD objects with variable subobjects
#83
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
4f46e40

Fix handling of objects with repeated subobjects, "each"


sveinse Aug 19, 2026
8353500

Add stop-gap for linting errors


sveinse Aug 19, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


35 changes: 35 additions & 0 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,41 @@ line-length = 120
exclude = [
 "tests/od/**",
]
lint.ignore = [
 # Stop gap for getting ruff checks to pass. These should be fixed and removed in the future.
 "B015", # Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it.
 "B017", # Do not assert blind exception: `Exception`
 "BLE001", # Do not catch blind exception: `Exception`
 "C400", # Unnecessary generator (rewrite as a list comprehension)
 "C401", # Unnecessary generator (rewrite as a set comprehension)
 "C408", # Unnecessary `tuple()` call (rewrite as a literal)
 "C413", # Unnecessary `list()` call around `sorted()`
 "FURB122", # Use of `fo.write` in a for loop
 "FURB167", # Use of regular expression alias `re.M`
 "I001", # Import block is un-sorted or un-formatted
 "PIE810", # Call `startswith` once with a `tuple`
 "PLR1711", # Useless `return` statement at end of function
 "PLR1736", # List index lookup in `enumerate()` loop
 "PLW0127", # Self-assignment of variable
 "PLW1510", # `subprocess.run` without explicit `check` argument
 "PYI034", # `__iadd__`/`__imod__` methods usually return `self` at runtime
 "PYI041", # Use `complex` instead of `int | float | complex`
 "RET501", # Do not explicitly `return None` in function if it is the only possible return value
 "RUF010", # Use explicit conversion flag
 "RUF012", # Mutable default value for class attribute
 "RUF059", # Unpacked variable `editors` is never used
 "RUF100", # Unused `noqa` directive
 "S102", # Use of `exec` detected
 "SIM102", # Use a single `if` statement instead of nested `if` statements
 "SIM114", # Combine `if` branches using logical `or` operator
 "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements
 "SIM202", # Use `x == y` instead of `not x != y`
 "TRY004", # Prefer `TypeError` exception for invalid type
 "UP031", # Use format specifiers instead of percent format
 "UP034", # Avoid extraneous parentheses
 "UP035", # Import from `collections.abc` instead
 "UP037", # Remove quotes from type annotation
]

[tool.ruff.lint.per-file-ignores]
"tests/*" = [
Expand Down

78 changes: 44 additions & 34 deletions
src/objdictgen/jsonod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,6 +120,7 @@ class ValidationError(Exception):
# node.ParamDictionary[index] = { N: { ..dict..}, ..dict.. }
FIELDS_PARAMS = {'comment', 'save', 'buffer_size'}
FIELDS_PARAMS_PROMOTE = {'callback'}
FIELDS_PARAMS_EACH = {"start_index"}

# Fields representing the dictionary value
FIELDS_VALUE = {'value'}
Expand DownExpand Up@@ -421,7 +422,7 @@ def generate_node(contents: str|TODJson, validate: bool = True) -> Node:
 if validate:
 validate_fromdict(jd, objtypes_i2s, objtypes_s2i)

 return node_fromdict(jd, objtypes_s2i)
 return node_fromdict(jd, objtypes_s2i, validate=validate)


def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True) -> TODJson:
Expand DownExpand Up@@ -484,7 +485,6 @@ def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True
 finally:
 # Add in a fancyer index (do it here after index is finished being used)
 if rich:
 index = obj["index"]
 obj["index"] = f'@@"0x{index:04X}" // {index}@@'

 dictionary.append(obj)
Expand DownExpand Up@@ -598,9 +598,12 @@ def indexentry_to_jsondict(ientry: TIndexEntry) -> TODObjJson:
 obj['struct'] = struct
 obj['sub'] = obj.pop('values', []) # type: ignore[typeddict-item] # values is about to be renamed

 # Move subindex[1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][1]:
 obj['each'] = obj['sub'].pop(1) # type: ignore[typeddict-item]
 # Move subindex[-1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][-1]:
 obj['each'] = obj['sub'].pop(-1) # type: ignore[typeddict-item]
 start_index = len(obj["sub"])
 if start_index > 1:
 obj["each"]["start_index"] = start_index

 # Baseobj should have been emptied
 if odobj != {}:
Expand DownExpand Up@@ -892,20 +895,18 @@ def validate_indexentry(ientry: TIndexEntry):
 nbmaxok = True

 elif struct in (OD.ARRAY, OD.NARRAY):
 if len(nbmax) == 2:
 if len(nbmax) == 2: # Array only have length + repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[1]:
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True

 elif struct in (OD.RECORD, OD.NRECORD):
 if sum(nbmax) and len(nbmax) > 1 and nbmax[1]:
 if len(nbmax) >= 2: # Record can have more than one item before repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True
 if len(nbmax) == 2:
 lenok = True
 elif sum(nbmax) == 0:
 if sum(nbmax) == 0:
 nbmaxok = True
 if len(nbmax) > 1:
 lenok = True
 else:
 raise ValidationError(f"Unknown struct '{struct}'")

Expand All@@ -915,7 +916,7 @@ def validate_indexentry(ientry: TIndexEntry):
 raise ValidationError(f"Unexpexted count of subindexes in mapping object, found {len(nbmax)}")


def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int], validate: bool = True) -> Node:
 """ Convert a dict jd into a Node """

 # Create the node and fill the most basic data
Expand DownExpand Up@@ -983,15 +984,16 @@ def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
 elif 'built-in' in groups:
 refobj = maps.MAPPING_DICTIONARY.get(index)

 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )
 if validate:
 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )

 return node

Expand DownExpand Up@@ -1093,7 +1095,9 @@ def rearrange_for_node(obj: TODObjJson, objtypes_s2i: dict[str, int]) -> TIndexE

 # Move back the each object
 if 'each' in obj:
 subitems.append(obj.pop('each')) # type: ignore[arg-type]
 each = obj.pop('each')
 each.pop("start_index", None) # Remove the start_index if present
 subitems.append(each) # type: ignore[arg-type]

 # Check if the object is a repeat object
 repeat = obj.pop('repeat', False)
Expand DownExpand Up@@ -1173,7 +1177,7 @@ def validate_fromdict(jsonobj: TODJson, objtypes_i2s: dict[int, str], objtypes_s
 # Verify that we have the expected members
 member_compare(jsonobj.keys(), must=FIELDS_DATA_MUST, optional=FIELDS_DATA_OPT)

 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False, is_index_each=False):

 # Validated: (See FIELDS_MAPVAPS_*, FIELDS_PARAMS and FIELDS_VALUE)
 # ----------
Expand DownExpand Up@@ -1209,7 +1213,7 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):

 # Set what parameters should be present, optional or not present
 if idx == -1: # Checking "each" section. No object or value
 params = 'no'
 params = "each"

 elif is_repeat: # Object repeat = defined elsewhere. No definition needed.
 defs = 'no'
Expand All@@ -1219,10 +1223,9 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 elif is_var: # VAR type, guaranteed idx==0 here
 value = 'opt'

 elif is_each: # Param have "each". Should never have any defs in idx > 0
 if idx > 0:
 defs = 'no'
 value = 'must'
 elif is_each and idx >= is_index_each: # Param have "each" and the index is in the "each" items
 defs = 'no'
 value = 'must'

 else: # All other (not each, not repeat, not VAR)
 if idx > 0:
Expand All@@ -1240,6 +1243,8 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 # must |= FIELDS_PARAMS
 if params == 'opt':
 opts |= FIELDS_PARAMS
 if params == 'each':
 opts |= FIELDS_PARAMS_EACH
 if value == 'must':
 must |= FIELDS_VALUE
 if value == 'opt':
Expand DownExpand Up@@ -1335,7 +1340,10 @@ def _validate_dictionary(index, obj):
 for idx, sub in enumerate(subitems):
 try:
 is_var = struct in (OD.VAR, OD.NVAR)
 _validate_sub(sub, idx, is_var=is_var, is_repeat=is_repeat, is_each='each' in obj)
 _validate_sub(
 sub, idx, is_var=is_var, is_repeat=is_repeat, is_each="each" in obj,
 is_index_each=obj.get("each", {}).get("start_index", 1),
 )
 except Exception as exc:
 exc_amend(exc, f"sub[{idx}]: ")
 raise
Expand All@@ -1347,9 +1355,11 @@ def _validate_dictionary(index, obj):
 if struct in (OD.VAR, OD.NVAR):
 raise ValidationError("Unexpected 'each' use in VAR/NVAR object")

 # Having 'each' requires use of only one sub item with 'name' in it
 if not (sum(has_name) == 1 and has_name[0]):
 raise ValidationError("Unexpected subitems. Subitem 0 must contain name")
 # When each is present, the other items must have a name field in them
 if sum(has_name) != sub.get("start_index", 1):
 raise ValidationError(
 f"Unexpected subitems. Subitems 0..{sub['start_index'] - 1} must contain name"
 )

 try:
 _validate_sub(sub, idx=-1)
Expand Down

8 changes: 7 additions & 1 deletion
src/objdictgen/schema/od.schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -187,6 +187,7 @@
 "pdo": { "$ref": "#pdo" },
 "nbmin": { "$ref": "#nbmin" },
 "nbmax": { "$ref": "#nbmax" },
 "start_index": { "$ref": "#index" },
 "default": { "$ref": "#value" }
 },
 "additionalProperties": false,
Expand DownExpand Up@@ -305,6 +306,11 @@
 "type": "integer"
 },

 "start_index": {
 "$id": "#start_index",
 "type": "integer"
 },

 "pdo": {
 "$id": "#pdo",
 "description": "Set if object is usable in a PDO",
Expand DownExpand Up@@ -411,4 +417,4 @@
 }

 }
}
}

1 change: 1 addition & 0 deletions
src/objdictgen/typing.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -198,6 +198,7 @@ def GetDefaultStringSize(self) -> int:
 "pdo": bool,
 "nbmin": int,
 "nbmax": int,
 "start_index": int,
 "default": TODValue,
})
"""JSON object dictionary "each" type definition."""
Expand Down

2 changes: 1 addition & 1 deletion
tests/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,7 +100,7 @@ class Fn:
 def diff(a, b, predicate=None, postprocess=None, **kw):
 """ Diff two files """
 if predicate is None:
 predicate = lambda x: True # noqa: E731
 predicate = lambda x: True
 with open(a, 'r', encoding="utf-8") as f:
 da = [n.rstrip() for n in f if predicate(n)]
 with open(b, 'r', encoding="utf-8") as f:
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix handling of RECORD objects with variable subobjects#83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

sveinse
merged 2 commits into

from

Aug 20, 2026
 Merged


Fix handling of RECORD objects with variable subobjects
#83
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
4f46e40

Fix handling of objects with repeated subobjects, "each"


sveinse Aug 19, 2026
8353500

Add stop-gap for linting errors


sveinse Aug 19, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


35 changes: 35 additions & 0 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,41 @@ line-length = 120
exclude = [
 "tests/od/**",
]
lint.ignore = [
 # Stop gap for getting ruff checks to pass. These should be fixed and removed in the future.
 "B015", # Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it.
 "B017", # Do not assert blind exception: `Exception`
 "BLE001", # Do not catch blind exception: `Exception`
 "C400", # Unnecessary generator (rewrite as a list comprehension)
 "C401", # Unnecessary generator (rewrite as a set comprehension)
 "C408", # Unnecessary `tuple()` call (rewrite as a literal)
 "C413", # Unnecessary `list()` call around `sorted()`
 "FURB122", # Use of `fo.write` in a for loop
 "FURB167", # Use of regular expression alias `re.M`
 "I001", # Import block is un-sorted or un-formatted
 "PIE810", # Call `startswith` once with a `tuple`
 "PLR1711", # Useless `return` statement at end of function
 "PLR1736", # List index lookup in `enumerate()` loop
 "PLW0127", # Self-assignment of variable
 "PLW1510", # `subprocess.run` without explicit `check` argument
 "PYI034", # `__iadd__`/`__imod__` methods usually return `self` at runtime
 "PYI041", # Use `complex` instead of `int | float | complex`
 "RET501", # Do not explicitly `return None` in function if it is the only possible return value
 "RUF010", # Use explicit conversion flag
 "RUF012", # Mutable default value for class attribute
 "RUF059", # Unpacked variable `editors` is never used
 "RUF100", # Unused `noqa` directive
 "S102", # Use of `exec` detected
 "SIM102", # Use a single `if` statement instead of nested `if` statements
 "SIM114", # Combine `if` branches using logical `or` operator
 "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements
 "SIM202", # Use `x == y` instead of `not x != y`
 "TRY004", # Prefer `TypeError` exception for invalid type
 "UP031", # Use format specifiers instead of percent format
 "UP034", # Avoid extraneous parentheses
 "UP035", # Import from `collections.abc` instead
 "UP037", # Remove quotes from type annotation
]

[tool.ruff.lint.per-file-ignores]
"tests/*" = [
Expand Down

78 changes: 44 additions & 34 deletions
src/objdictgen/jsonod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,6 +120,7 @@ class ValidationError(Exception):
# node.ParamDictionary[index] = { N: { ..dict..}, ..dict.. }
FIELDS_PARAMS = {'comment', 'save', 'buffer_size'}
FIELDS_PARAMS_PROMOTE = {'callback'}
FIELDS_PARAMS_EACH = {"start_index"}

# Fields representing the dictionary value
FIELDS_VALUE = {'value'}
Expand DownExpand Up@@ -421,7 +422,7 @@ def generate_node(contents: str|TODJson, validate: bool = True) -> Node:
 if validate:
 validate_fromdict(jd, objtypes_i2s, objtypes_s2i)

 return node_fromdict(jd, objtypes_s2i)
 return node_fromdict(jd, objtypes_s2i, validate=validate)


def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True) -> TODJson:
Expand DownExpand Up@@ -484,7 +485,6 @@ def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True
 finally:
 # Add in a fancyer index (do it here after index is finished being used)
 if rich:
 index = obj["index"]
 obj["index"] = f'@@"0x{index:04X}" // {index}@@'

 dictionary.append(obj)
Expand DownExpand Up@@ -598,9 +598,12 @@ def indexentry_to_jsondict(ientry: TIndexEntry) -> TODObjJson:
 obj['struct'] = struct
 obj['sub'] = obj.pop('values', []) # type: ignore[typeddict-item] # values is about to be renamed

 # Move subindex[1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][1]:
 obj['each'] = obj['sub'].pop(1) # type: ignore[typeddict-item]
 # Move subindex[-1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][-1]:
 obj['each'] = obj['sub'].pop(-1) # type: ignore[typeddict-item]
 start_index = len(obj["sub"])
 if start_index > 1:
 obj["each"]["start_index"] = start_index

 # Baseobj should have been emptied
 if odobj != {}:
Expand DownExpand Up@@ -892,20 +895,18 @@ def validate_indexentry(ientry: TIndexEntry):
 nbmaxok = True

 elif struct in (OD.ARRAY, OD.NARRAY):
 if len(nbmax) == 2:
 if len(nbmax) == 2: # Array only have length + repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[1]:
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True

 elif struct in (OD.RECORD, OD.NRECORD):
 if sum(nbmax) and len(nbmax) > 1 and nbmax[1]:
 if len(nbmax) >= 2: # Record can have more than one item before repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True
 if len(nbmax) == 2:
 lenok = True
 elif sum(nbmax) == 0:
 if sum(nbmax) == 0:
 nbmaxok = True
 if len(nbmax) > 1:
 lenok = True
 else:
 raise ValidationError(f"Unknown struct '{struct}'")

Expand All@@ -915,7 +916,7 @@ def validate_indexentry(ientry: TIndexEntry):
 raise ValidationError(f"Unexpexted count of subindexes in mapping object, found {len(nbmax)}")


def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int], validate: bool = True) -> Node:
 """ Convert a dict jd into a Node """

 # Create the node and fill the most basic data
Expand DownExpand Up@@ -983,15 +984,16 @@ def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
 elif 'built-in' in groups:
 refobj = maps.MAPPING_DICTIONARY.get(index)

 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )
 if validate:
 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )

 return node

Expand DownExpand Up@@ -1093,7 +1095,9 @@ def rearrange_for_node(obj: TODObjJson, objtypes_s2i: dict[str, int]) -> TIndexE

 # Move back the each object
 if 'each' in obj:
 subitems.append(obj.pop('each')) # type: ignore[arg-type]
 each = obj.pop('each')
 each.pop("start_index", None) # Remove the start_index if present
 subitems.append(each) # type: ignore[arg-type]

 # Check if the object is a repeat object
 repeat = obj.pop('repeat', False)
Expand DownExpand Up@@ -1173,7 +1177,7 @@ def validate_fromdict(jsonobj: TODJson, objtypes_i2s: dict[int, str], objtypes_s
 # Verify that we have the expected members
 member_compare(jsonobj.keys(), must=FIELDS_DATA_MUST, optional=FIELDS_DATA_OPT)

 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False, is_index_each=False):

 # Validated: (See FIELDS_MAPVAPS_*, FIELDS_PARAMS and FIELDS_VALUE)
 # ----------
Expand DownExpand Up@@ -1209,7 +1213,7 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):

 # Set what parameters should be present, optional or not present
 if idx == -1: # Checking "each" section. No object or value
 params = 'no'
 params = "each"

 elif is_repeat: # Object repeat = defined elsewhere. No definition needed.
 defs = 'no'
Expand All@@ -1219,10 +1223,9 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 elif is_var: # VAR type, guaranteed idx==0 here
 value = 'opt'

 elif is_each: # Param have "each". Should never have any defs in idx > 0
 if idx > 0:
 defs = 'no'
 value = 'must'
 elif is_each and idx >= is_index_each: # Param have "each" and the index is in the "each" items
 defs = 'no'
 value = 'must'

 else: # All other (not each, not repeat, not VAR)
 if idx > 0:
Expand All@@ -1240,6 +1243,8 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 # must |= FIELDS_PARAMS
 if params == 'opt':
 opts |= FIELDS_PARAMS
 if params == 'each':
 opts |= FIELDS_PARAMS_EACH
 if value == 'must':
 must |= FIELDS_VALUE
 if value == 'opt':
Expand DownExpand Up@@ -1335,7 +1340,10 @@ def _validate_dictionary(index, obj):
 for idx, sub in enumerate(subitems):
 try:
 is_var = struct in (OD.VAR, OD.NVAR)
 _validate_sub(sub, idx, is_var=is_var, is_repeat=is_repeat, is_each='each' in obj)
 _validate_sub(
 sub, idx, is_var=is_var, is_repeat=is_repeat, is_each="each" in obj,
 is_index_each=obj.get("each", {}).get("start_index", 1),
 )
 except Exception as exc:
 exc_amend(exc, f"sub[{idx}]: ")
 raise
Expand All@@ -1347,9 +1355,11 @@ def _validate_dictionary(index, obj):
 if struct in (OD.VAR, OD.NVAR):
 raise ValidationError("Unexpected 'each' use in VAR/NVAR object")

 # Having 'each' requires use of only one sub item with 'name' in it
 if not (sum(has_name) == 1 and has_name[0]):
 raise ValidationError("Unexpected subitems. Subitem 0 must contain name")
 # When each is present, the other items must have a name field in them
 if sum(has_name) != sub.get("start_index", 1):
 raise ValidationError(
 f"Unexpected subitems. Subitems 0..{sub['start_index'] - 1} must contain name"
 )

 try:
 _validate_sub(sub, idx=-1)
Expand Down

8 changes: 7 additions & 1 deletion
src/objdictgen/schema/od.schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -187,6 +187,7 @@
 "pdo": { "$ref": "#pdo" },
 "nbmin": { "$ref": "#nbmin" },
 "nbmax": { "$ref": "#nbmax" },
 "start_index": { "$ref": "#index" },
 "default": { "$ref": "#value" }
 },
 "additionalProperties": false,
Expand DownExpand Up@@ -305,6 +306,11 @@
 "type": "integer"
 },

 "start_index": {
 "$id": "#start_index",
 "type": "integer"
 },

 "pdo": {
 "$id": "#pdo",
 "description": "Set if object is usable in a PDO",
Expand DownExpand Up@@ -411,4 +417,4 @@
 }

 }
}
}

1 change: 1 addition & 0 deletions
src/objdictgen/typing.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -198,6 +198,7 @@ def GetDefaultStringSize(self) -> int:
 "pdo": bool,
 "nbmin": int,
 "nbmax": int,
 "start_index": int,
 "default": TODValue,
})
"""JSON object dictionary "each" type definition."""
Expand Down

2 changes: 1 addition & 1 deletion
tests/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,7 +100,7 @@ class Fn:
 def diff(a, b, predicate=None, postprocess=None, **kw):
 """ Diff two files """
 if predicate is None:
 predicate = lambda x: True # noqa: E731
 predicate = lambda x: True
 with open(a, 'r', encoding="utf-8") as f:
 da = [n.rstrip() for n in f if predicate(n)]
 with open(b, 'r', encoding="utf-8") as f:
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Fix handling of RECORD objects with variable subobjects#83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

sveinse
merged 2 commits into

from

Aug 20, 2026
 Merged


Fix handling of RECORD objects with variable subobjects
#83
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
4f46e40

Fix handling of objects with repeated subobjects, "each"


sveinse Aug 19, 2026
8353500

Add stop-gap for linting errors


sveinse Aug 19, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


35 changes: 35 additions & 0 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,41 @@ line-length = 120
exclude = [
 "tests/od/**",
]
lint.ignore = [
 # Stop gap for getting ruff checks to pass. These should be fixed and removed in the future.
 "B015", # Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it.
 "B017", # Do not assert blind exception: `Exception`
 "BLE001", # Do not catch blind exception: `Exception`
 "C400", # Unnecessary generator (rewrite as a list comprehension)
 "C401", # Unnecessary generator (rewrite as a set comprehension)
 "C408", # Unnecessary `tuple()` call (rewrite as a literal)
 "C413", # Unnecessary `list()` call around `sorted()`
 "FURB122", # Use of `fo.write` in a for loop
 "FURB167", # Use of regular expression alias `re.M`
 "I001", # Import block is un-sorted or un-formatted
 "PIE810", # Call `startswith` once with a `tuple`
 "PLR1711", # Useless `return` statement at end of function
 "PLR1736", # List index lookup in `enumerate()` loop
 "PLW0127", # Self-assignment of variable
 "PLW1510", # `subprocess.run` without explicit `check` argument
 "PYI034", # `__iadd__`/`__imod__` methods usually return `self` at runtime
 "PYI041", # Use `complex` instead of `int | float | complex`
 "RET501", # Do not explicitly `return None` in function if it is the only possible return value
 "RUF010", # Use explicit conversion flag
 "RUF012", # Mutable default value for class attribute
 "RUF059", # Unpacked variable `editors` is never used
 "RUF100", # Unused `noqa` directive
 "S102", # Use of `exec` detected
 "SIM102", # Use a single `if` statement instead of nested `if` statements
 "SIM114", # Combine `if` branches using logical `or` operator
 "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements
 "SIM202", # Use `x == y` instead of `not x != y`
 "TRY004", # Prefer `TypeError` exception for invalid type
 "UP031", # Use format specifiers instead of percent format
 "UP034", # Avoid extraneous parentheses
 "UP035", # Import from `collections.abc` instead
 "UP037", # Remove quotes from type annotation
]

[tool.ruff.lint.per-file-ignores]
"tests/*" = [
Expand Down

78 changes: 44 additions & 34 deletions
src/objdictgen/jsonod.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,6 +120,7 @@ class ValidationError(Exception):
# node.ParamDictionary[index] = { N: { ..dict..}, ..dict.. }
FIELDS_PARAMS = {'comment', 'save', 'buffer_size'}
FIELDS_PARAMS_PROMOTE = {'callback'}
FIELDS_PARAMS_EACH = {"start_index"}

# Fields representing the dictionary value
FIELDS_VALUE = {'value'}
Expand DownExpand Up@@ -421,7 +422,7 @@ def generate_node(contents: str|TODJson, validate: bool = True) -> Node:
 if validate:
 validate_fromdict(jd, objtypes_i2s, objtypes_s2i)

 return node_fromdict(jd, objtypes_s2i)
 return node_fromdict(jd, objtypes_s2i, validate=validate)


def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True) -> TODJson:
Expand DownExpand Up@@ -484,7 +485,6 @@ def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True
 finally:
 # Add in a fancyer index (do it here after index is finished being used)
 if rich:
 index = obj["index"]
 obj["index"] = f'@@"0x{index:04X}" // {index}@@'

 dictionary.append(obj)
Expand DownExpand Up@@ -598,9 +598,12 @@ def indexentry_to_jsondict(ientry: TIndexEntry) -> TODObjJson:
 obj['struct'] = struct
 obj['sub'] = obj.pop('values', []) # type: ignore[typeddict-item] # values is about to be renamed

 # Move subindex[1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][1]:
 obj['each'] = obj['sub'].pop(1) # type: ignore[typeddict-item]
 # Move subindex[-1] to 'each' on objecs that contain 'nbmax'
 if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][-1]:
 obj['each'] = obj['sub'].pop(-1) # type: ignore[typeddict-item]
 start_index = len(obj["sub"])
 if start_index > 1:
 obj["each"]["start_index"] = start_index

 # Baseobj should have been emptied
 if odobj != {}:
Expand DownExpand Up@@ -892,20 +895,18 @@ def validate_indexentry(ientry: TIndexEntry):
 nbmaxok = True

 elif struct in (OD.ARRAY, OD.NARRAY):
 if len(nbmax) == 2:
 if len(nbmax) == 2: # Array only have length + repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[1]:
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True

 elif struct in (OD.RECORD, OD.NRECORD):
 if sum(nbmax) and len(nbmax) > 1 and nbmax[1]:
 if len(nbmax) >= 2: # Record can have more than one item before repeat
 lenok = True
 if sum(nbmax) == 1 and nbmax[-1]:
 nbmaxok = True
 if len(nbmax) == 2:
 lenok = True
 elif sum(nbmax) == 0:
 if sum(nbmax) == 0:
 nbmaxok = True
 if len(nbmax) > 1:
 lenok = True
 else:
 raise ValidationError(f"Unknown struct '{struct}'")

Expand All@@ -915,7 +916,7 @@ def validate_indexentry(ientry: TIndexEntry):
 raise ValidationError(f"Unexpexted count of subindexes in mapping object, found {len(nbmax)}")


def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int], validate: bool = True) -> Node:
 """ Convert a dict jd into a Node """

 # Create the node and fill the most basic data
Expand DownExpand Up@@ -983,15 +984,16 @@ def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node:
 elif 'built-in' in groups:
 refobj = maps.MAPPING_DICTIONARY.get(index)

 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )
 if validate:
 diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree')
 if diff:
 log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index)
 for line in diff.pretty().splitlines():
 log.debug(' %s', line)
 raise ValidationError(
 f"Built-in object index 0x{index:04x} ({index}) "
 "does not match against system parameters"
 )

 return node

Expand DownExpand Up@@ -1093,7 +1095,9 @@ def rearrange_for_node(obj: TODObjJson, objtypes_s2i: dict[str, int]) -> TIndexE

 # Move back the each object
 if 'each' in obj:
 subitems.append(obj.pop('each')) # type: ignore[arg-type]
 each = obj.pop('each')
 each.pop("start_index", None) # Remove the start_index if present
 subitems.append(each) # type: ignore[arg-type]

 # Check if the object is a repeat object
 repeat = obj.pop('repeat', False)
Expand DownExpand Up@@ -1173,7 +1177,7 @@ def validate_fromdict(jsonobj: TODJson, objtypes_i2s: dict[int, str], objtypes_s
 # Verify that we have the expected members
 member_compare(jsonobj.keys(), must=FIELDS_DATA_MUST, optional=FIELDS_DATA_OPT)

 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False, is_index_each=False):

 # Validated: (See FIELDS_MAPVAPS_*, FIELDS_PARAMS and FIELDS_VALUE)
 # ----------
Expand DownExpand Up@@ -1209,7 +1213,7 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):

 # Set what parameters should be present, optional or not present
 if idx == -1: # Checking "each" section. No object or value
 params = 'no'
 params = "each"

 elif is_repeat: # Object repeat = defined elsewhere. No definition needed.
 defs = 'no'
Expand All@@ -1219,10 +1223,9 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 elif is_var: # VAR type, guaranteed idx==0 here
 value = 'opt'

 elif is_each: # Param have "each". Should never have any defs in idx > 0
 if idx > 0:
 defs = 'no'
 value = 'must'
 elif is_each and idx >= is_index_each: # Param have "each" and the index is in the "each" items
 defs = 'no'
 value = 'must'

 else: # All other (not each, not repeat, not VAR)
 if idx > 0:
Expand All@@ -1240,6 +1243,8 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False):
 # must |= FIELDS_PARAMS
 if params == 'opt':
 opts |= FIELDS_PARAMS
 if params == 'each':
 opts |= FIELDS_PARAMS_EACH
 if value == 'must':
 must |= FIELDS_VALUE
 if value == 'opt':
Expand DownExpand Up@@ -1335,7 +1340,10 @@ def _validate_dictionary(index, obj):
 for idx, sub in enumerate(subitems):
 try:
 is_var = struct in (OD.VAR, OD.NVAR)
 _validate_sub(sub, idx, is_var=is_var, is_repeat=is_repeat, is_each='each' in obj)
 _validate_sub(
 sub, idx, is_var=is_var, is_repeat=is_repeat, is_each="each" in obj,
 is_index_each=obj.get("each", {}).get("start_index", 1),
 )
 except Exception as exc:
 exc_amend(exc, f"sub[{idx}]: ")
 raise
Expand All@@ -1347,9 +1355,11 @@ def _validate_dictionary(index, obj):
 if struct in (OD.VAR, OD.NVAR):
 raise ValidationError("Unexpected 'each' use in VAR/NVAR object")

 # Having 'each' requires use of only one sub item with 'name' in it
 if not (sum(has_name) == 1 and has_name[0]):
 raise ValidationError("Unexpected subitems. Subitem 0 must contain name")
 # When each is present, the other items must have a name field in them
 if sum(has_name) != sub.get("start_index", 1):
 raise ValidationError(
 f"Unexpected subitems. Subitems 0..{sub['start_index'] - 1} must contain name"
 )

 try:
 _validate_sub(sub, idx=-1)
Expand Down

8 changes: 7 additions & 1 deletion
src/objdictgen/schema/od.schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -187,6 +187,7 @@
 "pdo": { "$ref": "#pdo" },
 "nbmin": { "$ref": "#nbmin" },
 "nbmax": { "$ref": "#nbmax" },
 "start_index": { "$ref": "#index" },
 "default": { "$ref": "#value" }
 },
 "additionalProperties": false,
Expand DownExpand Up@@ -305,6 +306,11 @@
 "type": "integer"
 },

 "start_index": {
 "$id": "#start_index",
 "type": "integer"
 },

 "pdo": {
 "$id": "#pdo",
 "description": "Set if object is usable in a PDO",
Expand DownExpand Up@@ -411,4 +417,4 @@
 }

 }
}
}

1 change: 1 addition & 0 deletions
src/objdictgen/typing.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -198,6 +198,7 @@ def GetDefaultStringSize(self) -> int:
 "pdo": bool,
 "nbmin": int,
 "nbmax": int,
 "start_index": int,
 "default": TODValue,
})
"""JSON object dictionary "each" type definition."""
Expand Down

2 changes: 1 addition & 1 deletion
tests/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,7 +100,7 @@ class Fn:
 def diff(a, b, predicate=None, postprocess=None, **kw):
 """ Diff two files """
 if predicate is None:
 predicate = lambda x: True # noqa: E731
 predicate = lambda x: True
 with open(a, 'r', encoding="utf-8") as f:
 da = [n.rstrip() for n in f if predicate(n)]
 with open(b, 'r', encoding="utf-8") as f:
Expand Down
Loading
Loading