Serialize text and binary headers#600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tasansal merged 14 commits into from
Aug 12, 2025
Merged

Serialize text and binary headers #600

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
99a702b
Fix integration import tests
dmitriyrepin Aug 9, 2025
03251c0
mask_and_scale=False
dmitriyrepin Aug 11, 2025
c40be99
pre-commit
dmitriyrepin Aug 11, 2025
b7632ef
PR Review issues
dmitriyrepin Aug 11, 2025
9d0d40b
serialize-text-and-binary-headers
dmitriyrepin Aug 11, 2025
88c2865
remove dev test data
tasansal Aug 12, 2025
3a9e672
add back whitespace
tasansal Aug 12, 2025
28d07b4
revert import changes
tasansal Aug 12, 2025
9a74f15
fix attribute initialization in `_add_text_binary_headers`
tasansal Aug 12, 2025
c4928cc
Merge branch 'v1' into serialize-text-and-binary-headers
tasansal Aug 12, 2025
c6714cb
Add tests
dmitriyrepin Aug 12, 2025
024bed7
refactor: improve type annotations and docstrings in test utilities
tasansal Aug 12, 2025
3bb2d40
fix formatting
tasansal Aug 12, 2025
e04bb7c
remove redundant `str()` casting in `xr.open_dataset` calls
tasansal Aug 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions src/mdio/converters/segy.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -280,6 +280,40 @@ def _populate_coordinates(
return dataset, drop_vars_delayed


def _add_text_binary_headers(dataset: Dataset, segy_file: SegyFile) -> None:
text_header = segy_file.text_header.splitlines()
# Validate:
# text_header this should be a 40-items array of strings with width of 80 characters.
item_count = 40
if len(text_header) != item_count:
err = f"Invalid text header count: expected {item_count}, got {len(text_header)}"
raise ValueError(err)
char_count = 80
for i, line in enumerate(text_header):
if len(line) != char_count:
err = f"Invalid text header {i} line length: expected {char_count}, got {len(line)}"
raise ValueError(err)
ext_text_header = segy_file.ext_text_header

# If using SegyFile.ext_text_header this should be a minimum of 40 elements and must
# capture all textual information (ensure text_header is a subset of ext_text_header).
if ext_text_header is not None:
for ext_hdr in ext_text_header:
text_header.append(ext_hdr.splitlines())

# Handle case where it may not have any metadata yet
if dataset.metadata.attributes is None:
dataset.attrs["attributes"] = {}

# Update the attributes with the text and binary headers.
dataset.metadata.attributes.update(
{
"textHeader": text_header,
"binaryHeader": segy_file.binary_header.to_dict(),
}
)


def segy_to_mdio(
segy_spec: SegySpec,
mdio_template: AbstractDatasetTemplate,
Expand DownExpand Up@@ -324,6 +358,8 @@ def segy_to_mdio(
name=mdio_template.name, sizes=shape, horizontal_coord_unit=horizontal_unit, headers=headers
)

_add_text_binary_headers(dataset=mdio_ds, segy_file=segy_file)

xr_dataset: xr_Dataset = to_xarray_dataset(mdio_ds=mdio_ds)

xr_dataset, drop_vars_delayed = _populate_coordinates(
Expand Down
44 changes: 24 additions & 20 deletions tests/integration/test_segy_import_export.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
import xarray as xr
from segy import SegyFile
from segy.standards import get_segy_standard
from tests.integration.testing_data import binary_header_teapot_dome
from tests.integration.testing_data import text_header_teapot_dome
from tests.integration.testing_helpers import customize_segy_specs
from tests.integration.testing_helpers import get_inline_header_values
from tests.integration.testing_helpers import get_values
Expand DownExpand Up@@ -266,8 +268,8 @@ def test_3d_import(
segy_to_mdio(
segy_spec=segy_spec,
mdio_template=TemplateRegistry().get("PostStack3DTime"),
input_location=StorageLocation(segy_input.__str__()),
output_location=StorageLocation(zarr_tmp.__str__()),
input_location=StorageLocation(str(segy_input)),
output_location=StorageLocation(str(zarr_tmp)),
overwrite=True,
)

Expand All@@ -278,11 +280,9 @@ class TestReader:

def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
"""Metadata reading tests."""
path = zarr_tmp.__str__()
# path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
# NOTE: If mask_and_scale is not set,
# Xarray will convert int to float and replace _FillValue with NaN
ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
expected_attrs = {
"apiVersion": "1.0.0a1",
"createdOn": "2025-08-06 16:21:54.747880+00:00",
Expand All@@ -297,13 +297,25 @@ def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
else:
assert actual_attrs_json[key] == value

attributes = ds.attrs["attributes"]
assert attributes is not None

# Validate attributes provided by the template
assert attributes["surveyDimensionality"] == "3D"
assert attributes["ensembleType"] == "line"
assert attributes["processingStage"] == "post-stack"

# Validate text header
assert attributes["textHeader"] == text_header_teapot_dome()

# Validate binary header
assert attributes["binaryHeader"] == binary_header_teapot_dome()

def test_meta_variable_read(self, zarr_tmp: Path) -> None:
"""Metadata reading tests."""
path = zarr_tmp.__str__()
# NOTE: If mask_and_scale is not set,
# Xarray will convert int to float and replace _FillValue with NaN
# path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
expected_attrs = {
"count": 97354860,
"sum": -8594.551666259766,
Expand All@@ -318,11 +330,9 @@ def test_meta_variable_read(self, zarr_tmp: Path) -> None:
def test_grid(self, zarr_tmp: Path) -> None:
"""Test validating MDIO variables."""
# Load Xarray dataset from the MDIO file
path = zarr_tmp.__str__()
# path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
# NOTE: If mask_and_scale is not set,
# Xarray will convert int to float and replace _FillValue with NaN
ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)

# Note: in order to create the dataset we used the Time template, so the
# sample dimension is called "time"
Expand DownExpand Up@@ -366,34 +376,28 @@ def test_grid(self, zarr_tmp: Path) -> None:

def test_inline(self, zarr_tmp: Path) -> None:
"""Read and compare every 75 inlines' mean and std. dev."""
path = zarr_tmp.__str__()
# path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
# NOTE: If mask_and_scale is not set,
# Xarray will convert int to float and replace _FillValue with NaN
ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
inlines = ds["amplitude"][::75, :, :]
mean, std = inlines.mean(), inlines.std()
npt.assert_allclose([mean, std], [1.0555277e-04, 6.0027051e-01])

def test_crossline(self, zarr_tmp: Path) -> None:
"""Read and compare every 75 crosslines' mean and std. dev."""
path = zarr_tmp.__str__()
# path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
# NOTE: If mask_and_scale is not set,
# Xarray will convert int to float and replace _FillValue with NaN
ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
xlines = ds["amplitude"][:, ::75, :]
mean, std = xlines.mean(), xlines.std()

npt.assert_allclose([mean, std], [-5.0329847e-05, 5.9406823e-01])

def test_zslice(self, zarr_tmp: Path) -> None:
"""Read and compare every 225 z-slices' mean and std. dev."""
path = zarr_tmp.__str__()
# path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
# NOTE: If mask_and_scale is not set,
# Xarray will convert int to float and replace _FillValue with NaN
ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
slices = ds["amplitude"][:, :, ::225]
mean, std = slices.mean(), slices.std()
npt.assert_allclose([mean, std], [0.005236923, 0.61279935])
Expand Down
84 changes: 84 additions & 0 deletions tests/integration/testing_data.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
"""Integration tests data for teapot dome SEG-Y."""


def text_header_teapot_dome() -> list[str]:
"""Return the teapot dome expected text header."""
return [
"C 1 CLIENT: ROCKY MOUNTAIN OILFIELD TESTING CENTER ",
"C 2 PROJECT: NAVAL PETROLEUM RESERVE #3 (TEAPOT DOME); NATRONA COUNTY, WYOMING ",
"C 3 LINE: 3D ",
"C 4 ",
"C 5 THIS IS THE FILTERED POST STACK MIGRATION ",
"C 6 ",
"C 7 INLINE 1, XLINE 1: X COORDINATE: 788937 Y COORDINATE: 938845 ",
"C 8 INLINE 1, XLINE 188: X COORDINATE: 809501 Y COORDINATE: 939333 ",
"C 9 INLINE 188, XLINE 1: X COORDINATE: 788039 Y COORDINATE: 976674 ",
"C10 INLINE NUMBER: MIN: 1 MAX: 345 TOTAL: 345 ",
"C11 CROSSLINE NUMBER: MIN: 1 MAX: 188 TOTAL: 188 ",
"C12 TOTAL NUMBER OF CDPS: 64860 BIN DIMENSION: 110' X 110' ",
"C13 ",
"C14 ",
"C15 ",
"C16 ",
"C17 ",
"C18 ",
"C19 GENERAL SEGY INFORMATION ",
"C20 RECORD LENGHT (MS): 3000 ",
"C21 SAMPLE RATE (MS): 2.0 ",
"C22 DATA FORMAT: 4 BYTE IBM FLOATING POINT ",
"C23 BYTES 13- 16: CROSSLINE NUMBER (TRACE) ",
"C24 BYTES 17- 20: INLINE NUMBER (LINE) ",
"C25 BYTES 81- 84: CDP_X COORD ",
"C26 BYTES 85- 88: CDP_Y COORD ",
"C27 BYTES 181-184: INLINE NUMBER (LINE) ",
"C28 BYTES 185-188: CROSSLINE NUMBER (TRACE) ",
"C29 BYTES 189-192: CDP_X COORD ",
"C30 BYTES 193-196: CDP_Y COORD ",
"C31 ",
"C32 ",
"C33 ",
"C34 ",
"C35 ",
"C36 Processed by: Excel Geophysical Services, Inc. ",
"C37 8301 East Prentice Ave. Ste. 402 ",
"C38 Englewood, Colorado 80111 ",
"C39 (voice) 303.694.9629 (fax) 303.771.1646 ",
"C40 END EBCDIC ",
]


def binary_header_teapot_dome() -> dict[str, int]:
"""Return the teapot dome expected binary header."""
return {
"job_id": 9999,
"line_num": 9999,
"reel_num": 1,
"data_traces_per_ensemble": 188,
"aux_traces_per_ensemble": 0,
"sample_interval": 2000,
"orig_sample_interval": 0,
"samples_per_trace": 1501,
"orig_samples_per_trace": 1501,
"data_sample_format": 1,
"ensemble_fold": 57,
"trace_sorting_code": 4,
"vertical_sum_code": 1,
"sweep_freq_start": 0,
"sweep_freq_end": 0,
"sweep_length": 0,
"sweep_type_code": 0,
"sweep_trace_num": 0,
"sweep_taper_start": 0,
"sweep_taper_end": 0,
"taper_type_code": 0,
"correlated_data_code": 2,
"binary_gain_code": 1,
"amp_recovery_code": 4,
"measurement_system_code": 2,
"impulse_polarity_code": 1,
"vibratory_polarity_code": 0,
"fixed_length_trace_flag": 0,
"num_extended_text_headers": 0,
"segy_revision_major": 0,
"segy_revision_minor": 0,
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Serialize text and binary headers#600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tasansal
merged 14 commits into

from

Aug 12, 2025
 Merged


Serialize text and binary headers
#600
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


14 commits


Select commit
Hold shift + click to select a range
99a702b

Fix integration import tests


dmitriyrepin Aug 9, 2025
03251c0

mask_and_scale=False


dmitriyrepin Aug 11, 2025
c40be99

pre-commit


dmitriyrepin Aug 11, 2025
b7632ef

PR Review issues


dmitriyrepin Aug 11, 2025
9d0d40b

serialize-text-and-binary-headers


dmitriyrepin Aug 11, 2025
88c2865

remove dev test data


tasansal Aug 12, 2025
3a9e672

add back whitespace


tasansal Aug 12, 2025
28d07b4

revert import changes


tasansal Aug 12, 2025
9a74f15

fix attribute initialization in `_add_text_binary_headers`


tasansal Aug 12, 2025
c4928cc

Merge branch 'v1' into serialize-text-and-binary-headers


tasansal Aug 12, 2025
c6714cb

Add tests


dmitriyrepin Aug 12, 2025
024bed7

refactor: improve type annotations and docstrings in test utilities


tasansal Aug 12, 2025
3bb2d40

fix formatting


tasansal Aug 12, 2025
e04bb7c

remove redundant `str()` casting in `xr.open_dataset` calls


tasansal Aug 12, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


36 changes: 36 additions & 0 deletions
src/mdio/converters/segy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -280,6 +280,40 @@ def _populate_coordinates(
 return dataset, drop_vars_delayed


def _add_text_binary_headers(dataset: Dataset, segy_file: SegyFile) -> None:
 text_header = segy_file.text_header.splitlines()
 # Validate:
 # text_header this should be a 40-items array of strings with width of 80 characters.
 item_count = 40
 if len(text_header) != item_count:
 err = f"Invalid text header count: expected {item_count}, got {len(text_header)}"
 raise ValueError(err)
 char_count = 80
 for i, line in enumerate(text_header):
 if len(line) != char_count:
 err = f"Invalid text header {i} line length: expected {char_count}, got {len(line)}"
 raise ValueError(err)
 ext_text_header = segy_file.ext_text_header

 # If using SegyFile.ext_text_header this should be a minimum of 40 elements and must
 # capture all textual information (ensure text_header is a subset of ext_text_header).
 if ext_text_header is not None:
 for ext_hdr in ext_text_header:
 text_header.append(ext_hdr.splitlines())

 # Handle case where it may not have any metadata yet
 if dataset.metadata.attributes is None:
 dataset.attrs["attributes"] = {}

 # Update the attributes with the text and binary headers.
 dataset.metadata.attributes.update(
 {
 "textHeader": text_header,
 "binaryHeader": segy_file.binary_header.to_dict(),
 }
 )


def segy_to_mdio(
 segy_spec: SegySpec,
 mdio_template: AbstractDatasetTemplate,
Expand DownExpand Up@@ -324,6 +358,8 @@ def segy_to_mdio(
 name=mdio_template.name, sizes=shape, horizontal_coord_unit=horizontal_unit, headers=headers
 )

 _add_text_binary_headers(dataset=mdio_ds, segy_file=segy_file)

 xr_dataset: xr_Dataset = to_xarray_dataset(mdio_ds=mdio_ds)

 xr_dataset, drop_vars_delayed = _populate_coordinates(
Expand Down

44 changes: 24 additions & 20 deletions
tests/integration/test_segy_import_export.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
import xarray as xr
from segy import SegyFile
from segy.standards import get_segy_standard
from tests.integration.testing_data import binary_header_teapot_dome
from tests.integration.testing_data import text_header_teapot_dome
from tests.integration.testing_helpers import customize_segy_specs
from tests.integration.testing_helpers import get_inline_header_values
from tests.integration.testing_helpers import get_values
Expand DownExpand Up@@ -266,8 +268,8 @@ def test_3d_import(
 segy_to_mdio(
 segy_spec=segy_spec,
 mdio_template=TemplateRegistry().get("PostStack3DTime"),
 input_location=StorageLocation(segy_input.__str__()),
 output_location=StorageLocation(zarr_tmp.__str__()),
 input_location=StorageLocation(str(segy_input)),
 output_location=StorageLocation(str(zarr_tmp)),
 overwrite=True,
 )

Expand All@@ -278,11 +280,9 @@ class TestReader:

 def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "apiVersion": "1.0.0a1",
 "createdOn": "2025-08-06 16:21:54.747880+00:00",
Expand All@@ -297,13 +297,25 @@ def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 else:
 assert actual_attrs_json[key] == value

 attributes = ds.attrs["attributes"]
 assert attributes is not None

 # Validate attributes provided by the template
 assert attributes["surveyDimensionality"] == "3D"
 assert attributes["ensembleType"] == "line"
 assert attributes["processingStage"] == "post-stack"

 # Validate text header
 assert attributes["textHeader"] == text_header_teapot_dome()

 # Validate binary header
 assert attributes["binaryHeader"] == binary_header_teapot_dome()

 def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "count": 97354860,
 "sum": -8594.551666259766,
Expand All@@ -318,11 +330,9 @@ def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 def test_grid(self, zarr_tmp: Path) -> None:
 """Test validating MDIO variables."""
 # Load Xarray dataset from the MDIO file
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)

 # Note: in order to create the dataset we used the Time template, so the
 # sample dimension is called "time"
Expand DownExpand Up@@ -366,34 +376,28 @@ def test_grid(self, zarr_tmp: Path) -> None:

 def test_inline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 inlines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 inlines = ds["amplitude"][::75, :, :]
 mean, std = inlines.mean(), inlines.std()
 npt.assert_allclose([mean, std], [1.0555277e-04, 6.0027051e-01])

 def test_crossline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 crosslines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 xlines = ds["amplitude"][:, ::75, :]
 mean, std = xlines.mean(), xlines.std()

 npt.assert_allclose([mean, std], [-5.0329847e-05, 5.9406823e-01])

 def test_zslice(self, zarr_tmp: Path) -> None:
 """Read and compare every 225 z-slices' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 slices = ds["amplitude"][:, :, ::225]
 mean, std = slices.mean(), slices.std()
 npt.assert_allclose([mean, std], [0.005236923, 0.61279935])
Expand Down

84 changes: 84 additions & 0 deletions
tests/integration/testing_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
"""Integration tests data for teapot dome SEG-Y."""


def text_header_teapot_dome() -> list[str]:
 """Return the teapot dome expected text header."""
 return [
 "C 1 CLIENT: ROCKY MOUNTAIN OILFIELD TESTING CENTER ",
 "C 2 PROJECT: NAVAL PETROLEUM RESERVE #3 (TEAPOT DOME); NATRONA COUNTY, WYOMING ",
 "C 3 LINE: 3D ",
 "C 4 ",
 "C 5 THIS IS THE FILTERED POST STACK MIGRATION ",
 "C 6 ",
 "C 7 INLINE 1, XLINE 1: X COORDINATE: 788937 Y COORDINATE: 938845 ",
 "C 8 INLINE 1, XLINE 188: X COORDINATE: 809501 Y COORDINATE: 939333 ",
 "C 9 INLINE 188, XLINE 1: X COORDINATE: 788039 Y COORDINATE: 976674 ",
 "C10 INLINE NUMBER: MIN: 1 MAX: 345 TOTAL: 345 ",
 "C11 CROSSLINE NUMBER: MIN: 1 MAX: 188 TOTAL: 188 ",
 "C12 TOTAL NUMBER OF CDPS: 64860 BIN DIMENSION: 110' X 110' ",
 "C13 ",
 "C14 ",
 "C15 ",
 "C16 ",
 "C17 ",
 "C18 ",
 "C19 GENERAL SEGY INFORMATION ",
 "C20 RECORD LENGHT (MS): 3000 ",
 "C21 SAMPLE RATE (MS): 2.0 ",
 "C22 DATA FORMAT: 4 BYTE IBM FLOATING POINT ",
 "C23 BYTES 13- 16: CROSSLINE NUMBER (TRACE) ",
 "C24 BYTES 17- 20: INLINE NUMBER (LINE) ",
 "C25 BYTES 81- 84: CDP_X COORD ",
 "C26 BYTES 85- 88: CDP_Y COORD ",
 "C27 BYTES 181-184: INLINE NUMBER (LINE) ",
 "C28 BYTES 185-188: CROSSLINE NUMBER (TRACE) ",
 "C29 BYTES 189-192: CDP_X COORD ",
 "C30 BYTES 193-196: CDP_Y COORD ",
 "C31 ",
 "C32 ",
 "C33 ",
 "C34 ",
 "C35 ",
 "C36 Processed by: Excel Geophysical Services, Inc. ",
 "C37 8301 East Prentice Ave. Ste. 402 ",
 "C38 Englewood, Colorado 80111 ",
 "C39 (voice) 303.694.9629 (fax) 303.771.1646 ",
 "C40 END EBCDIC ",
 ]


def binary_header_teapot_dome() -> dict[str, int]:
 """Return the teapot dome expected binary header."""
 return {
 "job_id": 9999,
 "line_num": 9999,
 "reel_num": 1,
 "data_traces_per_ensemble": 188,
 "aux_traces_per_ensemble": 0,
 "sample_interval": 2000,
 "orig_sample_interval": 0,
 "samples_per_trace": 1501,
 "orig_samples_per_trace": 1501,
 "data_sample_format": 1,
 "ensemble_fold": 57,
 "trace_sorting_code": 4,
 "vertical_sum_code": 1,
 "sweep_freq_start": 0,
 "sweep_freq_end": 0,
 "sweep_length": 0,
 "sweep_type_code": 0,
 "sweep_trace_num": 0,
 "sweep_taper_start": 0,
 "sweep_taper_end": 0,
 "taper_type_code": 0,
 "correlated_data_code": 2,
 "binary_gain_code": 1,
 "amp_recovery_code": 4,
 "measurement_system_code": 2,
 "impulse_polarity_code": 1,
 "vibratory_polarity_code": 0,
 "fixed_length_trace_flag": 0,
 "num_extended_text_headers": 0,
 "segy_revision_major": 0,
 "segy_revision_minor": 0,
 }
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Serialize text and binary headers#600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tasansal
merged 14 commits into

from

Aug 12, 2025
 Merged


Serialize text and binary headers
#600
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


14 commits


Select commit
Hold shift + click to select a range
99a702b

Fix integration import tests


dmitriyrepin Aug 9, 2025
03251c0

mask_and_scale=False


dmitriyrepin Aug 11, 2025
c40be99

pre-commit


dmitriyrepin Aug 11, 2025
b7632ef

PR Review issues


dmitriyrepin Aug 11, 2025
9d0d40b

serialize-text-and-binary-headers


dmitriyrepin Aug 11, 2025
88c2865

remove dev test data


tasansal Aug 12, 2025
3a9e672

add back whitespace


tasansal Aug 12, 2025
28d07b4

revert import changes


tasansal Aug 12, 2025
9a74f15

fix attribute initialization in `_add_text_binary_headers`


tasansal Aug 12, 2025
c4928cc

Merge branch 'v1' into serialize-text-and-binary-headers


tasansal Aug 12, 2025
c6714cb

Add tests


dmitriyrepin Aug 12, 2025
024bed7

refactor: improve type annotations and docstrings in test utilities


tasansal Aug 12, 2025
3bb2d40

fix formatting


tasansal Aug 12, 2025
e04bb7c

remove redundant `str()` casting in `xr.open_dataset` calls


tasansal Aug 12, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


36 changes: 36 additions & 0 deletions
src/mdio/converters/segy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -280,6 +280,40 @@ def _populate_coordinates(
 return dataset, drop_vars_delayed


def _add_text_binary_headers(dataset: Dataset, segy_file: SegyFile) -> None:
 text_header = segy_file.text_header.splitlines()
 # Validate:
 # text_header this should be a 40-items array of strings with width of 80 characters.
 item_count = 40
 if len(text_header) != item_count:
 err = f"Invalid text header count: expected {item_count}, got {len(text_header)}"
 raise ValueError(err)
 char_count = 80
 for i, line in enumerate(text_header):
 if len(line) != char_count:
 err = f"Invalid text header {i} line length: expected {char_count}, got {len(line)}"
 raise ValueError(err)
 ext_text_header = segy_file.ext_text_header

 # If using SegyFile.ext_text_header this should be a minimum of 40 elements and must
 # capture all textual information (ensure text_header is a subset of ext_text_header).
 if ext_text_header is not None:
 for ext_hdr in ext_text_header:
 text_header.append(ext_hdr.splitlines())

 # Handle case where it may not have any metadata yet
 if dataset.metadata.attributes is None:
 dataset.attrs["attributes"] = {}

 # Update the attributes with the text and binary headers.
 dataset.metadata.attributes.update(
 {
 "textHeader": text_header,
 "binaryHeader": segy_file.binary_header.to_dict(),
 }
 )


def segy_to_mdio(
 segy_spec: SegySpec,
 mdio_template: AbstractDatasetTemplate,
Expand DownExpand Up@@ -324,6 +358,8 @@ def segy_to_mdio(
 name=mdio_template.name, sizes=shape, horizontal_coord_unit=horizontal_unit, headers=headers
 )

 _add_text_binary_headers(dataset=mdio_ds, segy_file=segy_file)

 xr_dataset: xr_Dataset = to_xarray_dataset(mdio_ds=mdio_ds)

 xr_dataset, drop_vars_delayed = _populate_coordinates(
Expand Down

44 changes: 24 additions & 20 deletions
tests/integration/test_segy_import_export.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
import xarray as xr
from segy import SegyFile
from segy.standards import get_segy_standard
from tests.integration.testing_data import binary_header_teapot_dome
from tests.integration.testing_data import text_header_teapot_dome
from tests.integration.testing_helpers import customize_segy_specs
from tests.integration.testing_helpers import get_inline_header_values
from tests.integration.testing_helpers import get_values
Expand DownExpand Up@@ -266,8 +268,8 @@ def test_3d_import(
 segy_to_mdio(
 segy_spec=segy_spec,
 mdio_template=TemplateRegistry().get("PostStack3DTime"),
 input_location=StorageLocation(segy_input.__str__()),
 output_location=StorageLocation(zarr_tmp.__str__()),
 input_location=StorageLocation(str(segy_input)),
 output_location=StorageLocation(str(zarr_tmp)),
 overwrite=True,
 )

Expand All@@ -278,11 +280,9 @@ class TestReader:

 def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "apiVersion": "1.0.0a1",
 "createdOn": "2025-08-06 16:21:54.747880+00:00",
Expand All@@ -297,13 +297,25 @@ def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 else:
 assert actual_attrs_json[key] == value

 attributes = ds.attrs["attributes"]
 assert attributes is not None

 # Validate attributes provided by the template
 assert attributes["surveyDimensionality"] == "3D"
 assert attributes["ensembleType"] == "line"
 assert attributes["processingStage"] == "post-stack"

 # Validate text header
 assert attributes["textHeader"] == text_header_teapot_dome()

 # Validate binary header
 assert attributes["binaryHeader"] == binary_header_teapot_dome()

 def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "count": 97354860,
 "sum": -8594.551666259766,
Expand All@@ -318,11 +330,9 @@ def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 def test_grid(self, zarr_tmp: Path) -> None:
 """Test validating MDIO variables."""
 # Load Xarray dataset from the MDIO file
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)

 # Note: in order to create the dataset we used the Time template, so the
 # sample dimension is called "time"
Expand DownExpand Up@@ -366,34 +376,28 @@ def test_grid(self, zarr_tmp: Path) -> None:

 def test_inline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 inlines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 inlines = ds["amplitude"][::75, :, :]
 mean, std = inlines.mean(), inlines.std()
 npt.assert_allclose([mean, std], [1.0555277e-04, 6.0027051e-01])

 def test_crossline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 crosslines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 xlines = ds["amplitude"][:, ::75, :]
 mean, std = xlines.mean(), xlines.std()

 npt.assert_allclose([mean, std], [-5.0329847e-05, 5.9406823e-01])

 def test_zslice(self, zarr_tmp: Path) -> None:
 """Read and compare every 225 z-slices' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 slices = ds["amplitude"][:, :, ::225]
 mean, std = slices.mean(), slices.std()
 npt.assert_allclose([mean, std], [0.005236923, 0.61279935])
Expand Down

84 changes: 84 additions & 0 deletions
tests/integration/testing_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
"""Integration tests data for teapot dome SEG-Y."""


def text_header_teapot_dome() -> list[str]:
 """Return the teapot dome expected text header."""
 return [
 "C 1 CLIENT: ROCKY MOUNTAIN OILFIELD TESTING CENTER ",
 "C 2 PROJECT: NAVAL PETROLEUM RESERVE #3 (TEAPOT DOME); NATRONA COUNTY, WYOMING ",
 "C 3 LINE: 3D ",
 "C 4 ",
 "C 5 THIS IS THE FILTERED POST STACK MIGRATION ",
 "C 6 ",
 "C 7 INLINE 1, XLINE 1: X COORDINATE: 788937 Y COORDINATE: 938845 ",
 "C 8 INLINE 1, XLINE 188: X COORDINATE: 809501 Y COORDINATE: 939333 ",
 "C 9 INLINE 188, XLINE 1: X COORDINATE: 788039 Y COORDINATE: 976674 ",
 "C10 INLINE NUMBER: MIN: 1 MAX: 345 TOTAL: 345 ",
 "C11 CROSSLINE NUMBER: MIN: 1 MAX: 188 TOTAL: 188 ",
 "C12 TOTAL NUMBER OF CDPS: 64860 BIN DIMENSION: 110' X 110' ",
 "C13 ",
 "C14 ",
 "C15 ",
 "C16 ",
 "C17 ",
 "C18 ",
 "C19 GENERAL SEGY INFORMATION ",
 "C20 RECORD LENGHT (MS): 3000 ",
 "C21 SAMPLE RATE (MS): 2.0 ",
 "C22 DATA FORMAT: 4 BYTE IBM FLOATING POINT ",
 "C23 BYTES 13- 16: CROSSLINE NUMBER (TRACE) ",
 "C24 BYTES 17- 20: INLINE NUMBER (LINE) ",
 "C25 BYTES 81- 84: CDP_X COORD ",
 "C26 BYTES 85- 88: CDP_Y COORD ",
 "C27 BYTES 181-184: INLINE NUMBER (LINE) ",
 "C28 BYTES 185-188: CROSSLINE NUMBER (TRACE) ",
 "C29 BYTES 189-192: CDP_X COORD ",
 "C30 BYTES 193-196: CDP_Y COORD ",
 "C31 ",
 "C32 ",
 "C33 ",
 "C34 ",
 "C35 ",
 "C36 Processed by: Excel Geophysical Services, Inc. ",
 "C37 8301 East Prentice Ave. Ste. 402 ",
 "C38 Englewood, Colorado 80111 ",
 "C39 (voice) 303.694.9629 (fax) 303.771.1646 ",
 "C40 END EBCDIC ",
 ]


def binary_header_teapot_dome() -> dict[str, int]:
 """Return the teapot dome expected binary header."""
 return {
 "job_id": 9999,
 "line_num": 9999,
 "reel_num": 1,
 "data_traces_per_ensemble": 188,
 "aux_traces_per_ensemble": 0,
 "sample_interval": 2000,
 "orig_sample_interval": 0,
 "samples_per_trace": 1501,
 "orig_samples_per_trace": 1501,
 "data_sample_format": 1,
 "ensemble_fold": 57,
 "trace_sorting_code": 4,
 "vertical_sum_code": 1,
 "sweep_freq_start": 0,
 "sweep_freq_end": 0,
 "sweep_length": 0,
 "sweep_type_code": 0,
 "sweep_trace_num": 0,
 "sweep_taper_start": 0,
 "sweep_taper_end": 0,
 "taper_type_code": 0,
 "correlated_data_code": 2,
 "binary_gain_code": 1,
 "amp_recovery_code": 4,
 "measurement_system_code": 2,
 "impulse_polarity_code": 1,
 "vibratory_polarity_code": 0,
 "fixed_length_trace_flag": 0,
 "num_extended_text_headers": 0,
 "segy_revision_major": 0,
 "segy_revision_minor": 0,
 }
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Serialize text and binary headers#600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tasansal
merged 14 commits into

from

Aug 12, 2025
 Merged


Serialize text and binary headers
#600
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


14 commits


Select commit
Hold shift + click to select a range
99a702b

Fix integration import tests


dmitriyrepin Aug 9, 2025
03251c0

mask_and_scale=False


dmitriyrepin Aug 11, 2025
c40be99

pre-commit


dmitriyrepin Aug 11, 2025
b7632ef

PR Review issues


dmitriyrepin Aug 11, 2025
9d0d40b

serialize-text-and-binary-headers


dmitriyrepin Aug 11, 2025
88c2865

remove dev test data


tasansal Aug 12, 2025
3a9e672

add back whitespace


tasansal Aug 12, 2025
28d07b4

revert import changes


tasansal Aug 12, 2025
9a74f15

fix attribute initialization in `_add_text_binary_headers`


tasansal Aug 12, 2025
c4928cc

Merge branch 'v1' into serialize-text-and-binary-headers


tasansal Aug 12, 2025
c6714cb

Add tests


dmitriyrepin Aug 12, 2025
024bed7

refactor: improve type annotations and docstrings in test utilities


tasansal Aug 12, 2025
3bb2d40

fix formatting


tasansal Aug 12, 2025
e04bb7c

remove redundant `str()` casting in `xr.open_dataset` calls


tasansal Aug 12, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


36 changes: 36 additions & 0 deletions
src/mdio/converters/segy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -280,6 +280,40 @@ def _populate_coordinates(
 return dataset, drop_vars_delayed


def _add_text_binary_headers(dataset: Dataset, segy_file: SegyFile) -> None:
 text_header = segy_file.text_header.splitlines()
 # Validate:
 # text_header this should be a 40-items array of strings with width of 80 characters.
 item_count = 40
 if len(text_header) != item_count:
 err = f"Invalid text header count: expected {item_count}, got {len(text_header)}"
 raise ValueError(err)
 char_count = 80
 for i, line in enumerate(text_header):
 if len(line) != char_count:
 err = f"Invalid text header {i} line length: expected {char_count}, got {len(line)}"
 raise ValueError(err)
 ext_text_header = segy_file.ext_text_header

 # If using SegyFile.ext_text_header this should be a minimum of 40 elements and must
 # capture all textual information (ensure text_header is a subset of ext_text_header).
 if ext_text_header is not None:
 for ext_hdr in ext_text_header:
 text_header.append(ext_hdr.splitlines())

 # Handle case where it may not have any metadata yet
 if dataset.metadata.attributes is None:
 dataset.attrs["attributes"] = {}

 # Update the attributes with the text and binary headers.
 dataset.metadata.attributes.update(
 {
 "textHeader": text_header,
 "binaryHeader": segy_file.binary_header.to_dict(),
 }
 )


def segy_to_mdio(
 segy_spec: SegySpec,
 mdio_template: AbstractDatasetTemplate,
Expand DownExpand Up@@ -324,6 +358,8 @@ def segy_to_mdio(
 name=mdio_template.name, sizes=shape, horizontal_coord_unit=horizontal_unit, headers=headers
 )

 _add_text_binary_headers(dataset=mdio_ds, segy_file=segy_file)

 xr_dataset: xr_Dataset = to_xarray_dataset(mdio_ds=mdio_ds)

 xr_dataset, drop_vars_delayed = _populate_coordinates(
Expand Down

44 changes: 24 additions & 20 deletions
tests/integration/test_segy_import_export.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
import xarray as xr
from segy import SegyFile
from segy.standards import get_segy_standard
from tests.integration.testing_data import binary_header_teapot_dome
from tests.integration.testing_data import text_header_teapot_dome
from tests.integration.testing_helpers import customize_segy_specs
from tests.integration.testing_helpers import get_inline_header_values
from tests.integration.testing_helpers import get_values
Expand DownExpand Up@@ -266,8 +268,8 @@ def test_3d_import(
 segy_to_mdio(
 segy_spec=segy_spec,
 mdio_template=TemplateRegistry().get("PostStack3DTime"),
 input_location=StorageLocation(segy_input.__str__()),
 output_location=StorageLocation(zarr_tmp.__str__()),
 input_location=StorageLocation(str(segy_input)),
 output_location=StorageLocation(str(zarr_tmp)),
 overwrite=True,
 )

Expand All@@ -278,11 +280,9 @@ class TestReader:

 def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "apiVersion": "1.0.0a1",
 "createdOn": "2025-08-06 16:21:54.747880+00:00",
Expand All@@ -297,13 +297,25 @@ def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 else:
 assert actual_attrs_json[key] == value

 attributes = ds.attrs["attributes"]
 assert attributes is not None

 # Validate attributes provided by the template
 assert attributes["surveyDimensionality"] == "3D"
 assert attributes["ensembleType"] == "line"
 assert attributes["processingStage"] == "post-stack"

 # Validate text header
 assert attributes["textHeader"] == text_header_teapot_dome()

 # Validate binary header
 assert attributes["binaryHeader"] == binary_header_teapot_dome()

 def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "count": 97354860,
 "sum": -8594.551666259766,
Expand All@@ -318,11 +330,9 @@ def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 def test_grid(self, zarr_tmp: Path) -> None:
 """Test validating MDIO variables."""
 # Load Xarray dataset from the MDIO file
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)

 # Note: in order to create the dataset we used the Time template, so the
 # sample dimension is called "time"
Expand DownExpand Up@@ -366,34 +376,28 @@ def test_grid(self, zarr_tmp: Path) -> None:

 def test_inline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 inlines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 inlines = ds["amplitude"][::75, :, :]
 mean, std = inlines.mean(), inlines.std()
 npt.assert_allclose([mean, std], [1.0555277e-04, 6.0027051e-01])

 def test_crossline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 crosslines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 xlines = ds["amplitude"][:, ::75, :]
 mean, std = xlines.mean(), xlines.std()

 npt.assert_allclose([mean, std], [-5.0329847e-05, 5.9406823e-01])

 def test_zslice(self, zarr_tmp: Path) -> None:
 """Read and compare every 225 z-slices' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 slices = ds["amplitude"][:, :, ::225]
 mean, std = slices.mean(), slices.std()
 npt.assert_allclose([mean, std], [0.005236923, 0.61279935])
Expand Down

84 changes: 84 additions & 0 deletions
tests/integration/testing_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
"""Integration tests data for teapot dome SEG-Y."""


def text_header_teapot_dome() -> list[str]:
 """Return the teapot dome expected text header."""
 return [
 "C 1 CLIENT: ROCKY MOUNTAIN OILFIELD TESTING CENTER ",
 "C 2 PROJECT: NAVAL PETROLEUM RESERVE #3 (TEAPOT DOME); NATRONA COUNTY, WYOMING ",
 "C 3 LINE: 3D ",
 "C 4 ",
 "C 5 THIS IS THE FILTERED POST STACK MIGRATION ",
 "C 6 ",
 "C 7 INLINE 1, XLINE 1: X COORDINATE: 788937 Y COORDINATE: 938845 ",
 "C 8 INLINE 1, XLINE 188: X COORDINATE: 809501 Y COORDINATE: 939333 ",
 "C 9 INLINE 188, XLINE 1: X COORDINATE: 788039 Y COORDINATE: 976674 ",
 "C10 INLINE NUMBER: MIN: 1 MAX: 345 TOTAL: 345 ",
 "C11 CROSSLINE NUMBER: MIN: 1 MAX: 188 TOTAL: 188 ",
 "C12 TOTAL NUMBER OF CDPS: 64860 BIN DIMENSION: 110' X 110' ",
 "C13 ",
 "C14 ",
 "C15 ",
 "C16 ",
 "C17 ",
 "C18 ",
 "C19 GENERAL SEGY INFORMATION ",
 "C20 RECORD LENGHT (MS): 3000 ",
 "C21 SAMPLE RATE (MS): 2.0 ",
 "C22 DATA FORMAT: 4 BYTE IBM FLOATING POINT ",
 "C23 BYTES 13- 16: CROSSLINE NUMBER (TRACE) ",
 "C24 BYTES 17- 20: INLINE NUMBER (LINE) ",
 "C25 BYTES 81- 84: CDP_X COORD ",
 "C26 BYTES 85- 88: CDP_Y COORD ",
 "C27 BYTES 181-184: INLINE NUMBER (LINE) ",
 "C28 BYTES 185-188: CROSSLINE NUMBER (TRACE) ",
 "C29 BYTES 189-192: CDP_X COORD ",
 "C30 BYTES 193-196: CDP_Y COORD ",
 "C31 ",
 "C32 ",
 "C33 ",
 "C34 ",
 "C35 ",
 "C36 Processed by: Excel Geophysical Services, Inc. ",
 "C37 8301 East Prentice Ave. Ste. 402 ",
 "C38 Englewood, Colorado 80111 ",
 "C39 (voice) 303.694.9629 (fax) 303.771.1646 ",
 "C40 END EBCDIC ",
 ]


def binary_header_teapot_dome() -> dict[str, int]:
 """Return the teapot dome expected binary header."""
 return {
 "job_id": 9999,
 "line_num": 9999,
 "reel_num": 1,
 "data_traces_per_ensemble": 188,
 "aux_traces_per_ensemble": 0,
 "sample_interval": 2000,
 "orig_sample_interval": 0,
 "samples_per_trace": 1501,
 "orig_samples_per_trace": 1501,
 "data_sample_format": 1,
 "ensemble_fold": 57,
 "trace_sorting_code": 4,
 "vertical_sum_code": 1,
 "sweep_freq_start": 0,
 "sweep_freq_end": 0,
 "sweep_length": 0,
 "sweep_type_code": 0,
 "sweep_trace_num": 0,
 "sweep_taper_start": 0,
 "sweep_taper_end": 0,
 "taper_type_code": 0,
 "correlated_data_code": 2,
 "binary_gain_code": 1,
 "amp_recovery_code": 4,
 "measurement_system_code": 2,
 "impulse_polarity_code": 1,
 "vibratory_polarity_code": 0,
 "fixed_length_trace_flag": 0,
 "num_extended_text_headers": 0,
 "segy_revision_major": 0,
 "segy_revision_minor": 0,
 }
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Serialize text and binary headers#600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tasansal
merged 14 commits into

from

Aug 12, 2025
 Merged


Serialize text and binary headers
#600
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


14 commits


Select commit
Hold shift + click to select a range
99a702b

Fix integration import tests


dmitriyrepin Aug 9, 2025
03251c0

mask_and_scale=False


dmitriyrepin Aug 11, 2025
c40be99

pre-commit


dmitriyrepin Aug 11, 2025
b7632ef

PR Review issues


dmitriyrepin Aug 11, 2025
9d0d40b

serialize-text-and-binary-headers


dmitriyrepin Aug 11, 2025
88c2865

remove dev test data


tasansal Aug 12, 2025
3a9e672

add back whitespace


tasansal Aug 12, 2025
28d07b4

revert import changes


tasansal Aug 12, 2025
9a74f15

fix attribute initialization in `_add_text_binary_headers`


tasansal Aug 12, 2025
c4928cc

Merge branch 'v1' into serialize-text-and-binary-headers


tasansal Aug 12, 2025
c6714cb

Add tests


dmitriyrepin Aug 12, 2025
024bed7

refactor: improve type annotations and docstrings in test utilities


tasansal Aug 12, 2025
3bb2d40

fix formatting


tasansal Aug 12, 2025
e04bb7c

remove redundant `str()` casting in `xr.open_dataset` calls


tasansal Aug 12, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


36 changes: 36 additions & 0 deletions
src/mdio/converters/segy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -280,6 +280,40 @@ def _populate_coordinates(
 return dataset, drop_vars_delayed


def _add_text_binary_headers(dataset: Dataset, segy_file: SegyFile) -> None:
 text_header = segy_file.text_header.splitlines()
 # Validate:
 # text_header this should be a 40-items array of strings with width of 80 characters.
 item_count = 40
 if len(text_header) != item_count:
 err = f"Invalid text header count: expected {item_count}, got {len(text_header)}"
 raise ValueError(err)
 char_count = 80
 for i, line in enumerate(text_header):
 if len(line) != char_count:
 err = f"Invalid text header {i} line length: expected {char_count}, got {len(line)}"
 raise ValueError(err)
 ext_text_header = segy_file.ext_text_header

 # If using SegyFile.ext_text_header this should be a minimum of 40 elements and must
 # capture all textual information (ensure text_header is a subset of ext_text_header).
 if ext_text_header is not None:
 for ext_hdr in ext_text_header:
 text_header.append(ext_hdr.splitlines())

 # Handle case where it may not have any metadata yet
 if dataset.metadata.attributes is None:
 dataset.attrs["attributes"] = {}

 # Update the attributes with the text and binary headers.
 dataset.metadata.attributes.update(
 {
 "textHeader": text_header,
 "binaryHeader": segy_file.binary_header.to_dict(),
 }
 )


def segy_to_mdio(
 segy_spec: SegySpec,
 mdio_template: AbstractDatasetTemplate,
Expand DownExpand Up@@ -324,6 +358,8 @@ def segy_to_mdio(
 name=mdio_template.name, sizes=shape, horizontal_coord_unit=horizontal_unit, headers=headers
 )

 _add_text_binary_headers(dataset=mdio_ds, segy_file=segy_file)

 xr_dataset: xr_Dataset = to_xarray_dataset(mdio_ds=mdio_ds)

 xr_dataset, drop_vars_delayed = _populate_coordinates(
Expand Down

44 changes: 24 additions & 20 deletions
tests/integration/test_segy_import_export.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
import xarray as xr
from segy import SegyFile
from segy.standards import get_segy_standard
from tests.integration.testing_data import binary_header_teapot_dome
from tests.integration.testing_data import text_header_teapot_dome
from tests.integration.testing_helpers import customize_segy_specs
from tests.integration.testing_helpers import get_inline_header_values
from tests.integration.testing_helpers import get_values
Expand DownExpand Up@@ -266,8 +268,8 @@ def test_3d_import(
 segy_to_mdio(
 segy_spec=segy_spec,
 mdio_template=TemplateRegistry().get("PostStack3DTime"),
 input_location=StorageLocation(segy_input.__str__()),
 output_location=StorageLocation(zarr_tmp.__str__()),
 input_location=StorageLocation(str(segy_input)),
 output_location=StorageLocation(str(zarr_tmp)),
 overwrite=True,
 )

Expand All@@ -278,11 +280,9 @@ class TestReader:

 def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "apiVersion": "1.0.0a1",
 "createdOn": "2025-08-06 16:21:54.747880+00:00",
Expand All@@ -297,13 +297,25 @@ def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 else:
 assert actual_attrs_json[key] == value

 attributes = ds.attrs["attributes"]
 assert attributes is not None

 # Validate attributes provided by the template
 assert attributes["surveyDimensionality"] == "3D"
 assert attributes["ensembleType"] == "line"
 assert attributes["processingStage"] == "post-stack"

 # Validate text header
 assert attributes["textHeader"] == text_header_teapot_dome()

 # Validate binary header
 assert attributes["binaryHeader"] == binary_header_teapot_dome()

 def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "count": 97354860,
 "sum": -8594.551666259766,
Expand All@@ -318,11 +330,9 @@ def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 def test_grid(self, zarr_tmp: Path) -> None:
 """Test validating MDIO variables."""
 # Load Xarray dataset from the MDIO file
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)

 # Note: in order to create the dataset we used the Time template, so the
 # sample dimension is called "time"
Expand DownExpand Up@@ -366,34 +376,28 @@ def test_grid(self, zarr_tmp: Path) -> None:

 def test_inline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 inlines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 inlines = ds["amplitude"][::75, :, :]
 mean, std = inlines.mean(), inlines.std()
 npt.assert_allclose([mean, std], [1.0555277e-04, 6.0027051e-01])

 def test_crossline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 crosslines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 xlines = ds["amplitude"][:, ::75, :]
 mean, std = xlines.mean(), xlines.std()

 npt.assert_allclose([mean, std], [-5.0329847e-05, 5.9406823e-01])

 def test_zslice(self, zarr_tmp: Path) -> None:
 """Read and compare every 225 z-slices' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 slices = ds["amplitude"][:, :, ::225]
 mean, std = slices.mean(), slices.std()
 npt.assert_allclose([mean, std], [0.005236923, 0.61279935])
Expand Down

84 changes: 84 additions & 0 deletions
tests/integration/testing_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
"""Integration tests data for teapot dome SEG-Y."""


def text_header_teapot_dome() -> list[str]:
 """Return the teapot dome expected text header."""
 return [
 "C 1 CLIENT: ROCKY MOUNTAIN OILFIELD TESTING CENTER ",
 "C 2 PROJECT: NAVAL PETROLEUM RESERVE #3 (TEAPOT DOME); NATRONA COUNTY, WYOMING ",
 "C 3 LINE: 3D ",
 "C 4 ",
 "C 5 THIS IS THE FILTERED POST STACK MIGRATION ",
 "C 6 ",
 "C 7 INLINE 1, XLINE 1: X COORDINATE: 788937 Y COORDINATE: 938845 ",
 "C 8 INLINE 1, XLINE 188: X COORDINATE: 809501 Y COORDINATE: 939333 ",
 "C 9 INLINE 188, XLINE 1: X COORDINATE: 788039 Y COORDINATE: 976674 ",
 "C10 INLINE NUMBER: MIN: 1 MAX: 345 TOTAL: 345 ",
 "C11 CROSSLINE NUMBER: MIN: 1 MAX: 188 TOTAL: 188 ",
 "C12 TOTAL NUMBER OF CDPS: 64860 BIN DIMENSION: 110' X 110' ",
 "C13 ",
 "C14 ",
 "C15 ",
 "C16 ",
 "C17 ",
 "C18 ",
 "C19 GENERAL SEGY INFORMATION ",
 "C20 RECORD LENGHT (MS): 3000 ",
 "C21 SAMPLE RATE (MS): 2.0 ",
 "C22 DATA FORMAT: 4 BYTE IBM FLOATING POINT ",
 "C23 BYTES 13- 16: CROSSLINE NUMBER (TRACE) ",
 "C24 BYTES 17- 20: INLINE NUMBER (LINE) ",
 "C25 BYTES 81- 84: CDP_X COORD ",
 "C26 BYTES 85- 88: CDP_Y COORD ",
 "C27 BYTES 181-184: INLINE NUMBER (LINE) ",
 "C28 BYTES 185-188: CROSSLINE NUMBER (TRACE) ",
 "C29 BYTES 189-192: CDP_X COORD ",
 "C30 BYTES 193-196: CDP_Y COORD ",
 "C31 ",
 "C32 ",
 "C33 ",
 "C34 ",
 "C35 ",
 "C36 Processed by: Excel Geophysical Services, Inc. ",
 "C37 8301 East Prentice Ave. Ste. 402 ",
 "C38 Englewood, Colorado 80111 ",
 "C39 (voice) 303.694.9629 (fax) 303.771.1646 ",
 "C40 END EBCDIC ",
 ]


def binary_header_teapot_dome() -> dict[str, int]:
 """Return the teapot dome expected binary header."""
 return {
 "job_id": 9999,
 "line_num": 9999,
 "reel_num": 1,
 "data_traces_per_ensemble": 188,
 "aux_traces_per_ensemble": 0,
 "sample_interval": 2000,
 "orig_sample_interval": 0,
 "samples_per_trace": 1501,
 "orig_samples_per_trace": 1501,
 "data_sample_format": 1,
 "ensemble_fold": 57,
 "trace_sorting_code": 4,
 "vertical_sum_code": 1,
 "sweep_freq_start": 0,
 "sweep_freq_end": 0,
 "sweep_length": 0,
 "sweep_type_code": 0,
 "sweep_trace_num": 0,
 "sweep_taper_start": 0,
 "sweep_taper_end": 0,
 "taper_type_code": 0,
 "correlated_data_code": 2,
 "binary_gain_code": 1,
 "amp_recovery_code": 4,
 "measurement_system_code": 2,
 "impulse_polarity_code": 1,
 "vibratory_polarity_code": 0,
 "fixed_length_trace_flag": 0,
 "num_extended_text_headers": 0,
 "segy_revision_major": 0,
 "segy_revision_minor": 0,
 }
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Serialize text and binary headers#600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tasansal
merged 14 commits into

from

Aug 12, 2025
 Merged


Serialize text and binary headers
#600
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


14 commits


Select commit
Hold shift + click to select a range
99a702b

Fix integration import tests


dmitriyrepin Aug 9, 2025
03251c0

mask_and_scale=False


dmitriyrepin Aug 11, 2025
c40be99

pre-commit


dmitriyrepin Aug 11, 2025
b7632ef

PR Review issues


dmitriyrepin Aug 11, 2025
9d0d40b

serialize-text-and-binary-headers


dmitriyrepin Aug 11, 2025
88c2865

remove dev test data


tasansal Aug 12, 2025
3a9e672

add back whitespace


tasansal Aug 12, 2025
28d07b4

revert import changes


tasansal Aug 12, 2025
9a74f15

fix attribute initialization in `_add_text_binary_headers`


tasansal Aug 12, 2025
c4928cc

Merge branch 'v1' into serialize-text-and-binary-headers


tasansal Aug 12, 2025
c6714cb

Add tests


dmitriyrepin Aug 12, 2025
024bed7

refactor: improve type annotations and docstrings in test utilities


tasansal Aug 12, 2025
3bb2d40

fix formatting


tasansal Aug 12, 2025
e04bb7c

remove redundant `str()` casting in `xr.open_dataset` calls


tasansal Aug 12, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


36 changes: 36 additions & 0 deletions
src/mdio/converters/segy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -280,6 +280,40 @@ def _populate_coordinates(
 return dataset, drop_vars_delayed


def _add_text_binary_headers(dataset: Dataset, segy_file: SegyFile) -> None:
 text_header = segy_file.text_header.splitlines()
 # Validate:
 # text_header this should be a 40-items array of strings with width of 80 characters.
 item_count = 40
 if len(text_header) != item_count:
 err = f"Invalid text header count: expected {item_count}, got {len(text_header)}"
 raise ValueError(err)
 char_count = 80
 for i, line in enumerate(text_header):
 if len(line) != char_count:
 err = f"Invalid text header {i} line length: expected {char_count}, got {len(line)}"
 raise ValueError(err)
 ext_text_header = segy_file.ext_text_header

 # If using SegyFile.ext_text_header this should be a minimum of 40 elements and must
 # capture all textual information (ensure text_header is a subset of ext_text_header).
 if ext_text_header is not None:
 for ext_hdr in ext_text_header:
 text_header.append(ext_hdr.splitlines())

 # Handle case where it may not have any metadata yet
 if dataset.metadata.attributes is None:
 dataset.attrs["attributes"] = {}

 # Update the attributes with the text and binary headers.
 dataset.metadata.attributes.update(
 {
 "textHeader": text_header,
 "binaryHeader": segy_file.binary_header.to_dict(),
 }
 )


def segy_to_mdio(
 segy_spec: SegySpec,
 mdio_template: AbstractDatasetTemplate,
Expand DownExpand Up@@ -324,6 +358,8 @@ def segy_to_mdio(
 name=mdio_template.name, sizes=shape, horizontal_coord_unit=horizontal_unit, headers=headers
 )

 _add_text_binary_headers(dataset=mdio_ds, segy_file=segy_file)

 xr_dataset: xr_Dataset = to_xarray_dataset(mdio_ds=mdio_ds)

 xr_dataset, drop_vars_delayed = _populate_coordinates(
Expand Down

44 changes: 24 additions & 20 deletions
tests/integration/test_segy_import_export.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
import xarray as xr
from segy import SegyFile
from segy.standards import get_segy_standard
from tests.integration.testing_data import binary_header_teapot_dome
from tests.integration.testing_data import text_header_teapot_dome
from tests.integration.testing_helpers import customize_segy_specs
from tests.integration.testing_helpers import get_inline_header_values
from tests.integration.testing_helpers import get_values
Expand DownExpand Up@@ -266,8 +268,8 @@ def test_3d_import(
 segy_to_mdio(
 segy_spec=segy_spec,
 mdio_template=TemplateRegistry().get("PostStack3DTime"),
 input_location=StorageLocation(segy_input.__str__()),
 output_location=StorageLocation(zarr_tmp.__str__()),
 input_location=StorageLocation(str(segy_input)),
 output_location=StorageLocation(str(zarr_tmp)),
 overwrite=True,
 )

Expand All@@ -278,11 +280,9 @@ class TestReader:

 def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "apiVersion": "1.0.0a1",
 "createdOn": "2025-08-06 16:21:54.747880+00:00",
Expand All@@ -297,13 +297,25 @@ def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 else:
 assert actual_attrs_json[key] == value

 attributes = ds.attrs["attributes"]
 assert attributes is not None

 # Validate attributes provided by the template
 assert attributes["surveyDimensionality"] == "3D"
 assert attributes["ensembleType"] == "line"
 assert attributes["processingStage"] == "post-stack"

 # Validate text header
 assert attributes["textHeader"] == text_header_teapot_dome()

 # Validate binary header
 assert attributes["binaryHeader"] == binary_header_teapot_dome()

 def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "count": 97354860,
 "sum": -8594.551666259766,
Expand All@@ -318,11 +330,9 @@ def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 def test_grid(self, zarr_tmp: Path) -> None:
 """Test validating MDIO variables."""
 # Load Xarray dataset from the MDIO file
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)

 # Note: in order to create the dataset we used the Time template, so the
 # sample dimension is called "time"
Expand DownExpand Up@@ -366,34 +376,28 @@ def test_grid(self, zarr_tmp: Path) -> None:

 def test_inline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 inlines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 inlines = ds["amplitude"][::75, :, :]
 mean, std = inlines.mean(), inlines.std()
 npt.assert_allclose([mean, std], [1.0555277e-04, 6.0027051e-01])

 def test_crossline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 crosslines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 xlines = ds["amplitude"][:, ::75, :]
 mean, std = xlines.mean(), xlines.std()

 npt.assert_allclose([mean, std], [-5.0329847e-05, 5.9406823e-01])

 def test_zslice(self, zarr_tmp: Path) -> None:
 """Read and compare every 225 z-slices' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 slices = ds["amplitude"][:, :, ::225]
 mean, std = slices.mean(), slices.std()
 npt.assert_allclose([mean, std], [0.005236923, 0.61279935])
Expand Down

84 changes: 84 additions & 0 deletions
tests/integration/testing_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
"""Integration tests data for teapot dome SEG-Y."""


def text_header_teapot_dome() -> list[str]:
 """Return the teapot dome expected text header."""
 return [
 "C 1 CLIENT: ROCKY MOUNTAIN OILFIELD TESTING CENTER ",
 "C 2 PROJECT: NAVAL PETROLEUM RESERVE #3 (TEAPOT DOME); NATRONA COUNTY, WYOMING ",
 "C 3 LINE: 3D ",
 "C 4 ",
 "C 5 THIS IS THE FILTERED POST STACK MIGRATION ",
 "C 6 ",
 "C 7 INLINE 1, XLINE 1: X COORDINATE: 788937 Y COORDINATE: 938845 ",
 "C 8 INLINE 1, XLINE 188: X COORDINATE: 809501 Y COORDINATE: 939333 ",
 "C 9 INLINE 188, XLINE 1: X COORDINATE: 788039 Y COORDINATE: 976674 ",
 "C10 INLINE NUMBER: MIN: 1 MAX: 345 TOTAL: 345 ",
 "C11 CROSSLINE NUMBER: MIN: 1 MAX: 188 TOTAL: 188 ",
 "C12 TOTAL NUMBER OF CDPS: 64860 BIN DIMENSION: 110' X 110' ",
 "C13 ",
 "C14 ",
 "C15 ",
 "C16 ",
 "C17 ",
 "C18 ",
 "C19 GENERAL SEGY INFORMATION ",
 "C20 RECORD LENGHT (MS): 3000 ",
 "C21 SAMPLE RATE (MS): 2.0 ",
 "C22 DATA FORMAT: 4 BYTE IBM FLOATING POINT ",
 "C23 BYTES 13- 16: CROSSLINE NUMBER (TRACE) ",
 "C24 BYTES 17- 20: INLINE NUMBER (LINE) ",
 "C25 BYTES 81- 84: CDP_X COORD ",
 "C26 BYTES 85- 88: CDP_Y COORD ",
 "C27 BYTES 181-184: INLINE NUMBER (LINE) ",
 "C28 BYTES 185-188: CROSSLINE NUMBER (TRACE) ",
 "C29 BYTES 189-192: CDP_X COORD ",
 "C30 BYTES 193-196: CDP_Y COORD ",
 "C31 ",
 "C32 ",
 "C33 ",
 "C34 ",
 "C35 ",
 "C36 Processed by: Excel Geophysical Services, Inc. ",
 "C37 8301 East Prentice Ave. Ste. 402 ",
 "C38 Englewood, Colorado 80111 ",
 "C39 (voice) 303.694.9629 (fax) 303.771.1646 ",
 "C40 END EBCDIC ",
 ]


def binary_header_teapot_dome() -> dict[str, int]:
 """Return the teapot dome expected binary header."""
 return {
 "job_id": 9999,
 "line_num": 9999,
 "reel_num": 1,
 "data_traces_per_ensemble": 188,
 "aux_traces_per_ensemble": 0,
 "sample_interval": 2000,
 "orig_sample_interval": 0,
 "samples_per_trace": 1501,
 "orig_samples_per_trace": 1501,
 "data_sample_format": 1,
 "ensemble_fold": 57,
 "trace_sorting_code": 4,
 "vertical_sum_code": 1,
 "sweep_freq_start": 0,
 "sweep_freq_end": 0,
 "sweep_length": 0,
 "sweep_type_code": 0,
 "sweep_trace_num": 0,
 "sweep_taper_start": 0,
 "sweep_taper_end": 0,
 "taper_type_code": 0,
 "correlated_data_code": 2,
 "binary_gain_code": 1,
 "amp_recovery_code": 4,
 "measurement_system_code": 2,
 "impulse_polarity_code": 1,
 "vibratory_polarity_code": 0,
 "fixed_length_trace_flag": 0,
 "num_extended_text_headers": 0,
 "segy_revision_major": 0,
 "segy_revision_minor": 0,
 }
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Serialize text and binary headers#600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tasansal
merged 14 commits into

from

Aug 12, 2025
 Merged


Serialize text and binary headers
#600
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


14 commits


Select commit
Hold shift + click to select a range
99a702b

Fix integration import tests


dmitriyrepin Aug 9, 2025
03251c0

mask_and_scale=False


dmitriyrepin Aug 11, 2025
c40be99

pre-commit


dmitriyrepin Aug 11, 2025
b7632ef

PR Review issues


dmitriyrepin Aug 11, 2025
9d0d40b

serialize-text-and-binary-headers


dmitriyrepin Aug 11, 2025
88c2865

remove dev test data


tasansal Aug 12, 2025
3a9e672

add back whitespace


tasansal Aug 12, 2025
28d07b4

revert import changes


tasansal Aug 12, 2025
9a74f15

fix attribute initialization in `_add_text_binary_headers`


tasansal Aug 12, 2025
c4928cc

Merge branch 'v1' into serialize-text-and-binary-headers


tasansal Aug 12, 2025
c6714cb

Add tests


dmitriyrepin Aug 12, 2025
024bed7

refactor: improve type annotations and docstrings in test utilities


tasansal Aug 12, 2025
3bb2d40

fix formatting


tasansal Aug 12, 2025
e04bb7c

remove redundant `str()` casting in `xr.open_dataset` calls


tasansal Aug 12, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


36 changes: 36 additions & 0 deletions
src/mdio/converters/segy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -280,6 +280,40 @@ def _populate_coordinates(
 return dataset, drop_vars_delayed


def _add_text_binary_headers(dataset: Dataset, segy_file: SegyFile) -> None:
 text_header = segy_file.text_header.splitlines()
 # Validate:
 # text_header this should be a 40-items array of strings with width of 80 characters.
 item_count = 40
 if len(text_header) != item_count:
 err = f"Invalid text header count: expected {item_count}, got {len(text_header)}"
 raise ValueError(err)
 char_count = 80
 for i, line in enumerate(text_header):
 if len(line) != char_count:
 err = f"Invalid text header {i} line length: expected {char_count}, got {len(line)}"
 raise ValueError(err)
 ext_text_header = segy_file.ext_text_header

 # If using SegyFile.ext_text_header this should be a minimum of 40 elements and must
 # capture all textual information (ensure text_header is a subset of ext_text_header).
 if ext_text_header is not None:
 for ext_hdr in ext_text_header:
 text_header.append(ext_hdr.splitlines())

 # Handle case where it may not have any metadata yet
 if dataset.metadata.attributes is None:
 dataset.attrs["attributes"] = {}

 # Update the attributes with the text and binary headers.
 dataset.metadata.attributes.update(
 {
 "textHeader": text_header,
 "binaryHeader": segy_file.binary_header.to_dict(),
 }
 )


def segy_to_mdio(
 segy_spec: SegySpec,
 mdio_template: AbstractDatasetTemplate,
Expand DownExpand Up@@ -324,6 +358,8 @@ def segy_to_mdio(
 name=mdio_template.name, sizes=shape, horizontal_coord_unit=horizontal_unit, headers=headers
 )

 _add_text_binary_headers(dataset=mdio_ds, segy_file=segy_file)

 xr_dataset: xr_Dataset = to_xarray_dataset(mdio_ds=mdio_ds)

 xr_dataset, drop_vars_delayed = _populate_coordinates(
Expand Down

44 changes: 24 additions & 20 deletions
tests/integration/test_segy_import_export.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
import xarray as xr
from segy import SegyFile
from segy.standards import get_segy_standard
from tests.integration.testing_data import binary_header_teapot_dome
from tests.integration.testing_data import text_header_teapot_dome
from tests.integration.testing_helpers import customize_segy_specs
from tests.integration.testing_helpers import get_inline_header_values
from tests.integration.testing_helpers import get_values
Expand DownExpand Up@@ -266,8 +268,8 @@ def test_3d_import(
 segy_to_mdio(
 segy_spec=segy_spec,
 mdio_template=TemplateRegistry().get("PostStack3DTime"),
 input_location=StorageLocation(segy_input.__str__()),
 output_location=StorageLocation(zarr_tmp.__str__()),
 input_location=StorageLocation(str(segy_input)),
 output_location=StorageLocation(str(zarr_tmp)),
 overwrite=True,
 )

Expand All@@ -278,11 +280,9 @@ class TestReader:

 def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "apiVersion": "1.0.0a1",
 "createdOn": "2025-08-06 16:21:54.747880+00:00",
Expand All@@ -297,13 +297,25 @@ def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 else:
 assert actual_attrs_json[key] == value

 attributes = ds.attrs["attributes"]
 assert attributes is not None

 # Validate attributes provided by the template
 assert attributes["surveyDimensionality"] == "3D"
 assert attributes["ensembleType"] == "line"
 assert attributes["processingStage"] == "post-stack"

 # Validate text header
 assert attributes["textHeader"] == text_header_teapot_dome()

 # Validate binary header
 assert attributes["binaryHeader"] == binary_header_teapot_dome()

 def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "count": 97354860,
 "sum": -8594.551666259766,
Expand All@@ -318,11 +330,9 @@ def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 def test_grid(self, zarr_tmp: Path) -> None:
 """Test validating MDIO variables."""
 # Load Xarray dataset from the MDIO file
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)

 # Note: in order to create the dataset we used the Time template, so the
 # sample dimension is called "time"
Expand DownExpand Up@@ -366,34 +376,28 @@ def test_grid(self, zarr_tmp: Path) -> None:

 def test_inline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 inlines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 inlines = ds["amplitude"][::75, :, :]
 mean, std = inlines.mean(), inlines.std()
 npt.assert_allclose([mean, std], [1.0555277e-04, 6.0027051e-01])

 def test_crossline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 crosslines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 xlines = ds["amplitude"][:, ::75, :]
 mean, std = xlines.mean(), xlines.std()

 npt.assert_allclose([mean, std], [-5.0329847e-05, 5.9406823e-01])

 def test_zslice(self, zarr_tmp: Path) -> None:
 """Read and compare every 225 z-slices' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 slices = ds["amplitude"][:, :, ::225]
 mean, std = slices.mean(), slices.std()
 npt.assert_allclose([mean, std], [0.005236923, 0.61279935])
Expand Down

84 changes: 84 additions & 0 deletions
tests/integration/testing_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
"""Integration tests data for teapot dome SEG-Y."""


def text_header_teapot_dome() -> list[str]:
 """Return the teapot dome expected text header."""
 return [
 "C 1 CLIENT: ROCKY MOUNTAIN OILFIELD TESTING CENTER ",
 "C 2 PROJECT: NAVAL PETROLEUM RESERVE #3 (TEAPOT DOME); NATRONA COUNTY, WYOMING ",
 "C 3 LINE: 3D ",
 "C 4 ",
 "C 5 THIS IS THE FILTERED POST STACK MIGRATION ",
 "C 6 ",
 "C 7 INLINE 1, XLINE 1: X COORDINATE: 788937 Y COORDINATE: 938845 ",
 "C 8 INLINE 1, XLINE 188: X COORDINATE: 809501 Y COORDINATE: 939333 ",
 "C 9 INLINE 188, XLINE 1: X COORDINATE: 788039 Y COORDINATE: 976674 ",
 "C10 INLINE NUMBER: MIN: 1 MAX: 345 TOTAL: 345 ",
 "C11 CROSSLINE NUMBER: MIN: 1 MAX: 188 TOTAL: 188 ",
 "C12 TOTAL NUMBER OF CDPS: 64860 BIN DIMENSION: 110' X 110' ",
 "C13 ",
 "C14 ",
 "C15 ",
 "C16 ",
 "C17 ",
 "C18 ",
 "C19 GENERAL SEGY INFORMATION ",
 "C20 RECORD LENGHT (MS): 3000 ",
 "C21 SAMPLE RATE (MS): 2.0 ",
 "C22 DATA FORMAT: 4 BYTE IBM FLOATING POINT ",
 "C23 BYTES 13- 16: CROSSLINE NUMBER (TRACE) ",
 "C24 BYTES 17- 20: INLINE NUMBER (LINE) ",
 "C25 BYTES 81- 84: CDP_X COORD ",
 "C26 BYTES 85- 88: CDP_Y COORD ",
 "C27 BYTES 181-184: INLINE NUMBER (LINE) ",
 "C28 BYTES 185-188: CROSSLINE NUMBER (TRACE) ",
 "C29 BYTES 189-192: CDP_X COORD ",
 "C30 BYTES 193-196: CDP_Y COORD ",
 "C31 ",
 "C32 ",
 "C33 ",
 "C34 ",
 "C35 ",
 "C36 Processed by: Excel Geophysical Services, Inc. ",
 "C37 8301 East Prentice Ave. Ste. 402 ",
 "C38 Englewood, Colorado 80111 ",
 "C39 (voice) 303.694.9629 (fax) 303.771.1646 ",
 "C40 END EBCDIC ",
 ]


def binary_header_teapot_dome() -> dict[str, int]:
 """Return the teapot dome expected binary header."""
 return {
 "job_id": 9999,
 "line_num": 9999,
 "reel_num": 1,
 "data_traces_per_ensemble": 188,
 "aux_traces_per_ensemble": 0,
 "sample_interval": 2000,
 "orig_sample_interval": 0,
 "samples_per_trace": 1501,
 "orig_samples_per_trace": 1501,
 "data_sample_format": 1,
 "ensemble_fold": 57,
 "trace_sorting_code": 4,
 "vertical_sum_code": 1,
 "sweep_freq_start": 0,
 "sweep_freq_end": 0,
 "sweep_length": 0,
 "sweep_type_code": 0,
 "sweep_trace_num": 0,
 "sweep_taper_start": 0,
 "sweep_taper_end": 0,
 "taper_type_code": 0,
 "correlated_data_code": 2,
 "binary_gain_code": 1,
 "amp_recovery_code": 4,
 "measurement_system_code": 2,
 "impulse_polarity_code": 1,
 "vibratory_polarity_code": 0,
 "fixed_length_trace_flag": 0,
 "num_extended_text_headers": 0,
 "segy_revision_major": 0,
 "segy_revision_minor": 0,
 }
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Serialize text and binary headers#600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tasansal
merged 14 commits into

from

Aug 12, 2025
 Merged


Serialize text and binary headers
#600
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


14 commits


Select commit
Hold shift + click to select a range
99a702b

Fix integration import tests


dmitriyrepin Aug 9, 2025
03251c0

mask_and_scale=False


dmitriyrepin Aug 11, 2025
c40be99

pre-commit


dmitriyrepin Aug 11, 2025
b7632ef

PR Review issues


dmitriyrepin Aug 11, 2025
9d0d40b

serialize-text-and-binary-headers


dmitriyrepin Aug 11, 2025
88c2865

remove dev test data


tasansal Aug 12, 2025
3a9e672

add back whitespace


tasansal Aug 12, 2025
28d07b4

revert import changes


tasansal Aug 12, 2025
9a74f15

fix attribute initialization in `_add_text_binary_headers`


tasansal Aug 12, 2025
c4928cc

Merge branch 'v1' into serialize-text-and-binary-headers


tasansal Aug 12, 2025
c6714cb

Add tests


dmitriyrepin Aug 12, 2025
024bed7

refactor: improve type annotations and docstrings in test utilities


tasansal Aug 12, 2025
3bb2d40

fix formatting


tasansal Aug 12, 2025
e04bb7c

remove redundant `str()` casting in `xr.open_dataset` calls


tasansal Aug 12, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


36 changes: 36 additions & 0 deletions
src/mdio/converters/segy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -280,6 +280,40 @@ def _populate_coordinates(
 return dataset, drop_vars_delayed


def _add_text_binary_headers(dataset: Dataset, segy_file: SegyFile) -> None:
 text_header = segy_file.text_header.splitlines()
 # Validate:
 # text_header this should be a 40-items array of strings with width of 80 characters.
 item_count = 40
 if len(text_header) != item_count:
 err = f"Invalid text header count: expected {item_count}, got {len(text_header)}"
 raise ValueError(err)
 char_count = 80
 for i, line in enumerate(text_header):
 if len(line) != char_count:
 err = f"Invalid text header {i} line length: expected {char_count}, got {len(line)}"
 raise ValueError(err)
 ext_text_header = segy_file.ext_text_header

 # If using SegyFile.ext_text_header this should be a minimum of 40 elements and must
 # capture all textual information (ensure text_header is a subset of ext_text_header).
 if ext_text_header is not None:
 for ext_hdr in ext_text_header:
 text_header.append(ext_hdr.splitlines())

 # Handle case where it may not have any metadata yet
 if dataset.metadata.attributes is None:
 dataset.attrs["attributes"] = {}

 # Update the attributes with the text and binary headers.
 dataset.metadata.attributes.update(
 {
 "textHeader": text_header,
 "binaryHeader": segy_file.binary_header.to_dict(),
 }
 )


def segy_to_mdio(
 segy_spec: SegySpec,
 mdio_template: AbstractDatasetTemplate,
Expand DownExpand Up@@ -324,6 +358,8 @@ def segy_to_mdio(
 name=mdio_template.name, sizes=shape, horizontal_coord_unit=horizontal_unit, headers=headers
 )

 _add_text_binary_headers(dataset=mdio_ds, segy_file=segy_file)

 xr_dataset: xr_Dataset = to_xarray_dataset(mdio_ds=mdio_ds)

 xr_dataset, drop_vars_delayed = _populate_coordinates(
Expand Down

44 changes: 24 additions & 20 deletions
tests/integration/test_segy_import_export.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,8 @@
import xarray as xr
from segy import SegyFile
from segy.standards import get_segy_standard
from tests.integration.testing_data import binary_header_teapot_dome
from tests.integration.testing_data import text_header_teapot_dome
from tests.integration.testing_helpers import customize_segy_specs
from tests.integration.testing_helpers import get_inline_header_values
from tests.integration.testing_helpers import get_values
Expand DownExpand Up@@ -266,8 +268,8 @@ def test_3d_import(
 segy_to_mdio(
 segy_spec=segy_spec,
 mdio_template=TemplateRegistry().get("PostStack3DTime"),
 input_location=StorageLocation(segy_input.__str__()),
 output_location=StorageLocation(zarr_tmp.__str__()),
 input_location=StorageLocation(str(segy_input)),
 output_location=StorageLocation(str(zarr_tmp)),
 overwrite=True,
 )

Expand All@@ -278,11 +280,9 @@ class TestReader:

 def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "apiVersion": "1.0.0a1",
 "createdOn": "2025-08-06 16:21:54.747880+00:00",
Expand All@@ -297,13 +297,25 @@ def test_meta_dataset_read(self, zarr_tmp: Path) -> None:
 else:
 assert actual_attrs_json[key] == value

 attributes = ds.attrs["attributes"]
 assert attributes is not None

 # Validate attributes provided by the template
 assert attributes["surveyDimensionality"] == "3D"
 assert attributes["ensembleType"] == "line"
 assert attributes["processingStage"] == "post-stack"

 # Validate text header
 assert attributes["textHeader"] == text_header_teapot_dome()

 # Validate binary header
 assert attributes["binaryHeader"] == binary_header_teapot_dome()

 def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 """Metadata reading tests."""
 path = zarr_tmp.__str__()
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 expected_attrs = {
 "count": 97354860,
 "sum": -8594.551666259766,
Expand All@@ -318,11 +330,9 @@ def test_meta_variable_read(self, zarr_tmp: Path) -> None:
 def test_grid(self, zarr_tmp: Path) -> None:
 """Test validating MDIO variables."""
 # Load Xarray dataset from the MDIO file
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)

 # Note: in order to create the dataset we used the Time template, so the
 # sample dimension is called "time"
Expand DownExpand Up@@ -366,34 +376,28 @@ def test_grid(self, zarr_tmp: Path) -> None:

 def test_inline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 inlines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 inlines = ds["amplitude"][::75, :, :]
 mean, std = inlines.mean(), inlines.std()
 npt.assert_allclose([mean, std], [1.0555277e-04, 6.0027051e-01])

 def test_crossline(self, zarr_tmp: Path) -> None:
 """Read and compare every 75 crosslines' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 xlines = ds["amplitude"][:, ::75, :]
 mean, std = xlines.mean(), xlines.std()

 npt.assert_allclose([mean, std], [-5.0329847e-05, 5.9406823e-01])

 def test_zslice(self, zarr_tmp: Path) -> None:
 """Read and compare every 225 z-slices' mean and std. dev."""
 path = zarr_tmp.__str__()
 # path = "/tmp/pytest-of-vscode/my-mdio/mdio0"
 # NOTE: If mask_and_scale is not set,
 # Xarray will convert int to float and replace _FillValue with NaN
 ds = xr.open_dataset(path, engine="zarr", mask_and_scale=False)
 ds = xr.open_dataset(zarr_tmp, engine="zarr", mask_and_scale=False)
 slices = ds["amplitude"][:, :, ::225]
 mean, std = slices.mean(), slices.std()
 npt.assert_allclose([mean, std], [0.005236923, 0.61279935])
Expand Down

84 changes: 84 additions & 0 deletions
tests/integration/testing_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
"""Integration tests data for teapot dome SEG-Y."""


def text_header_teapot_dome() -> list[str]:
 """Return the teapot dome expected text header."""
 return [
 "C 1 CLIENT: ROCKY MOUNTAIN OILFIELD TESTING CENTER ",
 "C 2 PROJECT: NAVAL PETROLEUM RESERVE #3 (TEAPOT DOME); NATRONA COUNTY, WYOMING ",
 "C 3 LINE: 3D ",
 "C 4 ",
 "C 5 THIS IS THE FILTERED POST STACK MIGRATION ",
 "C 6 ",
 "C 7 INLINE 1, XLINE 1: X COORDINATE: 788937 Y COORDINATE: 938845 ",
 "C 8 INLINE 1, XLINE 188: X COORDINATE: 809501 Y COORDINATE: 939333 ",
 "C 9 INLINE 188, XLINE 1: X COORDINATE: 788039 Y COORDINATE: 976674 ",
 "C10 INLINE NUMBER: MIN: 1 MAX: 345 TOTAL: 345 ",
 "C11 CROSSLINE NUMBER: MIN: 1 MAX: 188 TOTAL: 188 ",
 "C12 TOTAL NUMBER OF CDPS: 64860 BIN DIMENSION: 110' X 110' ",
 "C13 ",
 "C14 ",
 "C15 ",
 "C16 ",
 "C17 ",
 "C18 ",
 "C19 GENERAL SEGY INFORMATION ",
 "C20 RECORD LENGHT (MS): 3000 ",
 "C21 SAMPLE RATE (MS): 2.0 ",
 "C22 DATA FORMAT: 4 BYTE IBM FLOATING POINT ",
 "C23 BYTES 13- 16: CROSSLINE NUMBER (TRACE) ",
 "C24 BYTES 17- 20: INLINE NUMBER (LINE) ",
 "C25 BYTES 81- 84: CDP_X COORD ",
 "C26 BYTES 85- 88: CDP_Y COORD ",
 "C27 BYTES 181-184: INLINE NUMBER (LINE) ",
 "C28 BYTES 185-188: CROSSLINE NUMBER (TRACE) ",
 "C29 BYTES 189-192: CDP_X COORD ",
 "C30 BYTES 193-196: CDP_Y COORD ",
 "C31 ",
 "C32 ",
 "C33 ",
 "C34 ",
 "C35 ",
 "C36 Processed by: Excel Geophysical Services, Inc. ",
 "C37 8301 East Prentice Ave. Ste. 402 ",
 "C38 Englewood, Colorado 80111 ",
 "C39 (voice) 303.694.9629 (fax) 303.771.1646 ",
 "C40 END EBCDIC ",
 ]


def binary_header_teapot_dome() -> dict[str, int]:
 """Return the teapot dome expected binary header."""
 return {
 "job_id": 9999,
 "line_num": 9999,
 "reel_num": 1,
 "data_traces_per_ensemble": 188,
 "aux_traces_per_ensemble": 0,
 "sample_interval": 2000,
 "orig_sample_interval": 0,
 "samples_per_trace": 1501,
 "orig_samples_per_trace": 1501,
 "data_sample_format": 1,
 "ensemble_fold": 57,
 "trace_sorting_code": 4,
 "vertical_sum_code": 1,
 "sweep_freq_start": 0,
 "sweep_freq_end": 0,
 "sweep_length": 0,
 "sweep_type_code": 0,
 "sweep_trace_num": 0,
 "sweep_taper_start": 0,
 "sweep_taper_end": 0,
 "taper_type_code": 0,
 "correlated_data_code": 2,
 "binary_gain_code": 1,
 "amp_recovery_code": 4,
 "measurement_system_code": 2,
 "impulse_polarity_code": 1,
 "vibratory_polarity_code": 0,
 "fixed_length_trace_flag": 0,
 "num_extended_text_headers": 0,
 "segy_revision_major": 0,
 "segy_revision_minor": 0,
 }
Loading