# file: src/tea_data_file_conversion/processor.py
r"""Processor module for fixed\-width and delimited (CSV) file conversion.
This module provides functions to:
\- Load and validate YAML schema configurations for either format.
\- Process fixed\-width or delimited files into structured DataFrame objects.
\- Export template YAML schema files.
\- Convert CSV files into YAML schema files interactively.
"""
import os
import shutil
import sys
from collections import Counter
import importlib_resources # Used to locate package data.
import pandas as pd
import yaml
[docs]
def load_yaml_config(file_path):
"""
Load a YAML configuration file for processing.
Parameters
----------
file_path : str
The path to the YAML configuration file.
Returns
-------
dict
The parsed YAML configuration.
Raises
------
ValueError
If there is an error parsing the YAML file.
"""
try:
with open(file_path) as f:
config = yaml.safe_load(f)
return config
except yaml.YAMLError as ye:
# Raise an error with details of parsing issues.
raise ValueError(f"Error parsing YAML file {file_path}: {ye}") from ye
DELIMITED_FIELD_KEY = "source_column"
FIXED_WIDTH_FIELD_KEYS = ("start", "end")
def _field_shape(field):
"""
Determine which schema shape a single field entry uses.
Parameters
----------
field : dict
A single entry from a schema's 'fields' list.
Returns
-------
str or None
"delimited" if the field names a source column, "fixed_width" if it
gives both boundaries, otherwise None.
"""
if DELIMITED_FIELD_KEY in field:
return "delimited"
if all(key in field for key in FIXED_WIDTH_FIELD_KEYS):
return "fixed_width"
return None
[docs]
def schema_shape(config):
"""
Report the shape of a schema that has already passed validation.
Parameters
----------
config : dict
A validated schema configuration.
Returns
-------
str
Either "delimited" or "fixed_width".
"""
return _field_shape(config["fields"][0])
[docs]
def validate_yaml_config(config, file_path):
"""
Validate the structure of the YAML configuration.
The configuration must be a dictionary containing a key 'fields' mapping to a
non-empty list. Every field must contain 'output_field', plus either
'source_column' (delimited schemas) or both 'start' and 'end' (fixed-width
schemas). A single schema must use one shape throughout. Delimited schemas
must additionally have unique 'output_field' values: process_delimited_file
renames columns by this name, and a duplicate silently fans a single source
column out into every column sharing that name. Fixed-width schemas are exempt
because process_fixed_width_file de-duplicates output names itself at runtime,
and shipped schemas rely on that behavior.
Parameters
----------
config : dict
The YAML configuration dictionary.
file_path : str
File path used for reporting in error messages.
Raises
------
ValueError
If the configuration does not adhere to the expected schema.
"""
if not isinstance(config, dict):
raise ValueError(f"YAML file {file_path} should be a dictionary at the top level.")
if "fields" not in config:
raise ValueError(f"YAML file {file_path} is missing the required key 'fields'.")
if not isinstance(config["fields"], list):
raise ValueError(f"YAML file {file_path} key 'fields' should be a list.")
if not config["fields"]:
raise ValueError(f"YAML file {file_path} key 'fields' must not be empty.")
shapes = set()
for index, field in enumerate(config["fields"]):
if not isinstance(field, dict):
raise ValueError(f"YAML file {file_path}, field at index {index} is not a dictionary.")
shape = _field_shape(field)
if shape is None:
raise ValueError(
f"YAML file {file_path}, field at index {index} must define either "
f"'{DELIMITED_FIELD_KEY}' (delimited) or both 'start' and 'end' (fixed width)."
)
shapes.add(shape)
if "output_field" not in field:
raise ValueError(f"YAML file {file_path}, field at index {index} is missing required key 'output_field'.")
if not isinstance(field["output_field"], str):
raise ValueError(f"YAML file {file_path}, field at index {index} key 'output_field' must be a string.")
if shape == "fixed_width":
for key in FIXED_WIDTH_FIELD_KEYS:
if not isinstance(field[key], int):
raise ValueError(f"YAML file {file_path}, field at index {index} key '{key}' must be an integer.")
elif not isinstance(field[DELIMITED_FIELD_KEY], str):
raise ValueError(
f"YAML file {file_path}, field at index {index} key '{DELIMITED_FIELD_KEY}' must be a string."
)
if "keep" in field and not isinstance(field["keep"], bool):
raise ValueError(f"YAML file {file_path}, field at index {index} key 'keep' must be a boolean.")
if len(shapes) > 1:
raise ValueError(
f"YAML file {file_path} mixes fixed-width fields ('start'/'end') and delimited fields "
f"('{DELIMITED_FIELD_KEY}'); a schema must use one shape throughout."
)
# Delimited schemas have no runtime de-duplication of output_field the way
# process_fixed_width_file does, so a collision here would silently fan a
# single source column out into every column sharing its output name.
if shapes == {"delimited"}:
output_fields = [field["output_field"] for field in config["fields"]]
duplicates = sorted(name for name, count in Counter(output_fields).items() if count > 1)
if duplicates:
raise ValueError(
f"YAML file {file_path} has duplicate 'output_field' values, which is not allowed in a "
f"delimited schema: {', '.join(duplicates)}."
)
[docs]
def process_fixed_width_file(input_file, schema_config, skip_header=False, filter_columns=False):
r"""
Process a fixed\-width file using the provided YAML schema configuration.
It determines column boundaries based on the schema, reads the file using pandas,
and applies optional filtering to only return columns marked to be kept.
Parameters
----------
input_file : str
The path to the fixed\-width text file.
schema_config : dict
Schema configuration dictionary with field definitions.
skip_header : bool, optional
Skip the header row if True (default is False).
filter_columns : bool, optional
If True, return only DataFrame columns that are marked with "keep": true.
Returns
-------
pd.DataFrame
DataFrame with the processed data.
"""
fields = schema_config["fields"]
colspecs = [] # List of tuples defining start and end positions for each field.
col_names = [] # List of column names derived from the schema.
keep_columns = [] # Track columns flagged to be retained.
for field in fields:
# Adjust the start position because the schema uses 1-based indexing.
start = field["start"] - 1
end = field["end"]
colspecs.append((start, end))
# Use 'mapped_field_name' when filtering columns if available.
if filter_columns:
col_name = (
field["mapped_field_name"] if not pd.isna(field.get("mapped_field_name")) else field["output_field"]
)
else:
col_name = field["output_field"]
col_names.append(col_name)
if field.get("keep", False):
keep_columns.append(col_name)
# Ensure each column name is unique by appending a counter if needed.
unique_col_names = []
for col_name in col_names:
if col_name in unique_col_names:
count = 1
new_col_name = f"{col_name}_{count}"
while new_col_name in unique_col_names:
count += 1
new_col_name = f"{col_name}_{count}"
unique_col_names.append(new_col_name)
else:
unique_col_names.append(col_name)
# Read the fixed\-width file into a DataFrame.
df = pd.read_fwf(input_file, colspecs=colspecs, header=None, names=unique_col_names, dtype=str)
if filter_columns:
df = df[keep_columns]
return df
WARN_NAME_LIMIT = 20 # Most column names to name individually in a warning.
def _format_column_list(columns):
"""
Render a column-name list for a warning, truncated to WARN_NAME_LIMIT names.
Parameters
----------
columns : list of str
Column names to render.
Returns
-------
str
Comma-separated names, with a trailing count if the list was truncated.
"""
shown = ", ".join(columns[:WARN_NAME_LIMIT])
if len(columns) > WARN_NAME_LIMIT:
shown += f", ... ({len(columns) - WARN_NAME_LIMIT} more)"
return shown
def _warn_column_mismatch(unknown, missing):
"""
Report columns that the schema and the input file disagree about.
TEA's format documents drift from the files they describe, so a mismatch is
reported rather than treated as fatal.
Parameters
----------
unknown : list of str
Columns present in the file but absent from the schema.
missing : list of str
Columns present in the schema but absent from the file.
"""
if unknown:
print(
f"WARNING: {len(unknown)} column(s) in the file are not in the schema; "
f"emitted with their original names: {_format_column_list(unknown)}",
file=sys.stderr,
)
if missing:
print(
f"WARNING: {len(missing)} schema column(s) are not in the file; skipped: {_format_column_list(missing)}",
file=sys.stderr,
)
[docs]
def process_delimited_file(input_file, schema_config, filter_columns=False):
"""
Process a delimited (CSV) file using the provided YAML schema configuration.
Each schema field maps a source column in the file to an output column name.
Columns keep the order they have in the input file, which is also the order
the TEA format document lists them in.
Parameters
----------
input_file : str
The path to the delimited input file.
schema_config : dict
Schema configuration dictionary whose fields use 'source_column'.
filter_columns : bool, optional
If True, return only columns that are marked with "keep": true.
Returns
-------
pd.DataFrame
DataFrame with the processed data.
"""
rename_map = {} # Source column name -> output column name.
keep_columns = [] # Output names of columns flagged to be retained.
for field in schema_config["fields"]:
# Use 'mapped_field_name' when filtering columns if available.
if filter_columns:
mapped_name = field.get("mapped_field_name")
output_name = field["output_field"] if pd.isna(mapped_name) else mapped_name
else:
output_name = field["output_field"]
rename_map[field[DELIMITED_FIELD_KEY]] = output_name
if field.get("keep", False):
keep_columns.append(output_name)
# dtype=str matches the fixed-width path. keep_default_na=False stops pandas
# from converting blanks to NaN and coercing literal TEA codes such as "NA".
df = pd.read_csv(input_file, dtype=str, keep_default_na=False)
file_columns = list(df.columns)
unknown = [column for column in file_columns if column not in rename_map]
missing = [column for column in rename_map if column not in file_columns]
_warn_column_mismatch(unknown, missing)
df = df.rename(columns=rename_map)
if filter_columns:
keep = set(keep_columns)
df = df[[column for column in df.columns if column in keep]]
return df
def _test_name_from_filename(input_file):
"""
Determine the test type from the input filename.
TELPAS shares STAAR's spring month range and the consolidated accountability
file uses a year where a month is expected, so neither can be distinguished
by header content alone.
Parameters
----------
input_file : str
Path to the input file.
Returns
-------
str or None
The test name, or None if the filename implies no particular type.
"""
basename_upper = os.path.basename(input_file).upper()
if "TELPAS" in basename_upper:
return "telpas"
if "ACCOUNTABILITY" in basename_upper:
return "consolidated_accountability"
return None
def _identify_fixed_width_file(input_file):
"""
Determine the test type and school year for a fixed-width input file.
Parameters
----------
input_file : str
Path to the fixed-width input file.
Returns
-------
tuple of (str, int)
The test name and the full school year.
Raises
------
ValueError
If the header line is too short to identify.
"""
with open(input_file) as f:
header_line = f.readline().strip()
if len(header_line) < 4:
raise ValueError("The header line must contain at least 4 characters.")
# Extract test month and abbreviated school year from header.
header = header_line[:4]
try:
test_month = int(header[:2])
full_school_year = 2000 + int(header[2:4])
except ValueError as ve:
raise ValueError(
f"The first 4 characters of {input_file} ({header!r}) are not a month/year header. "
f"If this is a delimited file, it must have a .csv extension to be read as one."
) from ve
# Filename-based detection runs first so files whose headers would otherwise
# collide still route correctly.
test_name = _test_name_from_filename(input_file)
if test_name is not None:
return test_name, full_school_year
if test_month < 10:
return "staar", full_school_year
if test_month < 15:
full_school_year += 1
return "staar_eoc", full_school_year
def _identify_delimited_file(input_file):
"""
Determine the test type and school year for a delimited input file.
The year comes from the file's own YEAR column rather than from the header
or the filename.
Parameters
----------
input_file : str
Path to the delimited input file.
Returns
-------
tuple of (str, int)
The test name and the full school year.
Raises
------
ValueError
If the test type cannot be determined, or the YEAR column is absent,
empty, or not a year.
"""
test_name = _test_name_from_filename(input_file)
if test_name is None:
raise ValueError(
f"Cannot determine the test type for {input_file}; the filename must contain 'TELPAS' or 'ACCOUNTABILITY'."
)
# Only the first data row is needed, so the rest of the file is not read.
first_row = pd.read_csv(input_file, nrows=1, dtype=str, keep_default_na=False)
if "YEAR" not in first_row.columns:
raise ValueError(
f"Delimited file {input_file} has no 'YEAR' column, so its accountability year cannot be determined."
)
if first_row.empty:
raise ValueError(f"Delimited file {input_file} has a header row but no data rows.")
year_value = first_row.loc[0, "YEAR"]
try:
full_school_year = int(year_value)
except (TypeError, ValueError) as ve:
raise ValueError(
f"Delimited file {input_file} has a 'YEAR' value of {year_value!r}, which is not a year."
) from ve
return test_name, full_school_year
[docs]
def process_file(input_file, output_file=None, schema_folder=None, filter_columns=False):
r"""
Process an input fixed\-width or delimited file and output a CSV file.
The function:
\- Infers the input format (fixed-width or delimited) from the input file extension.
\- Determines the appropriate YAML schema based on the file's format-specific identification.
\- Loads and validates the schema, and confirms its shape matches the input format.
\- Processes the input file and writes the output DataFrame to CSV.
Parameters
----------
input_file : str
The path to the fixed\-width or delimited input file.
output_file : str, optional
File path for the output CSV. Defaults to input file name with '_output.csv' appended.
schema_folder : str, optional
Folder where the YAML schema files are located; defaults to the current folder.
filter_columns : bool, optional
If True, only load columns flagged with "keep": true (default is False).
Returns
-------
pd.DataFrame
The processed DataFrame.
"""
# Define the output CSV file name if not explicitly provided.
if output_file is None:
base, _ = os.path.splitext(input_file)
output_file = f"{base}_output.csv"
# Format is inferred from the extension; identification must not assume
# fixed-width, because a CSV header line has no month/year to parse.
is_delimited = input_file.lower().endswith(".csv")
if is_delimited:
test_name, full_school_year = _identify_delimited_file(input_file)
else:
test_name, full_school_year = _identify_fixed_width_file(input_file)
# Compose the path to the expected YAML schema file.
base_folder = schema_folder if schema_folder is not None else "default_schema"
schema_config_file = os.path.join(base_folder, test_name, f"{test_name}_{full_school_year}.yaml")
if not os.path.isfile(schema_config_file):
raise FileNotFoundError(
f"No schema found for {test_name} {full_school_year}; expected it at {schema_config_file}."
)
print(f"Loading schema config: {schema_config_file}")
# Load and validate the YAML configuration.
schema_config = load_yaml_config(schema_config_file)
try:
validate_yaml_config(schema_config, schema_config_file)
except ValueError as ve:
print(f"YAML validation error: {ve}")
sys.exit(1)
# The schema's shape must match the input's format, or parsing fails obscurely.
expected_shape = "delimited" if is_delimited else "fixed_width"
if schema_shape(schema_config) != expected_shape:
raise ValueError(
f"Input file {input_file} is {expected_shape}, but schema {schema_config_file} is not "
f"{expected_shape}. A .csv input needs a schema whose fields use 'source_column'; any "
f"other input needs a schema whose fields use 'start' and 'end'."
)
# Process the file using the loaded schema.
if is_delimited:
df = process_delimited_file(input_file, schema_config, filter_columns=filter_columns)
else:
df = process_fixed_width_file(input_file, schema_config, skip_header=True, filter_columns=filter_columns)
# Write the processed data to a CSV file.
df.to_csv(output_file, index=False)
print(f"Data has been written to {output_file}")
return df
[docs]
def export_templates(schema_folder):
r"""
Export sample YAML template files to a specified folder.
The function copies files from the built\-in default_schema directory
(packaged with this module) into the target folder while preserving the
original directory structure.
Parameters
----------
schema_folder : str
The destination folder for exporting the template YAML files.
Notes
-----
The function exits after exporting the template files.
"""
# Locate the default_schema folder within the package.
with importlib_resources.path("fixedwidth_processor", "default_schema") as default_schema_path:
# Check if the default_schema_path is a valid directory.
if not os.path.isdir(str(default_schema_path)):
print("Default schema folder not found in package.")
sys.exit(1)
# Walk the directory using the string version of the path.
for root, _dirs, files in os.walk(str(default_schema_path)):
for file in files:
rel_path = os.path.relpath(os.path.join(root, file), str(default_schema_path))
target_file = os.path.join(schema_folder, rel_path)
os.makedirs(os.path.dirname(target_file), exist_ok=True)
shutil.copy(os.path.join(root, file), target_file)
print(f"Template YAML files exported to {schema_folder}.")
print(
"Please review and update the templates as needed, then run the script again using the --schema_folder option."
)
sys.exit(0)
[docs]
def csv_to_schema_yaml(csv_file, yaml_output_file=None):
r"""
Convert a CSV file into a YAML schema file for fixed\-width processing.
This function loads a CSV file, lists available columns, and interactively
prompts the user to select fields corresponding to start, end, and output
values, then writes out a YAML file with the chosen configuration.
Parameters
----------
csv_file : str
Path to the input CSV file.
yaml_output_file : str, optional
Output file path for the YAML schema. If omitted, a default name is generated.
"""
try:
df = pd.read_csv(csv_file)
except Exception as e:
print(f"Error loading CSV file: {e}")
return
# Display available CSV columns for user selection.
print("Available columns in the CSV:")
for col in df.columns:
print(f" - {col}")
# Request the user to enter the necessary columns.
start_col = input("Enter the name of the column representing the start value: ").strip()
end_col = input("Enter the name of the column representing the end value: ").strip()
output_field_col = input(
"Enter the name of the column representing the output field (e.g., 'Field Category - Field Title'): "
).strip()
fields = [] # Prepare a list for schema field definitions.
for index, row in df.iterrows():
try:
start_value = int(row[start_col])
except (ValueError, TypeError):
print(f"Row {index}: Could not convert start value '{row[start_col]}' to int. Skipping this row.")
continue
try:
end_value = int(row[end_col])
except (ValueError, TypeError):
print(f"Row {index}: Could not convert end value '{row[end_col]}' to int. Skipping this row.")
continue
# Clean the output field by replacing special dash characters.
output_field_value = (
str(row[output_field_col]).replace("\u2010", "-").replace("\u2013", "-").replace("\n", "").replace("\r", "")
)
field_entry = {
"start": start_value,
"end": end_value,
"output_field": output_field_value,
"keep": row.get("keep", False),
"mapped_field_name": row.get("Mapped Field Title", output_field_value),
}
fields.append(field_entry)
data = {"fields": fields}
# Set default output YAML file name if none provided.
if yaml_output_file is None:
base, _ = os.path.splitext(csv_file)
yaml_output_file = f"{base}_schema.yaml"
try:
with open(yaml_output_file, "w") as f:
yaml.dump(data, f, sort_keys=False)
print(f"Schema YAML file successfully created: {yaml_output_file}")
except Exception as e:
print(f"Error writing YAML file: {e}")