xparse

parse raw html or raw json file to structured data with yaml config file

demo

__raw:
site_url: https://xkcd.com/test_keys:
- bottom.comic_links.*
- middle.ctitle
- middle.transcriptbottom:
_locator: div#bottomcomic_links:
_locator: div#comicLinks>a_index: ~text:
href:
_attr: href_attr_refine: enrich_urlmiddle:
_locator: div#middleContainerctitle: div#ctitletranscript: div#transcript

constants

all reserved keys when we used to write yaml config file to map the HTML/JSON

package xparse
// Core extraction configuration keysconst (
// Index specifies which elements to extract from results// Formats: "_index" or "_i"// Values:// - nil/not existed: get all elements// - array: [0,1] gets elements[0] and elements[1]// - single: 0 gets elements[0]// Index types:// 1. without index// 2. index: ~ (index is null)// 3. index: 0// 4. index: [0, 1, ...]// 5. index: 0,4 => 0,1,2,3Index="_index"// Locator specifies the path/selector to find desired elements// Formats: "_locator" or "_l"// Supported types:// > string:// _locator: string//// > list:// _locator:// - div.001// - div.002// - div.003//// > map:// _locator:// key1: div.001// key2: div.002// key3: div.003Locator="_locator"// Element navigation keys// ExtractPrevElem is used when no proper locator exists// in most cases, we can use locator to get the elem we want,// but in some rare cases, there is no proper locator to use, so we have to use this to get prev elemExtractPrevElem="_extract_prev"ExtractParent="_extract_parent"
)
// Attribute related configuration keysconst (
// Attr specifies which attribute to extract// Default is element text// Special value "__html" returns raw HTMLAttr="_attr"// AttrRefine specifies how to refine the extracted attribute// Formats: "_attr_refine" or "_ar"// Values:// - bool(true): auto-generate method name// - string(_name): adds prefix "refine" so "_xxx" becomes "_refine_name"// - string(refine_xxx/_refine_xxx): used as-is// - string(not started with _): used as-isAttrRefine="_attr_refine"// AttrJoiner specifies the joiner for attributesAttrJoiner="_joiner"// AttrIndex configuration:// - _joiner: ","// - _attr_refine: _attr_by_index// - _attr_index: 0AttrIndex="_attr_index"AttrRegex="_attr_regex"// AttrPython runs Python script directly (requires Python environment)// Example:// import sys// raw = sys.argv[1] # raw is globally registered// arr = raw.split("_")// print(arr[1]) # required: output value as refined attr valueAttrPython="_attr_python"// AttrJS runs JavaScript code// Example:// arr = raw.split("_") // raw is registered by default// refined = arr[1] // refined is required value// Note: Underscore.js (https://underscorejs.org/) is supported by defaultAttrJS="_attr_js"
)
// Post-processing configuration keysconst (
// PostJoin joins parsed attributes array into string using joinerPostJoin="_post_join"// Strip controls string trimming// Values:// - if `_strip: true` or not existed: does strings.TrimSpace// - if `_strip: str`: does strings.ReplaceAll(raw, str, "")// - if `_strip: ["(", ")"]`: replaces one by one// Note: Called by default, use `_strip: false` to disableStrip="_strip"// Type converts output to specified type// Without `_type: b/i/f`, returns as string// Values:// - b: bool// - i: int// - f: floatType="_type"
)
// Abbreviated keysconst (
LocatorAbbr="_l"IndexAbbr="_i"AttrRefineAbbr="_ar"TypeAbbr="_t"
)
// Special locators and internal constantsconst (
// JSONArrayRootLocator is used for JSON arrays without root object// Used when JSON file has ordered list of values like: `[{...}, {...}]`JSONArrayRootLocator="*/*"// PrefixLocatorStub for multiple locators not in same stub// Recalculates from base locator (map root)// Example:// jobs:// _locator: jobs// _index:// taxo:// _locator: taxonomyAttributes// _index: 0// attr:// _locator:// - attributes// - ___.salarySnippetPrefixLocatorStub="___"// _prefixRefine defines the word we use as the prefix of method of attr refiner_prefixRefine="_refine"// AttrJoinerSep is a separator used to join an array to stringAttrJoinerSep="|||"
)
// Special attribute valuesconst (
// AttrJoinElemsText joins all elements inner text to string// Used only when parsing HTML// Warning: Rarely used, consider alternativesAttrJoinElemsText="__join_text"// AttrRawHTML returns the raw html of locatorAttrRawHTML="__html"// RefineWithKeyName uses key name as refiner method// Example:// root:// a_changeable_name:// _locator: div.xxx// _attr: title// _attr_refine: __keyRefineWithKeyName="__key"
)
// Type constantsconst (
AttrTypeB="b"// BooleanAttrTypeF="f"// FloatAttrTypeI="i"// Integer// Time typesAttrTypeT="t"// Quick modeAttrTypeT1="t1"// Search mode
)