Module:JSON pipeline
Documentation for this module may be created at Module:JSON pipeline/doc
--[[
Module:JSON pipeline
Loads and reads JSON data pages (the R export pipeline's output) for use in
templates: keyed record lookup, single-page and chunked ("_meta"-routed)
datasets.
Trimmed port of hopperwiki_compiler's Module:JSON pipeline -- only the
generic, domain-independent primitives (load/load_dataset/zip/get/lookup/
normalize_key) are here. hopperwiki's copy also has taxon/OTU-specific
functions (get_otu_from_title, resolve_focal_taxon, find_species_by_geography)
that don't apply to whiskerwiki's geography-by-QID use case -- see
Module:Geography's get_species_list_for_geography() for whiskerwiki's own
domain-specific query built on top of the primitives here.
Written the same extraction-friendly way as this project's other ported
logic: if hopperwiki's copy is ever trimmed to the same generic subset, this
becomes a candidate for shared_wiki_resources/lua_modules (already a proven
shared repo between both wikis for Lua/templates) instead of two copies.
--]]
local p = {}
-- ================================================================
-- Load a wiki page and decode JSON. Returns (data, err) -- err is nil on
-- success, a string reason ("no_content:<title>", "json_decode_failed:...")
-- on failure.
-- ================================================================
local function load_json_page(page_title)
local title_obj = mw.title.new(page_title)
if not title_obj then
return nil, "invalid_title_obj:" .. tostring(page_title)
end
local content = title_obj:getContent()
if not content or content == "" then
return nil, "no_content:" .. tostring(page_title)
end
local ok, data = pcall(mw.text.jsonDecode, content)
if not ok then
return nil, "json_decode_failed:" .. tostring(page_title)
end
return data, nil
end
--[[
Clean up a lookup key for matching: trims whitespace, turns underscores into
spaces, and collapses repeated whitespace. Use this any time a key comes from
a page title or user-supplied argument before comparing it against a
dataset's keys.
--]]
function p.normalize_key(key)
if key == nil then return nil end
key = tostring(key)
key = mw.text.trim(key)
key = key:gsub("_", " ")
key = key:gsub("%s+", " ")
return key
end
--[[
Converts all numeric JSON values/keys to strings recursively. Arrays keep
numeric indices; maps get string keys.
Rationale: QIDs and similar identifiers are identifiers, not numbers.
mw.text.jsonDecode preserves JSON number types, which causes silent lookup
failures ("558389" ~= 558389) if left unconverted.
--]]
local function normalize_numeric_values(x)
if type(x) ~= "table" then
return (type(x) == "number") and tostring(x) or x
end
local is_array = (#x > 0)
if is_array then
local count = 0
for _ in pairs(x) do count = count + 1 end
is_array = (count == #x)
end
local out = {}
for k, v in pairs(x) do
local new_v = (type(v) == "table") and normalize_numeric_values(v)
or (type(v) == "number") and tostring(v)
or v
local new_k = (is_array or type(k) ~= "number") and k or tostring(k)
out[new_k] = new_v
end
return out
end
--[[
Fetch, JSON-decode, and number-normalize a single page. This is the base
loader that load_dataset and load_dataset_multi build on -- prefer calling
one of those over this directly unless you genuinely just need the raw
decoded page.
--]]
function p.load(page_title)
local data, err = load_json_page(page_title)
if not data then return nil, err end
return normalize_numeric_values(data), nil
end
function p.records(data)
if type(data) == "table" and type(data.records) == "table" then
return data.records
end
return data
end
function p.fields(data)
if type(data) == "table" then
return data.fields
end
return nil
end
function p.index(data)
if type(data) == "table" then
return data.index
end
return nil
end
function p.chunk_for(index_table, key)
if type(index_table) ~= "table" or key == nil then return nil end
local norm = p.normalize_key(tostring(key))
return index_table[norm] or index_table[tostring(key)]
end
--[[
Load a page shaped like { records = {...}, fields = {...} } (or a bare
records array with no separate fields).
Returns (records, fields, raw_data, err).
Example:
local records, fields, _, err = p.load_dataset("JSON:Geography_to_species")
local rec = p.zip(fields, p.get(records, qid))
--]]
function p.load_dataset(page_title)
local data, err = p.load(page_title)
if not data then return nil, nil, nil, err end
return p.records(data), p.fields(data), data, nil
end
--[[
Chunk-aware sibling of p.load_dataset(), for datasets built by the R
pipeline's build_json_pages_from_structure()/build_scalar_lookup_pages() --
once a dataset grows past its char_limit, it stops writing a single
"JSON:<page_title>" page and instead writes "JSON:<page_title>_index" (key ->
chunk page) plus numbered chunk pages. Plain p.load_dataset(page_title) would
only ever find chunk 1 (or nothing) in that case.
Falls back to p.load_dataset(page_title) unchanged when no "_index" page
exists, so this is safe to use in place of p.load_dataset() everywhere,
including datasets that never get chunked at all.
--]]
function p.load_dataset_multi(page_title)
local index_page = page_title .. "_index"
local index_title = mw.title.new(index_page)
if not (index_title and index_title.exists) then
return p.load_dataset(page_title)
end
local index_data, err = p.load(index_page)
if not index_data then return nil, nil, nil, err end
local seen_chunks = {}
local chunk_pages = {}
for _, chunk_page in pairs(index_data) do
if type(chunk_page) == "string" and not seen_chunks[chunk_page] then
seen_chunks[chunk_page] = true
table.insert(chunk_pages, chunk_page)
end
end
local combined_records = {}
local combined_fields = nil
for _, chunk_page in ipairs(chunk_pages) do
local chunk_data = p.load(chunk_page)
if chunk_data then
if not combined_fields and p.fields(chunk_data) then
combined_fields = p.fields(chunk_data)
end
local recs = p.records(chunk_data)
if type(recs) == "table" then
for k, v in pairs(recs) do
combined_records[k] = v
end
end
end
end
return combined_records, combined_fields, index_data, nil
end
function p.zip(fields, record_array)
if type(fields) ~= "table" or type(record_array) ~= "table" then
return nil
end
local out = {}
for i, field_name in ipairs(fields) do
local v = record_array[i]
if v ~= nil then
out[field_name] = v
end
end
return out
end
function p.get(records_table, key)
if type(records_table) ~= "table" or key == nil then return nil end
local normalized = p.normalize_key(tostring(key))
return records_table[normalized] or records_table[tostring(key)]
end
--[[
Keyed lookup that transparently handles both single-page and multi-page
("<page_title>_meta") datasets. Returns a zipped named-field record, or nil
if nothing matches.
Example:
local rec = p.lookup("JSON:Geography_to_species", "Q337402")
local rodentia = rec and rec.Rodentia or {}
--]]
function p.lookup(page_title, key)
if not page_title or key == nil then return nil end
local meta_title = mw.title.new(page_title .. "_meta")
if meta_title and meta_title.exists then
local meta = p.load(page_title .. "_meta")
if not meta then return nil end
local fields = p.fields(meta)
local idx = p.index(meta)
if not fields or not idx then return nil end
local chunk_page = p.chunk_for(idx, key)
if not chunk_page then return nil end
local chunk = p.load(chunk_page)
if not chunk then return nil end
local record_array = p.get(chunk, key)
if not record_array then return nil end
return p.zip(fields, record_array)
else
local records, fields = p.load_dataset(page_title)
if not records then return nil end
local record_array = p.get(records, key)
if not record_array then return nil end
if fields then
return p.zip(fields, record_array)
end
return record_array
end
end
return p