"""Reproduce the State of the F.A.M.I.L.I.E.S. 2026 indicator and pillar ranks.

Usage:  python3 state-of-the-families-2026-reproduce.py
Needs:  state-of-the-families-2026-data.csv and state-of-the-families-2026-pillar-ranks.csv
        in the same folder (free at rushing4success.com/research). Python 3.8+, no extra packages.

Method (also described in the State Briefs):
  1. Indicator rank: states with a published value, sorted best to worst; ties share the
     better (lower-numbered) rank. States without a reliable published value are not ranked.
  2. Standard score: z = (value - mean) / population standard deviation, over states with data;
     sign reversed where lower is better. Yes/no indicators are scored 1/0.
  3. Pillar score: mean of a state's standard scores on the pillar's scored indicators
     (using only indicators with data for that state). Pillar rank 1-50 by unrounded score.
  The KIDS COUNT rank is an external ranking, shown for context and not scored.
"""
import csv, statistics as st
from collections import defaultdict

rows = list(csv.DictReader(open('state-of-the-families-2026-data.csv', encoding='utf-8')))
pub = list(csv.DictReader(open('state-of-the-families-2026-pillar-ranks.csv', encoding='utf-8')))
by_ind = defaultdict(dict); meta = {}
for r in rows:
    meta[r['indicator_key']] = r
    if r['value'] != '':
        by_ind[r['indicator_key']][r['state']] = float(r['value'])
states = sorted({r['state'] for r in rows})

problems = 0; z = defaultdict(dict)
for k, vals in by_ind.items():
    hib = meta[k]['higher_is_better'] == '1'
    if meta[k]['scored_in_pillar'] == '1':
        order = sorted(vals.values(), reverse=hib)
        for s, v in vals.items():
            rank = order.index(v) + 1
            published = next(r['indicator_rank'] for r in rows if r['state'] == s and r['indicator_key'] == k)
            if str(rank) != published:
                problems += 1; print('indicator rank differs:', s, k, rank, published)
        m, sd = st.mean(vals.values()), st.pstdev(vals.values())
        for s, v in vals.items():
            z[s][k] = (v - m) / sd * (1 if hib else -1)

pillars = sorted({r['pillar_code'] for r in pub})
for p in pillars:
    keys = [k for k, r in meta.items() if r['pillar'] == p and r['scored_in_pillar'] == '1']
    score = {s: st.mean([z[s][k] for k in keys if k in z[s]]) for s in states}
    order = sorted(states, key=lambda s: -score[s])
    for r in pub:
        if r['pillar_code'] != p: continue
        rank = order.index(r['state']) + 1
        if str(rank) != r['pillar_rank'] or abs(round(score[r['state']], 3) - float(r['pillar_score'])) > 0.0005:
            problems += 1; print('pillar differs:', r['state'], p, rank, r['pillar_rank'])
print('All ranks reproduced.' if problems == 0 else f'{problems} differences found.')
