Analysis of Financial Restatements and Non-Reliance Disclosures

from sec_api import Form_8K_Item_X_Api

item_X_api = Form_8K_Item_X_Api("YOUR_API_KEY")

YEARS = range(2024, 2003, -1)  # from 2024 to 2004
TARGET_FILE = "./form-8k-item-4-02-structured-data.jsonl"

if not os.path.exists(TARGET_FILE):
    for year in YEARS:
        done = False
        search_from = 0
        year_counter = 0

        while not done:
            searchRequest = {
                "query": f"item4_02:* AND filedAt:[{year}-01-01 TO {year}-12-31]",
                "from": search_from,
                "size": "50",
                "sort": [{"filedAt": {"order": "desc"}}],
            }

            response = item_X_api.get_data(searchRequest)

            if len(response["data"]) == 0:
                break

            search_from += 50
            year_counter += len(response["data"])

            with open(TARGET_FILE, "a") as f:
                for entry in response["data"]:
                    f.write(json.dumps(entry) + "\n")

        print(f"Finished loading {year_counter} Item 4.02 for year {year}")
else:
    print("File already exists, skipping download")

print(
    f"Loaded {len(structured_data):,} records from Item 4.02 disclosures between 2004 to 2024."
)
structured_data.head()

item_4_02_counts = (
    structured_data.drop_duplicates(subset=["accessionNo"])
    .groupby(["year"])
    .size()
    .to_frame(name="count")
)

print(f"Item 4.02 counts from 2004 to 2024.")
item_4_02_counts.T

def plot_timeseries(ts, title):
    fig, ax = plt.subplots(figsize=(4, 2.5))
    ts["count"].plot(ax=ax, legend=False)
    ax.set_title(title)
    ax.set_xlabel("Year")
    ax.set_ylabel("Number of\nItem 4.02 Filings")
    ax.set_xticks(np.arange(2004, 2025, 2))
    ax.yaxis.set_major_formatter(mtick.StrMethodFormatter("{x:,.0f}"))
    ax.set_xlim(2003, 2025)
    ax.grid(axis="x")
    ax.set_axisbelow(True)
    plt.xticks(rotation=45, ha="right")

    for year in YEARS:
        year_y_max = ts.loc[year, "count"]
        ax.vlines(year, 0, year_y_max, linestyles=":", colors="grey", alpha=0.5, lw=1)

    plt.tight_layout()
    plt.show()


plot_timeseries(
    item_4_02_counts,
    title="Disclosures of Financial Statement Non-Reliance\nForm 8-K with Item 4.02 per Year (2004  - 2024)",
)

structured_data["qtr"] = structured_data["month"].apply(lambda x: (x - 1) // 3 + 1)

counts_qtr_yr_piv = (
    structured_data.drop_duplicates(subset=["accessionNo"])
    .groupby(["year", "qtr"])
    .size()
    .unstack()
    .fillna(0)
).astype(int)

print(f"Item 4.02 counts by quarter from 2004 to 2024.")
counts_qtr_yr_piv.T

counts_qtr_yr = counts_qtr_yr_piv.stack().reset_index(name="count")

fig, ax = plt.subplots(figsize=(6, 2.5))
counts_qtr_yr_piv.plot(kind="bar", ax=ax, legend=True)
ax.legend(title="Quarter", loc="upper right", bbox_to_anchor=(1.02, 1))
ax.set_title("Number of Non-Reliance Disclosures per Quarter\n(2004 - 2024)")
ax.set_xlabel("Year")
ax.set_ylabel("Number of\nItem 4.02 Filings")
ax.yaxis.set_major_formatter(mtick.StrMethodFormatter("{x:,.0f}"))
ax.grid(axis="x")
ax.set_axisbelow(True)
plt.tight_layout()
plt.show()

counts_month_yr_piv = (
    structured_data.drop_duplicates(subset=["accessionNo"])
    .groupby(["year", "month"])
    .size()
    .unstack()
    .fillna(0)
).astype(int)

print(f"Item 4.02 counts by month from 2004 to 2024.")
counts_month_yr_piv

print(f"Descriptive statistics for Item 4.02 counts by month from 2005 to 2024.")
month_stats = (
    counts_month_yr_piv.loc[2005:]
    .describe(percentiles=[0.025, 0.975])
    .round(0)
    .astype(int)
)
month_stats

def plot_box_plot_as_line(
    data: pd.DataFrame,
    x_months=True,
    title="",
    x_label="",
    x_pos_mean_label=2,
    pos_labels=None,  # {"mean": {"x": 2, "y": 150}, "upper": {"x": 2, "y": 150}, "lower": {"x": 2, "y": 150}},
    pos_high_low=None,  # {"high": {"x": 2, "y": 150}, "low": {"x": 2, "y": 150}},
    y_label="",
    y_formatter=lambda x, p: "{:.0f}".format(int(x) / 1000),
    show_high_low_labels=True,
    show_inline_labels=True,
    show_bands=True,
    figsize=(4, 2.5),
    line_source="mean",
):
    fig, ax = plt.subplots(figsize=figsize)

    line_to_plot = data[line_source]
    lower_label = "2.5%"
    upper_label = "97.5%"
    lower = data[lower_label]
    upper = data[upper_label]

    line_to_plot.plot(ax=ax)

    if show_bands:
        ax.fill_between(line_to_plot.index, lower, upper, alpha=0.2)

    if x_months:
        ax.set_xlim(0.5, 12.5)
        ax.set_xticks(range(1, 13))
        ax.set_xticklabels(["J", "F", "M", "A", "M", "J", "J", "A", "S", "O", "N", "D"])

    ax.yaxis.set_major_formatter(mtick.FuncFormatter(y_formatter))
    ax.set_ylabel(y_label)
    ax.set_xlabel(x_label)
    ax.set_title(title)

    ymin, ymax = ax.get_ylim()
    y_scale = ymax - ymin

    max_x = int(line_to_plot.idxmax())
    max_y = line_to_plot.max()
    min_x = int(line_to_plot.idxmin())
    min_y = line_to_plot.min()

    ax.axvline(
        max_x,
        ymin=0,
        ymax=((max_y - ymin) / (ymax - ymin)),
        linestyle="dashed",
        color="tab:blue",
        alpha=0.5,
    )
    ax.scatter(max_x, max_y, color="tab:blue", s=10)
    ax.axvline(
        min_x,
        ymin=0,
        ymax=((min_y - ymin) / (ymax - ymin)),
        linestyle="dashed",
        color="tab:blue",
        alpha=0.5,
    )
    ax.scatter(min_x, min_y, color="tab:blue", s=10)

    x_pos_mean_label_int = int(x_pos_mean_label)
    if show_inline_labels:
        mean_x = x_pos_mean_label
        mean_y = line_to_plot.iloc[x_pos_mean_label_int] * 1.02
        upper_x = x_pos_mean_label
        upper_y = upper.iloc[x_pos_mean_label_int]
        lower_x = x_pos_mean_label
        lower_y = lower.iloc[x_pos_mean_label_int] * 0.95

        if pos_labels:
            mean_x = pos_labels["mean"]["x"]
            mean_y = pos_labels["mean"]["y"]
            upper_x = pos_labels["upper"]["x"]
            upper_y = pos_labels["upper"]["y"]
            lower_x = pos_labels["lower"]["x"]
            lower_y = pos_labels["lower"]["y"]

        ax.text(mean_x, mean_y, "Mean", color="tab:blue", fontsize=8)
        ax.text(upper_x, upper_y, upper_label, color="tab:blue", fontsize=8)
        ax.text(lower_x, lower_y, lower_label, color="tab:blue", fontsize=8)

    if show_high_low_labels:
        high_x_origin = max_x
        high_y_origin = max_y
        high_x_label = high_x_origin + 0.5
        high_y_label = high_y_origin + 0.1 * y_scale
        if pos_high_low:
            high_x_label = pos_high_low["high"]["x"]
            high_y_label = pos_high_low["high"]["y"]
        ax.annotate(
            "High",
            (high_x_origin, high_y_origin),
            xytext=(high_x_label, high_y_label),
            arrowprops=dict(facecolor="black", arrowstyle="->"),
        )

        low_x_origin = min_x * 1.01
        low_y_origin = min_y
        low_x_label = low_x_origin + 1.5
        low_y_label = low_y_origin - 0.1 * y_scale
        if pos_high_low:
            low_x_label = pos_high_low["low"]["x"]
            low_y_label = pos_high_low["low"]["y"]
        ax.annotate(
            "Low",
            (low_x_origin, low_y_origin),
            xytext=(low_x_label, low_y_label),
            arrowprops=dict(facecolor="black", arrowstyle="->"),
        )

    ax.grid(axis="x")
    ax.set_axisbelow(True)

    plt.tight_layout()
    plt.show()


plot_box_plot_as_line(
    data=month_stats.T,
    title="Descriptive Statistics for Item 4.02 Filings by Month\n(2005 - 2024)",
    x_label="Month",
    y_label="Number of\nItem 4.02 Filings",
    y_formatter=lambda x, p: "{:.0f}".format(int(x)),
)

fig, ax = plt.subplots(figsize=(3.5, 3))

counts_month_yr_piv.loc[2005:].boxplot(
    ax=ax,
    grid=False,
    showfliers=False,
    flierprops=dict(marker="o", markersize=3),
    patch_artist=True,
    boxprops=dict(facecolor="white", color="tab:blue"),
    showmeans=True,
    meanline=True,
    meanprops={"color": "tab:blue", "linestyle": ":"},
    medianprops={"color": "black"},
    capprops={"color": "none"},
)

ax.set_title("Item 4.02 Disclosures by Month\n(2005 - 2024)")
ax.set_xlabel("Month")
ax.set_ylabel("Item 4.02 Count")
xticklabels = [pd.to_datetime(str(x), format="%m").strftime("%b") for x in range(1, 13)]
ax.set_xticklabels(xticklabels)
plt.xticks(rotation=45)
plt.tight_layout()
plt.show()

counts_filedAtClass = (
    structured_data.drop_duplicates(subset=["accessionNo"])
    .groupby(["filedAtClass"])
    .size()
    .sort_values(ascending=False)
    .to_frame(name="Count")
).rename_axis("Publication Time")
counts_filedAtClass["Pct"] = (
    counts_filedAtClass["Count"].astype(int)
    / counts_filedAtClass["Count"].astype(int).sum()
).map("{:.0%}".format)
counts_filedAtClass["Count"] = counts_filedAtClass["Count"].map(lambda x: f"{x:,}")
counts_filedAtClass.index = (
    counts_filedAtClass.index.str.replace("preMarket", "Pre-Market (4:00 - 9:30 AM)")
    .str.replace("regularMarket", "Market Hours (9:30 AM - 4:00 PM)")
    .str.replace("afterMarket", "After Market (4:00 - 8:00 PM)")
)
counts_filedAtClass = counts_filedAtClass.reindex(counts_filedAtClass.index[::-1])

print(
    f"Item 4.02 counts by pre-market, regular market hours,\nand after-market publication time (2004 - 2025)."
)
counts_filedAtClass

counts_dayOfWeek = (
    structured_data.drop_duplicates(subset=["accessionNo"])
    .groupby(["dayOfWeek"])
    .size()
    .to_frame(name="Count")
).rename_axis("Day of the Week")
counts_dayOfWeek["Pct"] = (
    counts_dayOfWeek["Count"].astype(int) / counts_dayOfWeek["Count"].astype(int).sum()
).map("{:.0%}".format)
counts_dayOfWeek["Count"] = counts_dayOfWeek["Count"].map(lambda x: f"{x:,}")

print(f"Item 4.02 disclosures by day of the week (2004 - 2024).")
counts_dayOfWeek.loc[["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"]]

bool_variables_to_analyze = [
    "impactIsMaterial",
    "restatementIsNecessary",
    "impactYetToBeDetermined",
    "materialWeaknessIdentified",
    "reportedWithOtherItems",
    "reportedWithEarnings",
    "netIncomeDecreased",
    "netIncomeIncreased",
    "netIncomeAdjustmentContainsWordMillion",
    "revenueDecreased",
    "revenueIncreased",
    "revenueAdjustmentContainsWordMillion",
    "identifiedByAuditor",
    "identifiedByCompany",
    "identifiedBySec",
]

var_to_label = {
    "impactIsMaterial": "Impact is Material",
    "restatementIsNecessary": "Restatement is Necessary",
    "impactYetToBeDetermined": "Impact Yet to be Determined",
    "materialWeaknessIdentified": "Material Weakness Identified",
    "reportedWithOtherItems": "Reported with Other Items",
    "reportedWithEarnings": "Reported with Earnings Announcement",
    "netIncomeDecreased": "Net Income Decreased",
    "netIncomeIncreased": "Net Income Increased",
    "netIncomeAdjustmentContainsWordMillion": "Net Inc. Adj. Contains 'Million'",
    "revenueDecreased": "Revenue Decreased",
    "revenueIncreased": "Revenue Increased",
    "revenueAdjustmentContainsWordMillion": "Revenue Adj. Contains 'Million'",
    "identifiedByAuditor": "Identified by Auditor",
    "identifiedByCompany": "Identified by Company",
    "identifiedBySec": "Identified by SEC",
}

bool_variables_stats = []

for variable in bool_variables_to_analyze:
    variable_stats = (
        structured_data[variable]
        .value_counts()
        .to_frame()
        .reset_index()
        .rename(columns={variable: "value"})
    )
    variable_stats = variable_stats.sort_values(by="value", ascending=False)
    variable_stats["pct"] = (
        variable_stats["count"] / variable_stats["count"].sum() * 100
    ).round(1)
    variable_stats.index = pd.MultiIndex.from_tuples(
        [(variable, row["value"]) for _, row in variable_stats.iterrows()],
    )
    variable_stats.drop(columns="value", inplace=True)

    bool_variables_stats.append(variable_stats)


bool_variables_stats = pd.concat(bool_variables_stats, axis=0)
bool_variables_stats.index.set_names(["Variable", "Value"], inplace=True)
bool_variables_stats.rename(index=var_to_label, columns={"count": "Samples", "pct": "Pct."}, inplace=True)
bool_variables_stats["Samples"] = bool_variables_stats["Samples"].apply(lambda x: f"{x:,.0f}")

print(f"Number of non-reliance filings by \ntheir disclosed characteristics (2004 - 2024):")
bool_variables_stats

identifiedBy = (
    structured_data["identifiedBy"].explode().value_counts().to_frame().head(3)
)
identifiedBy.index.name = "Identified By"
identifiedBy.columns = ["Count"]
identifiedBy["Pct."] = identifiedBy["Count"] / identifiedBy["Count"].sum() * 100
identifiedBy["Pct."] = identifiedBy["Pct."].round(1)
identifiedBy["Count"] = identifiedBy["Count"].apply(lambda x: f"{x:,.0f}")

print(
    f"Top 3 entities identifying issues in\npreviously reported financial statements (2004 - 2024):"
)
identifiedBy

all_auditors = structured_data["auditors"].explode()
all_auditors = all_auditors[all_auditors.str.len() > 0].reset_index(drop=True)
auditors = all_auditors.value_counts().to_frame().reset_index()
auditors["pct"] = auditors["count"] / auditors["count"].sum() * 100
auditors["pct"] = auditors["pct"].round(2)

print("Top 10 auditors involved in \nnon-reliance disclosures from 2004 to 2024:")
auditors.head(10)

auditors_year = structured_data[["auditors", "year", "accessionNo"]].explode("auditors")

auditors_year_pivot = pd.pivot_table(
    auditors_year,
    index="auditors",
    columns="year",
    values="accessionNo",
    aggfunc="count",
    fill_value=0,
)

auditors_year_pivot["total"] = auditors_year_pivot.sum(axis=1)
auditors_year_pivot = auditors_year_pivot.sort_values(by="total", ascending=False)

top_10_auditors = auditors_year_pivot.head(10)

others = auditors_year_pivot[~auditors_year_pivot.index.isin(top_10_auditors.index)]
others = others.sum().to_frame().T
others.index = ["Others"]

top_10_auditors = pd.concat([top_10_auditors, others], axis=0)

top_10_auditors

fig, ax = plt.subplots(figsize=(5, 3))

top_10_auditors.drop(columns="total").T.plot(
    kind="bar", stacked=True, ax=ax, cmap="tab20"
)

ax.set_title("Number of Item 4.02 Filings\nby Auditor and Year")
ax.set_xlabel("Year")
ax.set_ylabel("Number of Filings")
ax.xaxis.grid(False)
ax.set_axisbelow(True)
handles, labels = ax.get_legend_handles_labels()  # reverse order of legend items
ax.legend(
    reversed(handles),
    reversed(labels),
    title="Auditor",
    bbox_to_anchor=(1.05, 1),
    labelspacing=0.3,
    fontsize=8,
)

plt.tight_layout()
plt.show()

print(
    f"Descriptive statistics for number of years and quarters \naffected by Item 4.02 filings (2004 - 2024):"
)
quarters_stats = (
    structured_data[["numberQuartersAffected", "numberYearsAffected"]]
    .describe()
    .round(0)
    .astype(int)
)
quarters_stats.T

affectedLineItems_stats = (
    structured_data["affectedLineItems"]
    .explode()
    .value_counts()
    .to_frame()
    .reset_index()
    .head(10)
)
print(
    f"Top 10 line items affected by non-reliance disclosures\nacross all years (2004 - 2024):"
)
affectedLineItems_stats

line_items_year_pivot = pd.pivot_table(
    structured_data.explode("affectedLineItems"),
    index="affectedLineItems",
    columns="year",
    values="accessionNo",
    aggfunc="count",
    fill_value=0,
    margins=True,
)
line_items_year_pivot = line_items_year_pivot.sort_values(by="All", ascending=False)
print(f"Top 20 line items affected by Item 4.02 filings per year (2004 - 2023):")
line_items_year_pivot.head(20)