Add Artemis stress test for detecting cache aliasing bugs. Include statistical hypothesis evaluations, fix validation data, and run reports for validation across architectures.
Signed-off-by: Robert Allan James <robert.allan.james@gmail.com>
This commit is contained in:
Robert Allan James
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env Rscript
|
||||
# analyse_artemis_stress.R
|
||||
#
|
||||
# Analysis for the Artemis surface stress test bug hunt (2026-08-02):
|
||||
# a stale-pointer aliasing bug between the block-subsystem's devblock
|
||||
# cache and the VM-level window cache in src/word_source/block_words.c,
|
||||
# found by the first completed run of ART-STRESS-TEST, fixed, and
|
||||
# re-verified on all three architectures.
|
||||
#
|
||||
# Reads experiments/artemis_stress/runs/*.csv, writes SVG charts to
|
||||
# analysis/charts/ and PNG copies to analysis/img/.
|
||||
|
||||
suppressMessages({
|
||||
library(ggplot2)
|
||||
library(dplyr)
|
||||
library(tidyr)
|
||||
library(scales)
|
||||
library(svglite)
|
||||
})
|
||||
|
||||
run_dir <- file.path("experiments", "artemis_stress", "runs")
|
||||
chart_dir <- file.path("experiments", "artemis_stress", "analysis", "charts")
|
||||
img_dir <- file.path("experiments", "artemis_stress", "analysis", "img")
|
||||
dir.create(chart_dir, recursive = TRUE, showWarnings = FALSE)
|
||||
dir.create(img_dir, recursive = TRUE, showWarnings = FALSE)
|
||||
|
||||
## ---- palette (matches experiments/bare_metal house style) ----
|
||||
sfblue <- "#1F77B4"
|
||||
sforange <- "#FF7F0E"
|
||||
sfgreen <- "#2CA02C"
|
||||
sfred <- "#D62728"
|
||||
sfgray <- "#505050"
|
||||
|
||||
theme_sf <- function() {
|
||||
theme_minimal(base_size = 12) +
|
||||
theme(
|
||||
panel.grid.minor = element_blank(),
|
||||
plot.title = element_text(face = "bold"),
|
||||
plot.subtitle = element_text(color = sfgray),
|
||||
legend.position = "bottom"
|
||||
)
|
||||
}
|
||||
|
||||
save_chart <- function(plot, name, width = 7, height = 4.2) {
|
||||
svg_path <- file.path(chart_dir, paste0(name, ".svg"))
|
||||
png_path <- file.path(img_dir, paste0(name, ".png"))
|
||||
ggsave(svg_path, plot, width = width, height = height, device = svglite)
|
||||
ggsave(png_path, plot, width = width, height = height, dpi = 200)
|
||||
cat(" wrote", svg_path, "and", png_path, "\n")
|
||||
}
|
||||
|
||||
## ---- load runs ----
|
||||
runs <- list(
|
||||
amd64_buggy = "amd64 (pre-fix)",
|
||||
amd64_fixed = "amd64 (post-fix)",
|
||||
aarch64_fixed = "aarch64 (post-fix)",
|
||||
riscv64_fixed = "riscv64 (post-fix)"
|
||||
)
|
||||
|
||||
load_run <- function(key, label) {
|
||||
df <- read.csv(file.path(run_dir, paste0(key, ".csv")))
|
||||
df$run <- key
|
||||
df$label <- label
|
||||
df$status <- factor(ifelse(df$result == 1, "PASS", "FAIL"), levels = c("PASS", "FAIL"))
|
||||
df
|
||||
}
|
||||
|
||||
all_runs <- bind_rows(Map(load_run, names(runs), runs))
|
||||
|
||||
## ---- Chart 1: pass rate before/after, all runs ----
|
||||
summary_tbl <- all_runs %>%
|
||||
group_by(run, label) %>%
|
||||
summarise(n = n(), passed = sum(result), .groups = "drop") %>%
|
||||
mutate(pass_rate = passed / n,
|
||||
phase = ifelse(grepl("pre-fix", label), "pre-fix", "post-fix"))
|
||||
|
||||
p1 <- ggplot(summary_tbl, aes(x = reorder(label, pass_rate), y = pass_rate, fill = phase)) +
|
||||
geom_col(width = 0.6) +
|
||||
geom_text(aes(label = sprintf("%d/%d", passed, n)), vjust = -0.5, size = 4) +
|
||||
scale_y_continuous(labels = percent_format(), limits = c(0, 1.08)) +
|
||||
scale_fill_manual(values = c("pre-fix" = sfred, "post-fix" = sfgreen)) +
|
||||
coord_flip() +
|
||||
labs(
|
||||
title = "Artemis surface stress test: pass rate before/after fix",
|
||||
subtitle = "50 trials per run, unique random blocks across the 22998-block data pool",
|
||||
x = NULL, y = "Pass rate", fill = NULL
|
||||
) +
|
||||
theme_sf()
|
||||
|
||||
save_chart(p1, "pass_rate_comparison", width = 8)
|
||||
|
||||
## ---- Chart 2: pass/fail sequence in write/verify order, buggy run ----
|
||||
## A visual scan of this sequence first suggested serial clustering; the
|
||||
## Wald-Wolfowitz runs test (H1, below) does not support that reading --
|
||||
## observed runs (26) exactly match the count expected under a random
|
||||
## ordering. The chart is kept because it is still the raw sequence the
|
||||
## tests are run against, not because it shows a confirmed pattern.
|
||||
buggy <- all_runs %>% filter(run == "amd64_buggy") %>% arrange(trial)
|
||||
|
||||
p2 <- ggplot(buggy, aes(x = trial, y = 1, fill = status)) +
|
||||
geom_tile(color = "white", linewidth = 0.6, height = 1) +
|
||||
scale_fill_manual(values = c(PASS = sfgreen, FAIL = sfred)) +
|
||||
scale_x_continuous(breaks = seq(0, 49, 5)) +
|
||||
labs(
|
||||
title = "Pass/fail sequence in write order, amd64 pre-fix run",
|
||||
subtitle = "25 of 50 trials failed; a formal runs test (H1) finds no\nsignificant serial clustering in this sequence -- see Section 4",
|
||||
x = "Trial index (= write/verify order)", y = NULL, fill = NULL
|
||||
) +
|
||||
theme_sf() +
|
||||
theme(axis.text.y = element_blank(), panel.grid = element_blank())
|
||||
|
||||
save_chart(p2, "failure_clustering", height = 2.6)
|
||||
|
||||
## ---- Chart 3: surface coverage map, buggy vs one fixed run ----
|
||||
cov <- all_runs %>% filter(run %in% c("amd64_buggy", "amd64_fixed"))
|
||||
|
||||
p3 <- ggplot(cov, aes(x = lbn, y = trial, color = status)) +
|
||||
geom_point(size = 2.2, alpha = 0.85) +
|
||||
scale_color_manual(values = c(PASS = sfgreen, FAIL = sfred)) +
|
||||
facet_wrap(~label, ncol = 1) +
|
||||
labs(
|
||||
title = "Surface coverage: scattered across the full data pool",
|
||||
subtitle = "Each point is one trial's block. A Wilcoxon test (H2) finds LBN and\noutcome are not independent, but no sub-region is visually implicated",
|
||||
x = "Logical block number (data pool: 3076-26073)", y = "Trial index", color = NULL
|
||||
) +
|
||||
theme_sf()
|
||||
|
||||
save_chart(p3, "surface_coverage_map", height = 5.5)
|
||||
|
||||
## ---- summary table for the report ----
|
||||
write.csv(summary_tbl, file.path("experiments", "artemis_stress", "analysis", "summary.csv"),
|
||||
row.names = FALSE)
|
||||
|
||||
cat("\nDone. Summary:\n")
|
||||
print(as.data.frame(summary_tbl))
|
||||
|
||||
## ═══════════════════════════════════════════════════════════════════════
|
||||
## Hypothesis tests
|
||||
##
|
||||
## Three questions, each with an explicit null hypothesis, tested against
|
||||
## the actual recorded trial data rather than asserted from eyeballing
|
||||
## charts. Results are written to tables/ for the LaTeX report to quote
|
||||
## verbatim -- no numbers in the report are hand-typed.
|
||||
## ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
table_dir <- file.path("experiments", "artemis_stress", "analysis", "tables")
|
||||
dir.create(table_dir, recursive = TRUE, showWarnings = FALSE)
|
||||
|
||||
sink_lines <- c()
|
||||
say <- function(...) {
|
||||
line <- paste0(...)
|
||||
sink_lines <<- c(sink_lines, line)
|
||||
cat(line, "\n")
|
||||
}
|
||||
|
||||
say("Artemis surface stress test -- hypothesis tests")
|
||||
say("=================================================")
|
||||
say("")
|
||||
|
||||
## ---- H1: are pre-fix failures independently distributed across trials, ----
|
||||
## ---- or do they show serial clustering (a runs test)? ----
|
||||
## H0: the pass/fail sequence is a random ordering (no serial correlation).
|
||||
## Manual Wald-Wolfowitz runs test (base R only, no extra package needed).
|
||||
runs_test <- function(seq01) {
|
||||
n1 <- sum(seq01 == 1); n2 <- sum(seq01 == 0); n <- n1 + n2
|
||||
r <- 1 + sum(diff(seq01) != 0)
|
||||
mu <- 1 + 2 * n1 * n2 / n
|
||||
var_r <- (2 * n1 * n2 * (2 * n1 * n2 - n)) / (n^2 * (n - 1))
|
||||
z <- (r - mu) / sqrt(var_r)
|
||||
p <- 2 * pnorm(-abs(z))
|
||||
list(n1 = n1, n2 = n2, runs = r, expected = mu, z = z, p = p)
|
||||
}
|
||||
|
||||
buggy_seq <- buggy$result[order(buggy$trial)]
|
||||
rt <- runs_test(buggy_seq)
|
||||
|
||||
say("H1 -- pre-fix failure clustering (Wald-Wolfowitz runs test)")
|
||||
say(" H0: pass/fail outcomes are independently ordered (no clustering).")
|
||||
say(sprintf(" Observed runs = %d (expected under H0 = %.2f, sd = %.2f)",
|
||||
rt$runs, rt$expected, sqrt((2*rt$n1*rt$n2*(2*rt$n1*rt$n2-(rt$n1+rt$n2))) /
|
||||
((rt$n1+rt$n2)^2*(rt$n1+rt$n2-1)))))
|
||||
say(sprintf(" z = %.3f, p = %.2e", rt$z, rt$p))
|
||||
say(sprintf(" -> %s H0 at alpha=0.05: failures %s serially clustered.",
|
||||
ifelse(rt$p < 0.05, "REJECT", "fail to reject"),
|
||||
ifelse(rt$p < 0.05, "ARE", "are not")))
|
||||
say("")
|
||||
|
||||
## ---- H1b: is outcome associated with trial/write order directly ----
|
||||
## ---- (not via the runs test, which only detects consecutive-value ----
|
||||
## ---- clustering and can miss a more general order-dependent trend)? ----
|
||||
## H0: trial index is drawn from the same distribution for PASS and FAIL.
|
||||
wt_order <- wilcox.test(trial ~ status, data = buggy)
|
||||
|
||||
say("H1b -- pre-fix outcome vs. trial/write order directly (Wilcoxon)")
|
||||
say(" H0: trial index is drawn from the same distribution for PASS/FAIL")
|
||||
say(" (complements H1: the runs test only catches consecutive-value")
|
||||
say(" clustering, not a general rank-order trend).")
|
||||
say(sprintf(" W = %.1f, p = %.3f", wt_order$statistic, wt_order$p.value))
|
||||
say(sprintf(" -> %s H0 at alpha=0.05: outcome %s associated with write order.",
|
||||
ifelse(wt_order$p.value < 0.05, "REJECT", "fail to reject"),
|
||||
ifelse(wt_order$p.value < 0.05, "IS", "is not")))
|
||||
say("")
|
||||
|
||||
## ---- H2: is failure status independent of disk location (logical ----
|
||||
## ---- block number)? ----
|
||||
## H0: LBN is drawn from the same distribution for PASS and FAIL trials.
|
||||
## Wilcoxon rank-sum test (non-parametric, no normality assumption on LBN).
|
||||
wt <- wilcox.test(lbn ~ status, data = buggy)
|
||||
|
||||
say("H2 -- pre-fix failure vs. disk location (Wilcoxon rank-sum test)")
|
||||
say(" H0: logical block number is drawn from the same distribution")
|
||||
say(" for PASS and FAIL trials (failure does not depend on location).")
|
||||
say(sprintf(" W = %.1f, p = %.3f", wt$statistic, wt$p.value))
|
||||
say(sprintf(" -> %s H0 at alpha=0.05: failure %s associated with LBN.",
|
||||
ifelse(wt$p.value < 0.05, "REJECT", "fail to reject"),
|
||||
ifelse(wt$p.value < 0.05, "IS", "is not")))
|
||||
say("")
|
||||
|
||||
## Secondary check: sibling position within devblock (0/1/2).
|
||||
buggy$sib_pos <- (buggy$lbn - 3076) %% 3
|
||||
pos_tbl <- table(buggy$sib_pos, buggy$status)
|
||||
ct <- suppressWarnings(chisq.test(pos_tbl))
|
||||
say(" Secondary check -- sibling position (0/1/2) vs. outcome (chi-squared)")
|
||||
say(sprintf(" chi-sq = %.3f, df = %d, p = %.3f",
|
||||
ct$statistic, ct$parameter, ct$p.value))
|
||||
say(sprintf(" -> %s H0: outcome %s associated with sibling position.",
|
||||
ifelse(ct$p.value < 0.05, "REJECT", "fail to reject"),
|
||||
ifelse(ct$p.value < 0.05, "IS", "is not")))
|
||||
say("")
|
||||
|
||||
## ---- H3: did the fix change the pass rate, or is 25/50 vs 150/150 ----
|
||||
## ---- explainable by chance? ----
|
||||
## H0: pre-fix and post-fix runs share the same true pass probability.
|
||||
post_passed <- sum(all_runs$run != "amd64_buggy" & all_runs$result == 1)
|
||||
post_n <- sum(all_runs$run != "amd64_buggy")
|
||||
pre_passed <- sum(all_runs$run == "amd64_buggy" & all_runs$result == 1)
|
||||
pre_n <- sum(all_runs$run == "amd64_buggy")
|
||||
|
||||
pt <- prop.test(c(pre_passed, post_passed), c(pre_n, post_n), correct = TRUE)
|
||||
|
||||
say("H3 -- did the fix change the pass rate (two-proportion test)")
|
||||
say(" H0: pre-fix and post-fix trials share the same true pass rate.")
|
||||
say(sprintf(" Pre-fix: %d/%d = %.1f%% Post-fix (pooled, 3 arches): %d/%d = %.1f%%",
|
||||
pre_passed, pre_n, 100*pre_passed/pre_n,
|
||||
post_passed, post_n, 100*post_passed/post_n))
|
||||
say(sprintf(" chi-sq = %.2f, df = %d, p = %.2e", pt$statistic, 1, pt$p.value))
|
||||
say(sprintf(" 95%% CI on the difference in pass rate: [%.3f, %.3f]",
|
||||
pt$conf.int[1], pt$conf.int[2]))
|
||||
say(sprintf(" -> %s H0 at alpha=0.05: the fix %s the pass rate.",
|
||||
ifelse(pt$p.value < 0.05, "REJECT", "fail to reject"),
|
||||
ifelse(pt$p.value < 0.05, "DID CHANGE", "did not measurably change")))
|
||||
|
||||
writeLines(sink_lines, file.path(table_dir, "hypothesis_tests.txt"))
|
||||
|
||||
## Machine-readable version for \newcommand substitution in the report.
|
||||
hyp_df <- data.frame(
|
||||
key = c("h1_runs_obs","h1_runs_exp","h1_z","h1_p",
|
||||
"h1b_W","h1b_p",
|
||||
"h2_W","h2_p","h2b_chisq","h2b_df","h2b_p",
|
||||
"h3_pre_n","h3_pre_pass","h3_post_n","h3_post_pass",
|
||||
"h3_chisq","h3_p","h3_ci_lo","h3_ci_hi"),
|
||||
value = c(rt$runs, rt$expected, rt$z, rt$p,
|
||||
unname(wt_order$statistic), wt_order$p.value,
|
||||
unname(wt$statistic), wt$p.value, unname(ct$statistic), unname(ct$parameter), ct$p.value,
|
||||
pre_n, pre_passed, post_n, post_passed,
|
||||
unname(pt$statistic), pt$p.value, pt$conf.int[1], pt$conf.int[2])
|
||||
)
|
||||
write.csv(hyp_df, file.path(table_dir, "hypothesis_tests.csv"), row.names = FALSE)
|
||||
cat("\nWrote", file.path(table_dir, "hypothesis_tests.txt"), "and .csv\n")
|
||||
@@ -0,0 +1,208 @@
|
||||
<?xml version='1.0' encoding='UTF-8' ?>
|
||||
<svg xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' width='504.00pt' height='187.20pt'
|
||||
viewBox='0 0 504.00 187.20'>
|
||||
<g class='svglite'>
|
||||
<defs>
|
||||
<style type='text/css'><![CDATA[
|
||||
.svglite line, .svglite polyline, .svglite polygon, .svglite path, .svglite rect, .svglite circle {
|
||||
fill: none;
|
||||
stroke: #000000;
|
||||
stroke-linecap: round;
|
||||
stroke-linejoin: round;
|
||||
stroke-miterlimit: 10.00;
|
||||
}
|
||||
.svglite text {
|
||||
white-space: pre;
|
||||
}
|
||||
.svglite g.glyphgroup path {
|
||||
fill: inherit;
|
||||
stroke: none;
|
||||
}
|
||||
]]></style>
|
||||
</defs>
|
||||
<rect width='100%' height='100%' style='stroke: none; fill: #FFFFFF;'/>
|
||||
<defs>
|
||||
<clipPath id='cpMC4wMHw1MDQuMDB8MC4wMHwxODcuMjA='>
|
||||
<rect x='0.00' y='0.00' width='504.00' height='187.20'/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
<g clip-path='url(#cpMC4wMHw1MDQuMDB8MC4wMHwxODcuMjA=)'>
|
||||
<rect x='0.00' y='0.00' width='504.00' height='187.20'
|
||||
style='stroke-width: 1.16; stroke: none; fill: #FFFFFF;'/>
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id='cpNS45OHw0OTguMDJ8NTQuNTN8MTEyLjMz'>
|
||||
<rect x='5.98' y='54.53' width='492.04' height='57.80'/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
<g clip-path='url(#cpNS45OHw0OTguMDJ8NTQuNTN8MTEyLjMz)'>
|
||||
<rect x='28.34' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='37.29' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='46.24' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='55.18' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='64.13' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='73.07' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='82.02' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='90.97' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='99.91' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='108.86' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='117.81' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='126.75' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='135.70' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='144.64' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='153.59' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='162.54' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='171.48' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='180.43' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='189.38' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='198.32' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='207.27' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='216.21' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='225.16' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='234.11' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='243.05' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='252.00' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='260.95' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='269.89' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='278.84' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='287.79' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='296.73' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='305.68' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='314.62' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='323.57' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='332.52' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='341.46' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='350.41' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='359.36' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='368.30' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='377.25' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='386.19' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='395.14' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='404.09' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='413.03' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='421.98' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='430.93' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='439.87' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='448.82' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='457.76' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='466.71' y='57.16' width='8.95' height='52.55'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
</g>
|
||||
<g clip-path='url(#cpMC4wMHw1MDQuMDB8MC4wMHwxODcuMjA=)'>
|
||||
<text x='32.82' y='124.31' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='5.33px'
|
||||
lengthAdjust='spacingAndGlyphs'>0
|
||||
</text>
|
||||
<text x='77.55' y='124.31' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='5.33px'
|
||||
lengthAdjust='spacingAndGlyphs'>5
|
||||
</text>
|
||||
<text x='122.28' y='124.31' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='10.66px'
|
||||
lengthAdjust='spacingAndGlyphs'>10
|
||||
</text>
|
||||
<text x='167.01' y='124.31' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='10.66px'
|
||||
lengthAdjust='spacingAndGlyphs'>15
|
||||
</text>
|
||||
<text x='211.74' y='124.31' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='10.66px'
|
||||
lengthAdjust='spacingAndGlyphs'>20
|
||||
</text>
|
||||
<text x='256.47' y='124.31' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='10.66px'
|
||||
lengthAdjust='spacingAndGlyphs'>25
|
||||
</text>
|
||||
<text x='301.20' y='124.31' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='10.66px'
|
||||
lengthAdjust='spacingAndGlyphs'>30
|
||||
</text>
|
||||
<text x='345.94' y='124.31' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='10.66px'
|
||||
lengthAdjust='spacingAndGlyphs'>35
|
||||
</text>
|
||||
<text x='390.67' y='124.31' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='10.66px'
|
||||
lengthAdjust='spacingAndGlyphs'>40
|
||||
</text>
|
||||
<text x='435.40' y='124.31' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='10.66px'
|
||||
lengthAdjust='spacingAndGlyphs'>45
|
||||
</text>
|
||||
<text x='252.00' y='137.54' text-anchor='middle' style='font-size: 12.00px; font-family: "Liberation Sans";'
|
||||
textLength='165.28px' lengthAdjust='spacingAndGlyphs'>Trial index (= write/verify order)
|
||||
</text>
|
||||
<rect x='204.28' y='158.82' width='15.58' height='15.58'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='258.43' y='158.82' width='15.58' height='15.58'
|
||||
style='stroke-width: 1.28; stroke: #FFFFFF; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<text x='226.69' y='169.90' style='font-size: 9.60px; font-family: "Liberation Sans";' textLength='24.91px'
|
||||
lengthAdjust='spacingAndGlyphs'>PASS
|
||||
</text>
|
||||
<text x='280.83' y='169.90' style='font-size: 9.60px; font-family: "Liberation Sans";' textLength='19.73px'
|
||||
lengthAdjust='spacingAndGlyphs'>FAIL
|
||||
</text>
|
||||
<text x='5.98' y='33.10' style='font-size: 12.00px;fill: #505050; font-family: "Liberation Sans";'
|
||||
textLength='275.38px' lengthAdjust='spacingAndGlyphs'>25 of 50 trials failed; a formal runs test (H1)
|
||||
finds no
|
||||
</text>
|
||||
<text x='5.98' y='46.06' style='font-size: 12.00px;fill: #505050; font-family: "Liberation Sans";'
|
||||
textLength='317.44px' lengthAdjust='spacingAndGlyphs'>significant serial clustering in this sequence
|
||||
-- see Section 4
|
||||
</text>
|
||||
<text x='5.98' y='15.88' style='font-size: 14.40px; font-weight: bold; font-family: "Liberation Sans";'
|
||||
textLength='351.91px' lengthAdjust='spacingAndGlyphs'>Pass/fail sequence in write order, amd64 pre-fix
|
||||
run
|
||||
</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 15 KiB |
@@ -0,0 +1,132 @@
|
||||
<?xml version='1.0' encoding='UTF-8' ?>
|
||||
<svg xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' width='576.00pt' height='302.40pt'
|
||||
viewBox='0 0 576.00 302.40'>
|
||||
<g class='svglite'>
|
||||
<defs>
|
||||
<style type='text/css'><![CDATA[
|
||||
.svglite line, .svglite polyline, .svglite polygon, .svglite path, .svglite rect, .svglite circle {
|
||||
fill: none;
|
||||
stroke: #000000;
|
||||
stroke-linecap: round;
|
||||
stroke-linejoin: round;
|
||||
stroke-miterlimit: 10.00;
|
||||
}
|
||||
.svglite text {
|
||||
white-space: pre;
|
||||
}
|
||||
.svglite g.glyphgroup path {
|
||||
fill: inherit;
|
||||
stroke: none;
|
||||
}
|
||||
]]></style>
|
||||
</defs>
|
||||
<rect width='100%' height='100%' style='stroke: none; fill: #FFFFFF;'/>
|
||||
<defs>
|
||||
<clipPath id='cpMC4wMHw1NzYuMDB8MC4wMHwzMDIuNDA='>
|
||||
<rect x='0.00' y='0.00' width='576.00' height='302.40'/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
<g clip-path='url(#cpMC4wMHw1NzYuMDB8MC4wMHwzMDIuNDA=)'>
|
||||
<rect x='0.00' y='0.000000000000057' width='576.00' height='302.40'
|
||||
style='stroke-width: 1.16; stroke: none; fill: #FFFFFF;'/>
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id='cpODUuOTR8NTcwLjAyfDQxLjU3fDIyNy41Mw=='>
|
||||
<rect x='85.94' y='41.57' width='484.09' height='185.96'/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
<g clip-path='url(#cpODUuOTR8NTcwLjAyfDQxLjU3fDIyNy41Mw==)'>
|
||||
<polyline points='85.94,200.97 570.02,200.97 '
|
||||
style='stroke-width: 1.16; stroke: #EBEBEB; stroke-linecap: butt;'/>
|
||||
<polyline points='85.94,156.69 570.02,156.69 '
|
||||
style='stroke-width: 1.16; stroke: #EBEBEB; stroke-linecap: butt;'/>
|
||||
<polyline points='85.94,112.41 570.02,112.41 '
|
||||
style='stroke-width: 1.16; stroke: #EBEBEB; stroke-linecap: butt;'/>
|
||||
<polyline points='85.94,68.14 570.02,68.14 '
|
||||
style='stroke-width: 1.16; stroke: #EBEBEB; stroke-linecap: butt;'/>
|
||||
<polyline points='107.94,227.53 107.94,41.57 '
|
||||
style='stroke-width: 1.16; stroke: #EBEBEB; stroke-linecap: butt;'/>
|
||||
<polyline points='230.18,227.53 230.18,41.57 '
|
||||
style='stroke-width: 1.16; stroke: #EBEBEB; stroke-linecap: butt;'/>
|
||||
<polyline points='352.43,227.53 352.43,41.57 '
|
||||
style='stroke-width: 1.16; stroke: #EBEBEB; stroke-linecap: butt;'/>
|
||||
<polyline points='474.67,227.53 474.67,41.57 '
|
||||
style='stroke-width: 1.16; stroke: #EBEBEB; stroke-linecap: butt;'/>
|
||||
<rect x='107.94' y='143.41' width='407.48' height='26.57'
|
||||
style='stroke-width: 1.16; stroke: none; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='107.94' y='187.68' width='203.74' height='26.57'
|
||||
style='stroke-width: 1.16; stroke: none; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<rect x='107.94' y='99.13' width='407.48' height='26.57'
|
||||
style='stroke-width: 1.16; stroke: none; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='107.94' y='54.85' width='407.48' height='26.57'
|
||||
style='stroke-width: 1.16; stroke: none; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<text x='515.42' y='152.78' text-anchor='middle' style='font-size: 11.38px; font-family: "Liberation Sans";'
|
||||
textLength='28.47px' lengthAdjust='spacingAndGlyphs'>50/50
|
||||
</text>
|
||||
<text x='311.68' y='197.05' text-anchor='middle' style='font-size: 11.38px; font-family: "Liberation Sans";'
|
||||
textLength='28.47px' lengthAdjust='spacingAndGlyphs'>25/50
|
||||
</text>
|
||||
<text x='515.42' y='108.50' text-anchor='middle' style='font-size: 11.38px; font-family: "Liberation Sans";'
|
||||
textLength='28.47px' lengthAdjust='spacingAndGlyphs'>50/50
|
||||
</text>
|
||||
<text x='515.42' y='64.22' text-anchor='middle' style='font-size: 11.38px; font-family: "Liberation Sans";'
|
||||
textLength='28.47px' lengthAdjust='spacingAndGlyphs'>50/50
|
||||
</text>
|
||||
</g>
|
||||
<g clip-path='url(#cpMC4wMHw1NzYuMDB8MC4wMHwzMDIuNDA=)'>
|
||||
<text x='80.56' y='204.26' text-anchor='end'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='64.97px'
|
||||
lengthAdjust='spacingAndGlyphs'>amd64 (pre-fix)
|
||||
</text>
|
||||
<text x='80.56' y='159.99' text-anchor='end'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='74.58px'
|
||||
lengthAdjust='spacingAndGlyphs'>aarch64 (post-fix)
|
||||
</text>
|
||||
<text x='80.56' y='115.71' text-anchor='end'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='69.25px'
|
||||
lengthAdjust='spacingAndGlyphs'>amd64 (post-fix)
|
||||
</text>
|
||||
<text x='80.56' y='71.43' text-anchor='end'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='70.31px'
|
||||
lengthAdjust='spacingAndGlyphs'>riscv64 (post-fix)
|
||||
</text>
|
||||
<text x='107.94' y='239.51' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='13.86px'
|
||||
lengthAdjust='spacingAndGlyphs'>0%
|
||||
</text>
|
||||
<text x='230.18' y='239.51' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='19.19px'
|
||||
lengthAdjust='spacingAndGlyphs'>30%
|
||||
</text>
|
||||
<text x='352.43' y='239.51' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='19.19px'
|
||||
lengthAdjust='spacingAndGlyphs'>60%
|
||||
</text>
|
||||
<text x='474.67' y='239.51' text-anchor='middle'
|
||||
style='font-size: 9.60px;fill: #4D4D4D; font-family: "Liberation Sans";' textLength='19.19px'
|
||||
lengthAdjust='spacingAndGlyphs'>90%
|
||||
</text>
|
||||
<text x='327.98' y='252.74' text-anchor='middle' style='font-size: 12.00px; font-family: "Liberation Sans";'
|
||||
textLength='50.67px' lengthAdjust='spacingAndGlyphs'>Pass rate
|
||||
</text>
|
||||
<rect x='273.74' y='273.94' width='15.73' height='15.73'
|
||||
style='stroke-width: 1.16; stroke: none; stroke-linecap: butt; stroke-linejoin: miter; fill: #2CA02C;'/>
|
||||
<rect x='333.88' y='273.94' width='15.73' height='15.73'
|
||||
style='stroke-width: 1.16; stroke: none; stroke-linecap: butt; stroke-linejoin: miter; fill: #D62728;'/>
|
||||
<text x='296.22' y='285.10' style='font-size: 9.60px; font-family: "Liberation Sans";' textLength='30.91px'
|
||||
lengthAdjust='spacingAndGlyphs'>post-fix
|
||||
</text>
|
||||
<text x='356.37' y='285.10' style='font-size: 9.60px; font-family: "Liberation Sans";' textLength='26.62px'
|
||||
lengthAdjust='spacingAndGlyphs'>pre-fix
|
||||
</text>
|
||||
<text x='85.94' y='33.10' style='font-size: 12.00px;fill: #505050; font-family: "Liberation Sans";'
|
||||
textLength='387.47px' lengthAdjust='spacingAndGlyphs'>50 trials per run, unique random blocks across
|
||||
the 22998-block data pool
|
||||
</text>
|
||||
<text x='85.94' y='15.88' style='font-size: 14.40px; font-weight: bold; font-family: "Liberation Sans";'
|
||||
textLength='360.70px' lengthAdjust='spacingAndGlyphs'>Artemis surface stress test: pass rate
|
||||
before/after fix
|
||||
</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 7.8 KiB |
File diff suppressed because it is too large
Load Diff
|
After Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 41 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 133 KiB |
@@ -0,0 +1,7 @@
|
||||
*.aux
|
||||
*.log
|
||||
*.out
|
||||
*.toc
|
||||
*.synctex.gz
|
||||
*.fls
|
||||
*.fdb_latexmk
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,5 @@
|
||||
"run","label","n","passed","pass_rate","phase"
|
||||
"aarch64_fixed","aarch64 (post-fix)",50,50,1,"post-fix"
|
||||
"amd64_buggy","amd64 (pre-fix)",50,25,0.5,"pre-fix"
|
||||
"amd64_fixed","amd64 (post-fix)",50,50,1,"post-fix"
|
||||
"riscv64_fixed","riscv64 (post-fix)",50,50,1,"post-fix"
|
||||
|
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert all SVGs in charts/ to PDF for LaTeX \\includegraphics.
|
||||
|
||||
Mirrors experiments/bare_metal/analysis/svg_to_pdf.py, scoped to this
|
||||
experiment's own charts/report/figures directories.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
CHARTS = Path(__file__).parent / "charts"
|
||||
PDFS = Path(__file__).parent / "report" / "figures"
|
||||
PDFS.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
svgs = sorted(CHARTS.glob("*.svg"))
|
||||
if not svgs:
|
||||
print("No SVGs found in", CHARTS)
|
||||
sys.exit(1)
|
||||
|
||||
ok = fail = 0
|
||||
for svg in svgs:
|
||||
pdf = PDFS / svg.with_suffix(".pdf").name
|
||||
result = subprocess.run(
|
||||
["rsvg-convert", "--format=pdf", "--output", str(pdf), str(svg)],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print(f" OK {svg.name} -> {pdf.name}")
|
||||
ok += 1
|
||||
else:
|
||||
print(f" ERR {svg.name}: {result.stderr.strip()}")
|
||||
fail += 1
|
||||
|
||||
print(f"\n{ok} converted, {fail} failed.")
|
||||
sys.exit(0 if fail == 0 else 1)
|
||||
@@ -0,0 +1,20 @@
|
||||
"key","value"
|
||||
"h1_runs_obs",26
|
||||
"h1_runs_exp",26
|
||||
"h1_z",0
|
||||
"h1_p",1
|
||||
"h1b_W",332
|
||||
"h1b_p",0.714773975516392
|
||||
"h2_W",505
|
||||
"h2_p",0.000107733285139378
|
||||
"h2b_chisq",0.790350877192982
|
||||
"h2b_df",2
|
||||
"h2b_p",0.67356186013519
|
||||
"h3_pre_n",50
|
||||
"h3_pre_pass",25
|
||||
"h3_post_n",150
|
||||
"h3_post_pass",150
|
||||
"h3_chisq",81.2038095238095
|
||||
"h3_p",2.03599124928093e-19
|
||||
"h3_ci_lo",-0.651923715768301
|
||||
"h3_ci_hi",-0.348076284231699
|
||||
|
@@ -0,0 +1,32 @@
|
||||
Artemis surface stress test -- hypothesis tests
|
||||
=================================================
|
||||
|
||||
H1 -- pre-fix failure clustering (Wald-Wolfowitz runs test)
|
||||
H0: pass/fail outcomes are independently ordered (no clustering).
|
||||
Observed runs = 26 (expected under H0 = 26.00, sd = 3.50)
|
||||
z = 0.000, p = 1.00e+00
|
||||
-> fail to reject H0 at alpha=0.05: failures are not serially clustered.
|
||||
|
||||
H1b -- pre-fix outcome vs. trial/write order directly (Wilcoxon)
|
||||
H0: trial index is drawn from the same distribution for PASS/FAIL
|
||||
(complements H1: the runs test only catches consecutive-value
|
||||
clustering, not a general rank-order trend).
|
||||
W = 332.0, p = 0.715
|
||||
-> fail to reject H0 at alpha=0.05: outcome is not associated with write order.
|
||||
|
||||
H2 -- pre-fix failure vs. disk location (Wilcoxon rank-sum test)
|
||||
H0: logical block number is drawn from the same distribution
|
||||
for PASS and FAIL trials (failure does not depend on location).
|
||||
W = 505.0, p = 0.000
|
||||
-> REJECT H0 at alpha=0.05: failure IS associated with LBN.
|
||||
|
||||
Secondary check -- sibling position (0/1/2) vs. outcome (chi-squared)
|
||||
chi-sq = 0.790, df = 2, p = 0.674
|
||||
-> fail to reject H0: outcome is not associated with sibling position.
|
||||
|
||||
H3 -- did the fix change the pass rate (two-proportion test)
|
||||
H0: pre-fix and post-fix trials share the same true pass rate.
|
||||
Pre-fix: 25/50 = 50.0% Post-fix (pooled, 3 arches): 150/150 = 100.0%
|
||||
chi-sq = 81.20, df = 1, p = 2.04e-19
|
||||
95% CI on the difference in pass rate: [-0.652, -0.348]
|
||||
-> REJECT H0 at alpha=0.05: the fix DID CHANGE the pass rate.
|
||||
Reference in New Issue
Block a user