/* * xhci.c — xHCI USB host controller driver for StarKernel: PCI discovery * (Milestone 2b), controller bring-up (2c), polled Event Ring servicing * (2d), and Command Ring submission + PORTSC connect/disconnect detection * (2e, in progress). Enumeration/BOT read-write (2f-2g) follow in later * increments. * * Memory model: BAR0 is mapped identity (virtual address == physical * address), matching virtio_blk.c's precedent and pci_map_bar()'s own * documented behavior (amd64: vmm_map_range(phys, phys, ...); other * arches: no-op, UEFI identity map already covers it). */ #ifndef __STARKERNEL__ #error "xhci.c is kernel-only" #endif #include #include #include #include "starkernel/pci.h" #include "starkernel/xhci.h" #include "starkernel/xhci_driver.h" #include "starkernel/usb_kbd.h" /* A3: a boot keyboard's reports and lights */ #include "starkernel/console.h" /* A3: "USB keyboard ready" */ #include "starkernel/kmalloc.h" #include "starkernel/timer.h" #include "starkernel/arch.h" #include "log.h" /* BAR0 is mapped at the size the BAR itself reports (pci_bar_size(), the * standard probe), never less than 64 KiB: the Capability, Operational, * Port, Runtime and Doorbell registers and the extended capabilities all * live in it, and a real controller's BAR is commonly larger than QEMU's * (2026-10-14, boards/BRINGUP.md A1). */ #define XHCI_BAR0_MIN_MAP_SIZE 0x10000ull /* xHCI is PCI class 0x0C (serial bus), subclass 0x03 (USB), programming * interface 0x30 (xHCI), whoever makes it. Until 2026-10-14 the controller * was matched by QEMU's own vendor and device (1B36:000D) on bus 0 alone, * so no real board's was ever found (A1). */ #define XHCI_PCI_CLASS 0x0Cu #define XHCI_PCI_SUBCLASS 0x03u #define XHCI_PCI_PROG_IF 0x30u static int xhci_wait_bit(volatile uint32_t *reg, uint32_t mask, int want_set, uint64_t max_ticks); static void xhci_log_hex32(const char *prefix, uint32_t v); /* Take the controller from the firmware (xHCI 1.2 section 4.22.1): a BIOS * that drives USB itself (a keyboard in its setup screen, booting from a * stick) owns the controller until the OS asks for it through the USB * Legacy Support extended capability, and its SMIs keep firing behind the * OS's back until they are switched off. QEMU's controller has no such * capability; every PC's has. */ #define XHCI_XCAP_LEGACY 1u #define XHCI_LEGSUP_BIOS_OWNED (1u << 16) #define XHCI_LEGSUP_OS_OWNED (1u << 24) static void xhci_take_from_bios(xhci_dev_t *dev) { uint32_t off = (dev->cap->hcc_params1 >> 16) & 0xFFFFu; /* in dwords */ unsigned guard = 0; while (off && guard++ < 64) { volatile uint32_t *cap = (volatile uint32_t *)((uint8_t *)dev->cap + (size_t)off * 4u); uint32_t v = *cap; if ((v & 0xFFu) == XHCI_XCAP_LEGACY) { if (v & XHCI_LEGSUP_BIOS_OWNED) { *cap = v | XHCI_LEGSUP_OS_OWNED; if (xhci_wait_bit(cap, XHCI_LEGSUP_BIOS_OWNED, 0, 100) != 0) { /* The firmware did not let go within a second: take * it anyway, as Linux does. */ *cap = (*cap & ~XHCI_LEGSUP_BIOS_OWNED) | XHCI_LEGSUP_OS_OWNED; log_message(LOG_WARN, "xhci: firmware did not hand the controller over -- taken"); } } else { *cap = v | XHCI_LEGSUP_OS_OWNED; } /* USBLEGCTLSTS: every SMI enable off (bits 0, 4, 13-15), every * SMI status cleared (write-1-to-clear, bits 29-31). */ cap[1] = (cap[1] & ~((1u << 0) | (1u << 4) | (7u << 13))) | (7u << 29); log_message(LOG_DEBUG, "xhci: controller taken from the firmware"); return; } if (((v >> 8) & 0xFFu) == 0) return; off += (v >> 8) & 0xFFu; } } int xhci_find_and_map(xhci_dev_t *dev, unsigned index) { uint64_t map_size; if (!dev) return -1; if (pci_find_class(XHCI_PCI_CLASS, XHCI_PCI_SUBCLASS, XHCI_PCI_PROG_IF, index, &dev->pci) != 0) { return 1; /* no index-th controller: there are fewer */ } xhci_log_hex32("xhci: controller vendor/device=", ((uint32_t)dev->pci.vendor_id << 16) | dev->pci.device_id); pci_enable(&dev->pci); dev->bar0_phys = pci_bar(&dev->pci, 0); if (!dev->bar0_phys) { log_message(LOG_ERROR, "xhci: BAR0 read failed (zero or I/O BAR)"); return -1; } map_size = pci_bar_size(&dev->pci, 0); if (map_size < XHCI_BAR0_MIN_MAP_SIZE) map_size = XHCI_BAR0_MIN_MAP_SIZE; if (pci_map_bar(dev->bar0_phys, map_size) != 0) { log_message(LOG_ERROR, "xhci: BAR0 mapping failed"); return -2; } dev->cap = (xhci_cap_regs_t *)(uintptr_t)dev->bar0_phys; dev->op = (xhci_op_regs_t *)((uint8_t *)dev->cap + dev->cap->cap_length); dev->runtime = (xhci_runtime_regs_t *)((uint8_t *)dev->cap + (dev->cap->rts_off & ~0x1Fu)); dev->doorbell = (xhci_doorbell_t *)((uint8_t *)dev->cap + (dev->cap->db_off & ~0x3u)); uint32_t hcs1 = dev->cap->hcs_params1; dev->max_slots = XHCI_HCSPARAMS1_MAX_SLOTS(hcs1); dev->max_intrs = XHCI_HCSPARAMS1_MAX_INTRS(hcs1); dev->max_ports = XHCI_HCSPARAMS1_MAX_PORTS(hcs1); dev->max_scratchpad_bufs = XHCI_HCSPARAMS2_MAX_SCRATCHPAD_BUFS(dev->cap->hcs_params2); xhci_take_from_bios(dev); log_message(LOG_DEBUG, "xhci: controller found, BAR0 mapped"); return 0; } /* Spin-count-bounded busy-wait, matching kernel_main.c's established * pattern (heartbeat_ticks() elapsed + a hard spin-count safety cap, not * just one or the other). Polls *reg for (val & mask) to equal want_set * (0 or 1), re-reading the register itself each iteration. */ static int xhci_wait_bit(volatile uint32_t *reg, uint32_t mask, int want_set, uint64_t max_ticks) { uint64_t start = heartbeat_ticks(); uint64_t spins = 0; for (;;) { uint32_t val = *reg; int is_set = (val & mask) != 0; if (is_set == want_set) return 0; spins++; if (heartbeat_ticks() - start >= max_ticks || spins >= 100000000ULL) { return -1; } /* FABRIC-3.md, 2026-09-09: a tight loop of nothing but MMIO reads * (no PAUSE/yield hint) can starve TCG's own host-side timer * injection under QEMU -- live-caught heartbeat_ticks() itself * (this loop's own timeout clock!) freezing for 30+ seconds * straight during a real reproduction, confirmed via a clean, * uninterfered-with measurement. That leaves only the 100M hard * spin cap as a backstop, which is far too permissive on its own. * arch_relax() (PAUSE on amd64) costs nothing on real hardware but * gives TCG a natural yield point between MMIO polls. */ arch_relax(); } } /* log_message()'s own %x has no zero-padding/width control (starkernel/ * log.h's kvsnprintf() is a minimal freestanding formatter), so this * small hex-dump helper is kept for its fixed 8-digit "0xXXXXXXXX" width * -- register values line up in the log this way. Routed through * log_message(LOG_DEBUG, ...) rather than console_puts()/console_println() * directly (was the reason this helper existed at all, before log.h's * formatted print was available to this driver) so every xHCI register * dump is silenced at the default LOG_INFO level, same as the rest of * this file's routine chatter -- re-enable with --log-level=debug. */ static void xhci_log_hex32(const char *label, uint32_t val) { char buf[11]; buf[0] = '0'; buf[1] = 'x'; for (int i = 9; i >= 2; i--) { int d = val & 0xF; buf[i] = (d < 10) ? ('0' + d) : ('a' + d - 10); val >>= 4; } buf[10] = '\0'; log_message(LOG_DEBUG, "%s%s", label, buf); } /* Every controller brought up, in the order found (2026-10-14: a PC has * several, and a stick or keyboard may be in any of their ports -- until * then only the first was ever driven). Latched at the end of a * successful xhci_bringup(); xhci_poll_events() services each in turn. */ static xhci_dev_t *g_xhci_devs[XHCI_MAX_CONTROLLERS]; static uint32_t g_xhci_dev_count; /* Forward declaration -- xhci_bringup() below issues one Enable Slot as a * command-ring smoke test; the implementation lives after xhci_bringup() * (see the "Command Ring submission" section) so it can stay close to * xhci_poll_events(), the read side of the same ring pair. */ int xhci_cmd_enable_slot(xhci_dev_t *dev); int xhci_cmd_disable_slot(xhci_dev_t *dev, uint32_t slot_id); int xhci_cmd_address_device(xhci_dev_t *dev, uint32_t slot_id, uint32_t port_id, uint32_t speed); int xhci_cmd_configure_endpoint(xhci_dev_t *dev, uint32_t slot_id); int xhci_cmd_reset_endpoint(xhci_dev_t *dev, uint32_t slot_id, uint32_t ep_id); int xhci_cmd_set_tr_dequeue_pointer(xhci_dev_t *dev, uint32_t slot_id, uint32_t ep_id, uint64_t new_dequeue, uint32_t dcs); int xhci_bot_send_read10(xhci_dev_t *dev, uint32_t slot_id, uint32_t lba, uint16_t num_blocks, uint32_t block_size); int xhci_bot_send_write10(xhci_dev_t *dev, uint32_t slot_id, uint32_t lba, uint16_t num_blocks, uint32_t block_size); int xhci_bot_send_test_unit_ready(xhci_dev_t *dev, uint32_t slot_id); int xhci_bot_send_read_capacity10(xhci_dev_t *dev, uint32_t slot_id); int xhci_bot_get_capacity(xhci_dev_t *dev, uint32_t slot_id); int xhci_bot_read_data_in(xhci_dev_t *dev, uint32_t slot_id); int xhci_bot_write_data_out(xhci_dev_t *dev, uint32_t slot_id); int xhci_bot_receive_csw(xhci_dev_t *dev, uint32_t slot_id); int xhci_ep0_get_device_descriptor(xhci_dev_t *dev, uint32_t slot_id); int xhci_ep0_get_config_descriptor(xhci_dev_t *dev, uint32_t slot_id, uint16_t length); int xhci_ep0_set_configuration(xhci_dev_t *dev, uint32_t slot_id, uint8_t config_value); /* Forward declaration -- xhci_bringup() below calls this once the * controller is running, to catch a device that was already connected * (present on the QEMU command line at launch, not hot-plugged after boot). * The implementation lives after xhci_poll_events() since it shares * xhci_port_regs() and the same connect-handling logic. See xhci_scan_ * ports_for_already_connected()'s own doc comment for why this exists. */ static void xhci_scan_ports_for_already_connected(xhci_dev_t *dev); int xhci_bringup(xhci_dev_t *dev) { if (!dev || !dev->op) return -2; /* 1. If running, stop first (Run/Stop must be cleared before HCRST is * guaranteed to behave per spec on some implementations). */ if (dev->op->usb_cmd & XHCI_USBCMD_RUN) { dev->op->usb_cmd &= ~XHCI_USBCMD_RUN; if (xhci_wait_bit(&dev->op->usb_sts, XHCI_USBSTS_HCH, 1, 10000) != 0) { log_message(LOG_ERROR, "xhci: timeout waiting for halt before reset"); return -1; } } /* 2. Host Controller Reset. HCRST self-clears; CNR (Controller Not * Ready) must also clear before touching any other operational * register. */ dev->op->usb_cmd |= XHCI_USBCMD_HCRST; if (xhci_wait_bit(&dev->op->usb_cmd, XHCI_USBCMD_HCRST, 0, 10000) != 0) { log_message(LOG_ERROR, "xhci: timeout waiting for HCRST to self-clear"); return -1; } if (xhci_wait_bit(&dev->op->usb_sts, XHCI_USBSTS_CNR, 0, 10000) != 0) { log_message(LOG_ERROR, "xhci: timeout waiting for CNR to clear"); return -1; } /* 3. Device Context Base Address Array — (max_slots+1) x 8-byte * pointers, 64-byte aligned, zeroed (DCBAAP requires 64-byte * alignment per spec; kmalloc_aligned enforces it). */ size_t dcbaa_bytes = (size_t)(dev->max_slots + 1) * sizeof(uint64_t); dev->dcbaa = kmalloc_aligned(dcbaa_bytes, 64); if (!dev->dcbaa) { log_message(LOG_ERROR, "xhci: DCBAA allocation failed"); return -2; } for (size_t i = 0; i < dcbaa_bytes / sizeof(uint64_t); i++) { ((uint64_t *)dev->dcbaa)[i] = 0; } /* 3b. Scratchpad buffers, only if the controller asks for them (slot 0 * of the DCBAA points at the scratchpad buffer array, not a device * context, when max_scratchpad_bufs > 0). PAGESIZE register: bit N * set means 2^(N+12)-byte pages; use the lowest set bit found. */ if (dev->max_scratchpad_bufs > 0) { uint32_t pagesize_bits = dev->op->page_size; uint32_t page_bytes = 4096u; for (uint32_t b = 0; b < 16; b++) { if (pagesize_bits & (1u << b)) { page_bytes = 1u << (b + 12); break; } } size_t arr_bytes = (size_t)dev->max_scratchpad_bufs * sizeof(uint64_t); dev->scratchpad_arr = kmalloc_aligned(arr_bytes, 64); if (!dev->scratchpad_arr) { log_message(LOG_ERROR, "xhci: scratchpad array allocation failed"); return -2; } for (uint32_t i = 0; i < dev->max_scratchpad_bufs; i++) { void *buf = kmalloc_aligned(page_bytes, page_bytes); if (!buf) { log_message(LOG_ERROR, "xhci: scratchpad buffer allocation failed"); return -2; } ((uint64_t *)dev->scratchpad_arr)[i] = (uint64_t)(uintptr_t)buf; } ((uint64_t *)dev->dcbaa)[0] = (uint64_t)(uintptr_t)dev->scratchpad_arr; } dev->op->dcbaap = (uint64_t)(uintptr_t)dev->dcbaa; /* FABRIC-3.md §VII (2026-09-05): per-slot device state registry, sized * off the same (max_slots+1) input as the DCBAA allocation just above * -- same kmalloc_aligned() pattern, no separate sizing policy. Zeroed * so every slot starts with in_use=0 and every pointer field NULL, * removing the need for this function's old per-boot "reset the single * device's fields to their idle values" block below (that block reset * exactly one implicit device; there is no single device to reset * anymore -- each slot starts clean from this allocation and is * populated when a real device actually attaches to it). */ size_t msc_slots_bytes = (size_t)(dev->max_slots + 1) * sizeof(xhci_msc_slot_t); dev->msc_slots = (xhci_msc_slot_t *)kmalloc_aligned(msc_slots_bytes, 64); if (!dev->msc_slots) { log_message(LOG_ERROR, "xhci: msc_slots registry allocation failed"); return -2; } memset(dev->msc_slots, 0, msc_slots_bytes); /* 4. Command Ring — XHCI_RING_TRB_COUNT TRBs, 64-byte aligned, zeroed. * Initial Ring Cycle State = 1 (software convention; the ring is * "owned" by software until the first TRB with a matching cycle bit * is consumed). CRCR low bits carry RCS, not the TRBs themselves. */ dev->cmd_ring = (xhci_trb_t *)kmalloc_aligned(XHCI_RING_BYTES, 64); if (!dev->cmd_ring) { log_message(LOG_ERROR, "xhci: command ring allocation failed"); return -2; } for (uint32_t i = 0; i < XHCI_RING_TRB_COUNT; i++) { dev->cmd_ring[i].parameter = 0; dev->cmd_ring[i].status = 0; dev->cmd_ring[i].control = 0; } dev->cmd_ring_cycle = 1; dev->cmd_ring_enq = 0; /* Last slot is a permanent Link TRB back to index 0 (xHCI 1.2 spec * §4.9.2 — software must terminate every ring segment with one; without * it the controller reads uninitialised memory past the segment instead * of wrapping). Toggle Cycle (TC) tells the controller to flip its own * consumer cycle state when it processes this TRB, matching the * producer-side toggle xhci_submit_command() below performs on wrap. */ dev->cmd_ring[XHCI_RING_TRB_COUNT - 1].parameter = (uint64_t)(uintptr_t)dev->cmd_ring; dev->cmd_ring[XHCI_RING_TRB_COUNT - 1].control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_TC | XHCI_TRB_CONTROL_CYCLE; dev->op->crcr = ((uint64_t)(uintptr_t)dev->cmd_ring & XHCI_CRCR_PTR_MASK) | XHCI_CRCR_RCS; /* 5. Event Ring — one segment (Event Ring Segment Table with a single * 16-byte entry: base address + size), wired to Interrupter 0. * Interrupter Register Sets start at runtime_base + 0x20; each is * sizeof(xhci_intr_regs_t) apart, but only Interrupter 0 is used * (single-interrupter design decided in 2a). */ dev->evt_ring = (xhci_trb_t *)kmalloc_aligned(XHCI_RING_BYTES, 64); if (!dev->evt_ring) { log_message(LOG_ERROR, "xhci: event ring allocation failed"); return -2; } for (uint32_t i = 0; i < XHCI_RING_TRB_COUNT; i++) { dev->evt_ring[i].parameter = 0; dev->evt_ring[i].status = 0; dev->evt_ring[i].control = 0; } dev->evt_ring_cycle = 1; dev->evt_ring_deq = 0; /* Event Ring Segment Table entry layout: u64 base + u32 size + u32 * reserved = 16 bytes. One segment is enough (ERST Max >= 1 always). */ dev->evt_ring_seg_table = kmalloc_aligned(16, 64); if (!dev->evt_ring_seg_table) { log_message(LOG_ERROR, "xhci: event ring segment table allocation failed"); return -2; } uint64_t *erst = (uint64_t *)dev->evt_ring_seg_table; erst[0] = (uint64_t)(uintptr_t)dev->evt_ring; /* base address */ erst[1] = (uint64_t)XHCI_RING_TRB_COUNT; /* size, low 32 bits used */ dev->intr0 = (xhci_intr_regs_t *)((uint8_t *)dev->runtime + 0x20); xhci_intr_regs_t *intr0 = dev->intr0; intr0->erstsz = 1; intr0->erstba = (uint64_t)(uintptr_t)dev->evt_ring_seg_table; intr0->erdp = ((uint64_t)(uintptr_t)dev->evt_ring & XHCI_ERDP_PTR_MASK); /* 6. Enable device slots (all of them — no reason to restrict for a * single-drive-at-a-time driver) and start the controller. */ dev->op->config = XHCI_CONFIG_MAX_SLOTS_EN(dev->max_slots); dev->op->usb_cmd |= XHCI_USBCMD_RUN; if (xhci_wait_bit(&dev->op->usb_sts, XHCI_USBSTS_HCH, 0, 10000) != 0) { log_message(LOG_ERROR, "xhci: controller did not leave halted state after RUN"); return -3; } /* Milestone 2e: per-port Enable Slot correlation state -- fixed array, * see xhci_dev_t's own doc comment; no allocation needed. */ for (uint32_t i = 0; i < XHCI_MAX_TRACKED_PORTS; i++) { dev->port_slot_id[i] = 0; dev->pending_events[i].valid = 0; } dev->pending_connect_port_id = 0; dev->pending_connect_speed = 0; dev->connect_state = XHCI_CONN_IDLE; dev->pending_connect_slot_id = 0; dev->pending_disable_slot_id = 0; /* transfer_purpose is per-slot now (msc_slots[], zeroed by that * allocation's own memset). */ dev->bot_next_tag = 1; dev->bot_last_tag = 0; dev->bot_expected_data_len = 0; dev->bot_cmd_kind = BOT_CMD_NONE; dev->bot_last_status = BOT_STATUS_IDLE; dev->bot_tur_chain_target = BOT_TUR_CHAIN_NONE; dev->bot_tur_retries = 0; dev->bot_read10_lba = 0; dev->bot_read10_num_blocks = 0; dev->bot_read10_block_size = 0; dev->bot_cap_last_lba = 0; dev->bot_cap_block_size = 0; /* next_action fields are per-slot now (msc_slots[], zeroed by that * allocation's own memset) -- nothing to reset here. */ log_message(LOG_DEBUG, "xhci: controller running"); /* Milestone 2e prep: HCCPARAMS1.CSZ decides 32- vs 64-byte Slot/ * Endpoint/Input Context layout for Address Device -- must be read * live, not assumed, before any context structure is designed. */ xhci_log_hex32("xhci: hcc_params1=", dev->cap->hcc_params1); log_message(LOG_DEBUG, XHCI_HCCPARAMS1_CSZ(dev->cap->hcc_params1) ? "xhci: context size = 64 bytes" : "xhci: context size = 32 bytes"); if (g_xhci_dev_count < XHCI_MAX_CONTROLLERS) g_xhci_devs[g_xhci_dev_count++] = dev; /* Catch a device already connected at launch -- xhci_poll_events() is * purely event-ring-driven (Port Status Change events only), and a * device present on the QEMU command line before this controller reset * never generates one (nothing "changed" from the controller's * perspective once it starts looking). g_xhci_dev must be set first -- * this reuses the same connect-handling path xhci_poll_events() uses, * which reads it as a singleton rather than taking dev as an arg. */ xhci_scan_ports_for_already_connected(dev); return 0; } uint32_t xhci_dev_count(void) { return g_xhci_dev_count; } xhci_dev_t *xhci_dev_at(uint32_t i) { return i < g_xhci_dev_count ? g_xhci_devs[i] : NULL; } xhci_msc_slot_t *xhci_msc_slot_for(xhci_dev_t *dev, uint32_t slot_id) { if (!dev || !dev->msc_slots) return NULL; if (slot_id < 1 || slot_id > dev->max_slots) return NULL; return &dev->msc_slots[slot_id]; } /* ------------------------------------------------------------------------- * Command Ring submission -- Milestone 2e. Shared by Enable Slot now and * Address Device next; xhci_poll_events() above is the read side of this * same ring pair, already verified live for Port Status Change events. * ------------------------------------------------------------------------- */ /* extra_control_bits ORs additional fields into the TRB's control dword * beyond type/cycle -- e.g. Address Device's Slot ID at bits[31:24] * (Enable Slot needs none, passes 0). Never includes the cycle bit itself; * that's computed here from cmd_ring_cycle so callers can't get it wrong. */ static void xhci_submit_command(xhci_dev_t *dev, uint64_t parameter, uint32_t status, uint32_t trb_type, uint32_t extra_control_bits) { xhci_trb_t *trb = &dev->cmd_ring[dev->cmd_ring_enq]; trb->parameter = parameter; trb->status = status; trb->control = (trb_type << XHCI_TRB_CONTROL_TYPE_SHIFT) | extra_control_bits | (dev->cmd_ring_cycle ? XHCI_TRB_CONTROL_CYCLE : 0); dev->cmd_ring_enq++; if (dev->cmd_ring_enq == XHCI_RING_TRB_COUNT - 1) { /* About to hand the Link TRB to the controller -- its cycle bit * must match the producer cycle state at the moment of handoff, * and the producer state flips here too (this is the wrap). */ dev->cmd_ring[XHCI_RING_TRB_COUNT - 1].control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_TC | (dev->cmd_ring_cycle ? XHCI_TRB_CONTROL_CYCLE : 0); dev->cmd_ring_enq = 0; dev->cmd_ring_cycle ^= 1u; } /* Doorbell 0 targets the Command Ring (XHCI_DB_TARGET(0)); write-only, * one write per new TRB posted -- xhci.h's own doc comment on the * Doorbell Array. */ dev->doorbell[0] = XHCI_DB_TARGET(0); } int xhci_cmd_enable_slot(xhci_dev_t *dev) { if (!dev || !dev->cmd_ring) return -1; xhci_submit_command(dev, 0, 0, XHCI_TRB_TYPE_ENABLE_SLOT_CMD, 0); log_message(LOG_DEBUG, "xhci: enable slot command submitted"); return 0; } int xhci_cmd_disable_slot(xhci_dev_t *dev, uint32_t slot_id) { if (!dev || !dev->cmd_ring) return -1; /* Slot ID goes in control[31:24], same field Address Device uses -- * no parameter/status payload needed, this command just names a slot. */ xhci_submit_command(dev, 0, 0, XHCI_TRB_TYPE_DISABLE_SLOT_CMD, slot_id << 24); log_message(LOG_DEBUG, "xhci: disable slot command submitted"); return 0; } /* Default EP0 Max Packet Size by PORTSC.Port Speed, used before any device * descriptor has been read (xHCI 1.2 spec's own recommended defaults -- * the real value comes from bMaxPacketSize0 once 2f reads the device * descriptor and issues an Evaluate Context to correct it if needed). */ static uint32_t xhci_default_ep0_max_packet(uint32_t speed) { switch (speed) { case 4: return 512; /* SuperSpeed */ case 3: return 64; /* High Speed */ case 2: return 8; /* Low Speed */ default: return 64; /* Full Speed (1) and anything unrecognised */ } } /* Every context -- Input Control, Slot, Endpoint -- is 32 bytes, or 64 * where HCCPARAMS1.CSZ says so (xHCI 1.2 section 6.2.1); a 64-byte * context is the same fields in its first 32 bytes, then reserved space. * So the 32-byte structs (xhci.h) stay right for every field, and only * where each context begins depends on CSZ. Until 2026-10-14 a CSZ=1 * controller was refused outright; QEMU's has CSZ=0, many real ones have * CSZ=1 (boards/BRINGUP.md A1). */ static size_t xhci_ctx_bytes(const xhci_dev_t *dev) { return XHCI_HCCPARAMS1_CSZ(dev->cap->hcc_params1) ? 64u : 32u; } int xhci_cmd_address_device(xhci_dev_t *dev, uint32_t slot_id, uint32_t port_id, uint32_t speed) { if (!dev || !dev->cmd_ring || !dev->dcbaa) return -1; xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!ms) return -1; const size_t cs = xhci_ctx_bytes(dev); /* 32 or 64, by HCCPARAMS1.CSZ */ /* Lazily allocate once per slot; reused across a slot's own re-attach * (FABRIC-3.md §VII, 2026-09-05: per-slot now, not single-device-wide). * All three re-initialised fully below regardless of whether this is * the first call for this slot. */ if (!ms->input_ctx) { ms->input_ctx = kmalloc_aligned(3 * cs, 64); /* Input Control, Slot, EP0 */ if (!ms->input_ctx) return -1; } if (!ms->device_ctx) { ms->device_ctx = kmalloc_aligned(2 * cs, 64); /* Slot, EP0 */ if (!ms->device_ctx) return -1; } if (!ms->ep0_ring) { ms->ep0_ring = (xhci_trb_t *)kmalloc_aligned(XHCI_RING_BYTES, 64); if (!ms->ep0_ring) return -1; } ms->in_use = 1; ms->hid_kbd = 0; /* a new device in this slot: not (yet) a keyboard */ ms->hid_running = 0; ms->hid_ep_addr = 0; /* EP0 Transfer Ring: same fixed-ring-plus-Link-TRB pattern as the * Command Ring (xhci_bringup()'s own comment on why). Freshly * reinitialised on every call, not just the first -- cheap (4KiB) and * avoids carrying stale TRBs from a previous connect on this slot. */ for (uint32_t i = 0; i < XHCI_RING_TRB_COUNT; i++) { ms->ep0_ring[i].parameter = 0; ms->ep0_ring[i].status = 0; ms->ep0_ring[i].control = 0; } ms->ep0_ring[XHCI_RING_TRB_COUNT - 1].parameter = (uint64_t)(uintptr_t)ms->ep0_ring; ms->ep0_ring[XHCI_RING_TRB_COUNT - 1].control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_TC | XHCI_TRB_CONTROL_CYCLE; ms->ep0_ring_cycle = 1; ms->ep0_ring_enq = 0; /* Device Context: Slot Context followed by EP0 Context, no Input * Control Context (that only exists in the Input Context below). * DCBAA[slot_id] must point here, per spec -- zeroed since the * controller writes this on Address Device success, software must not * pre-fill it. */ uint8_t *dctx = (uint8_t *)ms->device_ctx; for (size_t i = 0; i < 2 * cs; i++) dctx[i] = 0; ((uint64_t *)dev->dcbaa)[slot_id] = (uint64_t)(uintptr_t)ms->device_ctx; /* Input Context: Input Control Context, then Slot Context, then EP0 * Context -- this is what the command TRB's parameter points at (never * the Device Context; conflating the two is the standard mistake * here). */ uint8_t *ictx = (uint8_t *)ms->input_ctx; size_t total = 3 * cs; for (size_t i = 0; i < total; i++) ictx[i] = 0; xhci_input_ctrl_ctx32_t *ctrl = (xhci_input_ctrl_ctx32_t *)ictx; ctrl->add_flags = XHCI_INPUT_CTRL_ADD_SLOT | XHCI_INPUT_CTRL_ADD_EP0; xhci_slot_ctx32_t *slot = (xhci_slot_ctx32_t *)(ictx + cs); slot->dword0 = (speed << XHCI_SLOT_CTX_SPEED_SHIFT) | (1u << XHCI_SLOT_CTX_CONTEXT_ENTRIES_SHIFT); slot->dword1 = port_id << XHCI_SLOT_CTX_ROOT_PORT_SHIFT; slot->dword2 = 0u << XHCI_SLOT_CTX_INTR_TARGET_SHIFT; /* Interrupter 0 */ xhci_ep_ctx32_t *ep0 = (xhci_ep_ctx32_t *)(ictx + 2 * cs); ep0->dword1 = (3u << XHCI_EP_CTX_CERR_SHIFT) | (XHCI_EP_CTX_TYPE_CONTROL_BIDI << XHCI_EP_CTX_TYPE_SHIFT) | (xhci_default_ep0_max_packet(speed) << XHCI_EP_CTX_MAX_PACKET_SHIFT); ep0->tr_dequeue_ptr = ((uint64_t)(uintptr_t)ms->ep0_ring) | 1u; /* DCS = 1 */ ep0->dword4 = 8u; /* Average TRB Length -- spec's own recommended default for EP0 */ xhci_submit_command(dev, (uint64_t)(uintptr_t)ms->input_ctx, 0, XHCI_TRB_TYPE_ADDRESS_DEVICE_CMD, slot_id << 24); log_message(LOG_DEBUG, "xhci: address device command submitted"); return 0; } /* Initialise one bulk Transfer Ring in place -- same fixed-ring-plus- * Link-TRB pattern as ep0_ring/the Command Ring, factored out since * Configure Endpoint needs to do this twice (IN and OUT). */ static void xhci_init_bulk_ring(xhci_trb_t *ring) { for (uint32_t i = 0; i < XHCI_RING_TRB_COUNT; i++) { ring[i].parameter = 0; ring[i].status = 0; ring[i].control = 0; } ring[XHCI_RING_TRB_COUNT - 1].parameter = (uint64_t)(uintptr_t)ring; ring[XHCI_RING_TRB_COUNT - 1].control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_TC | XHCI_TRB_CONTROL_CYCLE; } int xhci_cmd_configure_endpoint(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !dev->cmd_ring || !ms || !ms->input_ctx || !ms->device_ctx) return -1; if (ms->bulk_in_ep_addr == 0 || ms->bulk_out_ep_addr == 0) return -1; const size_t cs = xhci_ctx_bytes(dev); /* 32 or 64, by HCCPARAMS1.CSZ */ if (!ms->bulk_in_ring) { ms->bulk_in_ring = (xhci_trb_t *)kmalloc_aligned(XHCI_RING_BYTES, 64); if (!ms->bulk_in_ring) return -1; } if (!ms->bulk_out_ring) { ms->bulk_out_ring = (xhci_trb_t *)kmalloc_aligned(XHCI_RING_BYTES, 64); if (!ms->bulk_out_ring) return -1; } xhci_init_bulk_ring(ms->bulk_in_ring); ms->bulk_in_ring_cycle = 1; ms->bulk_in_ring_enq = 0; xhci_init_bulk_ring(ms->bulk_out_ring); ms->bulk_out_ring_cycle = 1; ms->bulk_out_ring_enq = 0; uint32_t in_dci = XHCI_EP_ADDR_TO_DCI(ms->bulk_in_ep_addr); uint32_t out_dci = XHCI_EP_ADDR_TO_DCI(ms->bulk_out_ep_addr); uint32_t max_dci = (in_dci > out_dci) ? in_dci : out_dci; /* DCBAA[slot_id]'s Device Context also needs to grow to hold the new * EP Contexts -- it's currently sized for Slot+EP0 only (from Address * Device). Unlike the Input Context below, this one's *existing* * content must be preserved, not zeroed: xHCI 1.2 spec section 4.6.6 * only has the controller write the DCIs actually named in this * command's Add/Drop flags (EP0's entry here is neither), so it * expects to find EP0's live output state (its current TR Dequeue * Pointer in particular) still intact in the Device Context it reads * -- swapping in a freshly zeroed buffer would hand the controller a * blank EP0 out from under an endpoint it isn't being asked to touch. */ size_t old_device_ctx_bytes = 2 * cs; size_t new_device_ctx_bytes = (1 + (size_t)max_dci) * cs; void *new_device_ctx = kmalloc_aligned(new_device_ctx_bytes, 64); if (!new_device_ctx) return -1; uint8_t *ndctx = (uint8_t *)new_device_ctx; for (size_t i = 0; i < new_device_ctx_bytes; i++) ndctx[i] = 0; const uint8_t *odctx = (const uint8_t *)ms->device_ctx; for (size_t i = 0; i < old_device_ctx_bytes; i++) ndctx[i] = odctx[i]; ms->device_ctx = new_device_ctx; ((uint64_t *)dev->dcbaa)[slot_id] = (uint64_t)(uintptr_t)ms->device_ctx; /* ms->input_ctx is reused from Address Device -- same allocation * (three contexts: Input Control, Slot, EP0) is too small * to also hold two more EP Contexts. Configure Endpoint's Input * Context must span every DCI up to max_dci (xHCI 1.2 spec section * 6.2.5.1: "the Input Context data structure shall contain output * context data structures... up to the value of the Context Entries * field"), not just the ones actually being added -- unused slots * between EP0 (DCI 1) and the bulk endpoints are left zeroed * (Add/Drop flags for those DCIs are 0, so the controller ignores * their content). Reallocated here rather than growing the existing * block in place. */ size_t ctx_count = 1 /* Slot */ + max_dci; /* DCI 1..max_dci, one context each */ size_t total = cs + ctx_count * cs; void *new_input_ctx = kmalloc_aligned(total, 64); if (!new_input_ctx) return -1; ms->input_ctx = new_input_ctx; uint8_t *ictx = (uint8_t *)ms->input_ctx; for (size_t i = 0; i < total; i++) ictx[i] = 0; xhci_input_ctrl_ctx32_t *ctrl = (xhci_input_ctrl_ctx32_t *)ictx; ctrl->add_flags = XHCI_INPUT_CTRL_ADD_SLOT | (1u << in_dci) | (1u << out_dci); /* Slot Context: copied from the already-addressed device's own * Device Context (Route String/Speed/Root Hub Port/Interrupter * Target aren't retained anywhere else by this point in enumeration * -- see this function's own doc comment), Context Entries updated * to the highest DCI now in use. dword3 (Device Address, Slot State) * is an Output-only field and stays zero, matching Address Device's * own Input Context handling. */ xhci_slot_ctx32_t *dev_slot = (xhci_slot_ctx32_t *)ms->device_ctx; xhci_slot_ctx32_t *in_slot = (xhci_slot_ctx32_t *)(ictx + cs); uint32_t entries_mask = ~((uint32_t)0x1Fu << XHCI_SLOT_CTX_CONTEXT_ENTRIES_SHIFT); in_slot->dword0 = (dev_slot->dword0 & entries_mask) | (max_dci << XHCI_SLOT_CTX_CONTEXT_ENTRIES_SHIFT); in_slot->dword1 = dev_slot->dword1; in_slot->dword2 = dev_slot->dword2; /* EP Contexts for the two bulk endpoints, at their own DCI slot * within the Input Context (index = DCI - 1, since the Slot Context * occupies index -1 relative to DCI numbering -- DCI 1 is the first * EP Context). Average TRB Length is a scheduling hint, not a * correctness constraint (spec: "should approximate the length of * the transfers that will be enqueued") -- 1024 is a placeholder; * revisit once real CBW/data/CSW transfer sizes are known (2g's next * items). Max Burst Size is left 0 (single-burst) since SuperSpeed * Endpoint Companion descriptor parsing isn't implemented yet -- * matches every real device this driver has been tested against * under QEMU's emulation, revisit if a real high-throughput SS * device needs it. */ xhci_ep_ctx32_t *in_ep = (xhci_ep_ctx32_t *)(ictx + cs + (size_t)in_dci * cs); in_ep->dword1 = (3u << XHCI_EP_CTX_CERR_SHIFT) | (XHCI_EP_CTX_TYPE_BULK_IN << XHCI_EP_CTX_TYPE_SHIFT) | ((uint32_t)ms->bulk_in_max_packet << XHCI_EP_CTX_MAX_PACKET_SHIFT); in_ep->tr_dequeue_ptr = ((uint64_t)(uintptr_t)ms->bulk_in_ring) | 1u; /* DCS = 1 */ in_ep->dword4 = 1024u; xhci_ep_ctx32_t *out_ep = (xhci_ep_ctx32_t *)(ictx + cs + (size_t)out_dci * cs); out_ep->dword1 = (3u << XHCI_EP_CTX_CERR_SHIFT) | (XHCI_EP_CTX_TYPE_BULK_OUT << XHCI_EP_CTX_TYPE_SHIFT) | ((uint32_t)ms->bulk_out_max_packet << XHCI_EP_CTX_MAX_PACKET_SHIFT); out_ep->tr_dequeue_ptr = ((uint64_t)(uintptr_t)ms->bulk_out_ring) | 1u; /* DCS = 1 */ out_ep->dword4 = 1024u; xhci_submit_command(dev, (uint64_t)(uintptr_t)ms->input_ctx, 0, XHCI_TRB_TYPE_CONFIGURE_ENDPOINT_CMD, slot_id << 24); log_message(LOG_DEBUG, "xhci: configure endpoint command submitted"); return 0; } /* A boot keyboard's Configure Endpoint (2026-10-14, boards/BRINGUP.md A3): * its one interrupt IN endpoint, in the same shape as the bulk pair above * -- Device Context grown and EP0 kept, Input Context spanning every DCI * up to the new one, Slot Context copied with Context Entries raised. The * interval is the endpoint's bInterval in xHCI's terms (section 6.2.3.6): * frames (ms) for full and low speed, given as the exponent of a period * of 2^n x 125 us; already an exponent minus one at high speed and up. */ int xhci_cmd_configure_hid_endpoint(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !dev->cmd_ring || !ms || !ms->input_ctx || !ms->device_ctx) return -1; if (ms->hid_ep_addr == 0) return -1; const size_t cs = xhci_ctx_bytes(dev); if (!ms->hid_ring) { ms->hid_ring = (xhci_trb_t *)kmalloc_aligned(XHCI_RING_BYTES, 64); if (!ms->hid_ring) return -1; } if (!ms->hid_buf) { ms->hid_buf = (uint8_t *)kmalloc_aligned(128, 64); /* report [0,64), lights at 64 */ if (!ms->hid_buf) return -1; } xhci_init_bulk_ring(ms->hid_ring); ms->hid_ring_cycle = 1; ms->hid_ring_enq = 0; uint32_t dci = XHCI_EP_ADDR_TO_DCI(ms->hid_ep_addr); size_t old_bytes = 2 * cs; size_t new_bytes = (1 + (size_t)dci) * cs; uint8_t *ndctx = (uint8_t *)kmalloc_aligned(new_bytes, 64); if (!ndctx) return -1; for (size_t i = 0; i < new_bytes; i++) ndctx[i] = 0; for (size_t i = 0; i < old_bytes; i++) ndctx[i] = ((const uint8_t *)ms->device_ctx)[i]; ms->device_ctx = ndctx; ((uint64_t *)dev->dcbaa)[slot_id] = (uint64_t)(uintptr_t)ms->device_ctx; size_t total = cs + (1 + (size_t)dci) * cs; uint8_t *ictx = (uint8_t *)kmalloc_aligned(total, 64); if (!ictx) return -1; for (size_t i = 0; i < total; i++) ictx[i] = 0; ms->input_ctx = ictx; xhci_input_ctrl_ctx32_t *ctrl = (xhci_input_ctrl_ctx32_t *)ictx; ctrl->add_flags = XHCI_INPUT_CTRL_ADD_SLOT | (1u << dci); xhci_slot_ctx32_t *dev_slot = (xhci_slot_ctx32_t *)ms->device_ctx; xhci_slot_ctx32_t *in_slot = (xhci_slot_ctx32_t *)(ictx + cs); uint32_t entries_mask = ~((uint32_t)0x1Fu << XHCI_SLOT_CTX_CONTEXT_ENTRIES_SHIFT); in_slot->dword0 = (dev_slot->dword0 & entries_mask) | (dci << XHCI_SLOT_CTX_CONTEXT_ENTRIES_SHIFT); in_slot->dword1 = dev_slot->dword1; in_slot->dword2 = dev_slot->dword2; uint32_t speed = (dev_slot->dword0 >> XHCI_SLOT_CTX_SPEED_SHIFT) & 0xFu; uint32_t interval; if (speed >= 3) { /* high, super */ interval = ms->hid_interval ? (uint32_t)ms->hid_interval - 1u : 0u; if (interval > 15) interval = 15; } else { /* full, low: frames */ uint32_t v = (ms->hid_interval ? ms->hid_interval : 1u) * 8u; interval = 0; while ((1u << (interval + 1)) <= v) interval++; if (interval < 3) interval = 3; if (interval > 10) interval = 10; } xhci_ep_ctx32_t *ep = (xhci_ep_ctx32_t *)(ictx + cs + (size_t)dci * cs); ep->dword0 = interval << XHCI_EP_CTX_INTERVAL_SHIFT; ep->dword1 = (3u << XHCI_EP_CTX_CERR_SHIFT) | (XHCI_EP_CTX_TYPE_INTERRUPT_IN << XHCI_EP_CTX_TYPE_SHIFT) | ((uint32_t)ms->hid_max_packet << XHCI_EP_CTX_MAX_PACKET_SHIFT); ep->tr_dequeue_ptr = ((uint64_t)(uintptr_t)ms->hid_ring) | 1u; /* DCS = 1 */ ep->dword4 = 8u | ((uint32_t)ms->hid_max_packet << 16); /* avg TRB length, max ESIT payload */ xhci_submit_command(dev, (uint64_t)(uintptr_t)ms->input_ctx, 0, XHCI_TRB_TYPE_CONFIGURE_ENDPOINT_CMD, slot_id << 24); log_message(LOG_DEBUG, "xhci: keyboard configure endpoint command submitted"); return 0; } /* G.1 / §F.14 stall recovery: Reset Endpoint command. ep_id is a full * bEndpointAddress (bit 7 = direction) of the stalled bulk endpoint -- the * xHC is told which endpoint to un-halt via its Endpoint ID (the DCI) in * parameter[31:24], exactly the field layout Reset Endpoint uses (xHCI 1.2 * spec table 6-88): parameter[31:24] = Endpoint ID, control[31:24] = Slot * ID. Mirrors the existing xhci_cmd_disable_slot() shape (submit, ring * doorbell 0, don't wait) -- this touches no doorbell but [0]. */ int xhci_cmd_reset_endpoint(xhci_dev_t *dev, uint32_t slot_id, uint32_t ep_id) { if (!dev || !dev->cmd_ring) return -1; uint32_t dci = XHCI_EP_ADDR_TO_DCI(ep_id); if (dci == 0) return -1; /* EP0 (DCI 1) is the only non-bulk case -- refuse */ xhci_submit_command(dev, (uint64_t)dci << 24, 0, XHCI_TRB_TYPE_RESET_ENDPOINT_CMD, slot_id << 24); log_message(LOG_DEBUG, "xhci: reset endpoint command submitted"); return 0; } /* G.1 / §F.14 stall recovery: Set TR Dequeue Pointer command. new_dequeue * is a pointer into the stalled endpoint's Transfer Ring (the current * producer slot, past any failed TRBs) and dcs is the ring's current Cycle * State -- both written into parameter exactly as the spec's Set TR * Dequeue Pointer (xHCI 1.2 table 6-91) defines: parameter[63:4] = the * dequeue pointer, parameter[0] = DCS, control[31:24] = Slot ID. */ int xhci_cmd_set_tr_dequeue_pointer(xhci_dev_t *dev, uint32_t slot_id, uint32_t ep_id, uint64_t new_dequeue, uint32_t dcs) { if (!dev || !dev->cmd_ring) return -1; /* ep_id names the endpoint whose ring new_dequeue points into. The Set * TR Dequeue Pointer TRB itself carries no Endpoint ID field -- the * ring the controller should update is implied by the dequeue pointer * + the controller already having Reset this endpoint (G.1's sequence * runs Reset Endpoint before this) -- so ep_id is a documentation / * call-site-clarity parameter, kept for symmetry with its sibling. * Either way the controller knows which ring because RESET ENDPOINT * names it first. */ (void)ep_id; uint64_t param = (new_dequeue & ~0xFULL) | (uint64_t)(dcs ? 1u : 0u); xhci_submit_command(dev, param, 0, XHCI_TRB_TYPE_SET_TR_DEQUEUE_POINTER_CMD, slot_id << 24); log_message(LOG_DEBUG, "xhci: set TR dequeue pointer command submitted"); return 0; } /* Enqueue one Normal TRB to the bulk OUT Transfer Ring -- same fixed- * ring-plus-Link-TRB wraparound pattern as xhci_ep0_enqueue_trb(), * operating on bulk_out_ring/bulk_out_ring_enq/bulk_out_ring_cycle * instead of the EP0 ring's fields. A CBW is always exactly one TRB * (no Setup/Data/Status split -- that's a control-transfer-only * concept), so unlike xhci_ep0_enqueue_trb() this rings the doorbell * itself rather than leaving that to a caller assembling a group. */ static void xhci_bulk_out_enqueue_and_ring(xhci_dev_t *dev, uint32_t slot_id, uint64_t parameter, uint32_t status, uint32_t control_bits) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!ms) return; xhci_trb_t *trb = &ms->bulk_out_ring[ms->bulk_out_ring_enq]; trb->parameter = parameter; trb->status = status; trb->control = control_bits | (ms->bulk_out_ring_cycle ? XHCI_TRB_CONTROL_CYCLE : 0); ms->bulk_out_ring_enq++; if (ms->bulk_out_ring_enq == XHCI_RING_TRB_COUNT - 1) { ms->bulk_out_ring[XHCI_RING_TRB_COUNT - 1].control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_TC | (ms->bulk_out_ring_cycle ? XHCI_TRB_CONTROL_CYCLE : 0); ms->bulk_out_ring_enq = 0; ms->bulk_out_ring_cycle ^= 1u; } /* Doorbell target is the bulk OUT endpoint's own DCI, not target 1 * (EP0) -- distinct rings need distinct doorbell targets so the * controller knows which Transfer Ring just gained a new TRB. */ dev->doorbell[slot_id] = XHCI_DB_TARGET(XHCI_EP_ADDR_TO_DCI(ms->bulk_out_ep_addr)); } int xhci_bot_send_read10(xhci_dev_t *dev, uint32_t slot_id, uint32_t lba, uint16_t num_blocks, uint32_t block_size) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->bulk_out_ring || !ms->bulk_in_ring) return -1; if (ms->bulk_out_ep_addr == 0 || ms->bulk_in_ep_addr == 0) return -1; uint32_t data_len = (uint32_t)num_blocks * block_size; if (data_len > sizeof(dev->bot_data_buf)) return -1; dev->bot_cmd_kind = BOT_CMD_READ10; usb_bot_cbw_t *cbw = &dev->bot_cbw; cbw->dCBWSignature = USB_BOT_CBW_SIGNATURE; cbw->dCBWTag = dev->bot_next_tag++; dev->bot_last_tag = cbw->dCBWTag; dev->bot_expected_data_len = data_len; cbw->dCBWDataTransferLength = data_len; cbw->bmCBWFlags = USB_BOT_CBW_FLAG_DATA_IN; /* READ(10): device -> host data stage */ cbw->bCBWLUN = USB_BOT_CBW_LUN_DEFAULT; cbw->bCBWCBLength = SCSI_CDB_LEN_READ10; for (uint32_t i = 0; i < sizeof(cbw->CBWCB); i++) cbw->CBWCB[i] = 0; /* SCSI READ(10) CDB (SBC-3 section 5.13): opcode, then LBA and * Transfer Length as big-endian fields -- SCSI multi-byte fields are * big-endian on the wire regardless of host or USB byte order, unlike * every other multi-byte value in this driver (TRBs, contexts, CBW * itself), which are all little-endian. Written byte-by-byte rather * than via a struct + byte-swap helper, matching this codebase's * existing preference for explicit field layout over struct-based * binary formats wherever the layout isn't naturally what a C struct * would produce (see usb_bot_cbw_t's own doc comment, and the * Interface/Endpoint descriptor offset macros). */ cbw->CBWCB[0] = SCSI_CMD_READ10; cbw->CBWCB[1] = 0; /* flags: no FUA/DPO/RDPROTECT for this increment */ cbw->CBWCB[2] = (uint8_t)(lba >> 24); cbw->CBWCB[3] = (uint8_t)(lba >> 16); cbw->CBWCB[4] = (uint8_t)(lba >> 8); cbw->CBWCB[5] = (uint8_t)(lba); cbw->CBWCB[6] = 0; /* group number */ cbw->CBWCB[7] = (uint8_t)(num_blocks >> 8); cbw->CBWCB[8] = (uint8_t)(num_blocks); cbw->CBWCB[9] = 0; /* control */ ms->transfer_purpose = XHCI_XFER_CBW_SENT; /* IOC set -- the CBW is always exactly one TRB, so it alone signals * "this transfer is done" (matching the EP0 control-read pattern of * IOC on the one TRB whose completion means something). Length is * USB_BOT_CBW_LENGTH (31), not sizeof(*cbw) -- see usb_bot_cbw_t's * own doc comment on why. */ xhci_bulk_out_enqueue_and_ring(dev, slot_id, (uint64_t)(uintptr_t)cbw, USB_BOT_CBW_LENGTH, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC); log_message(LOG_DEBUG, "xhci: CBW (READ10) submitted"); return 0; } int xhci_bot_send_write10(xhci_dev_t *dev, uint32_t slot_id, uint32_t lba, uint16_t num_blocks, uint32_t block_size) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->bulk_out_ring || !ms->bulk_in_ring) return -1; if (ms->bulk_out_ep_addr == 0 || ms->bulk_in_ep_addr == 0) return -1; uint32_t data_len = (uint32_t)num_blocks * block_size; if (data_len > sizeof(dev->bot_data_buf)) return -1; dev->bot_cmd_kind = BOT_CMD_WRITE10; usb_bot_cbw_t *cbw = &dev->bot_cbw; cbw->dCBWSignature = USB_BOT_CBW_SIGNATURE; cbw->dCBWTag = dev->bot_next_tag++; dev->bot_last_tag = cbw->dCBWTag; dev->bot_expected_data_len = data_len; cbw->dCBWDataTransferLength = data_len; cbw->bmCBWFlags = 0; /* WRITE(10): host -> device data stage, unlike * READ10's USB_BOT_CBW_FLAG_DATA_IN above */ cbw->bCBWLUN = USB_BOT_CBW_LUN_DEFAULT; cbw->bCBWCBLength = SCSI_CDB_LEN_WRITE10; for (uint32_t i = 0; i < sizeof(cbw->CBWCB); i++) cbw->CBWCB[i] = 0; /* SCSI WRITE(10) CDB (SBC-3 section 5.32) -- identical layout to * READ(10)'s CDB above, only the opcode differs. Same big-endian * field packing, see xhci_bot_send_read10()'s own comment for why. */ cbw->CBWCB[0] = SCSI_CMD_WRITE10; cbw->CBWCB[1] = 0; /* flags: no FUA/DPO for this increment */ cbw->CBWCB[2] = (uint8_t)(lba >> 24); cbw->CBWCB[3] = (uint8_t)(lba >> 16); cbw->CBWCB[4] = (uint8_t)(lba >> 8); cbw->CBWCB[5] = (uint8_t)(lba); cbw->CBWCB[6] = 0; /* group number */ cbw->CBWCB[7] = (uint8_t)(num_blocks >> 8); cbw->CBWCB[8] = (uint8_t)(num_blocks); cbw->CBWCB[9] = 0; /* control */ ms->transfer_purpose = XHCI_XFER_CBW_SENT; xhci_bulk_out_enqueue_and_ring(dev, slot_id, (uint64_t)(uintptr_t)cbw, USB_BOT_CBW_LENGTH, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC); log_message(LOG_DEBUG, "xhci: CBW (WRITE10) submitted"); return 0; } int xhci_bot_send_test_unit_ready(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->bulk_out_ring || !ms->bulk_in_ring) return -1; if (ms->bulk_out_ep_addr == 0 || ms->bulk_in_ep_addr == 0) return -1; dev->bot_cmd_kind = BOT_CMD_TEST_UNIT_READY; usb_bot_cbw_t *cbw = &dev->bot_cbw; cbw->dCBWSignature = USB_BOT_CBW_SIGNATURE; cbw->dCBWTag = dev->bot_next_tag++; dev->bot_last_tag = cbw->dCBWTag; dev->bot_expected_data_len = 0; /* no data stage -- BOT spec section 6.3 */ cbw->dCBWDataTransferLength = 0; cbw->bmCBWFlags = 0; /* direction is irrelevant when length is 0 */ cbw->bCBWLUN = USB_BOT_CBW_LUN_DEFAULT; cbw->bCBWCBLength = SCSI_CDB_LEN_TEST_UNIT_READY; for (uint32_t i = 0; i < sizeof(cbw->CBWCB); i++) cbw->CBWCB[i] = 0; cbw->CBWCB[0] = SCSI_CMD_TEST_UNIT_READY; /* opcode 0x00, every other CDB byte reserved/zero */ ms->transfer_purpose = XHCI_XFER_CBW_SENT; xhci_bulk_out_enqueue_and_ring(dev, slot_id, (uint64_t)(uintptr_t)cbw, USB_BOT_CBW_LENGTH, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC); log_message(LOG_DEBUG, "xhci: CBW (TEST UNIT READY) submitted"); return 0; } int xhci_bot_send_read_capacity10(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->bulk_out_ring || !ms->bulk_in_ring) return -1; if (ms->bulk_out_ep_addr == 0 || ms->bulk_in_ep_addr == 0) return -1; dev->bot_cmd_kind = BOT_CMD_READ_CAPACITY10; usb_bot_cbw_t *cbw = &dev->bot_cbw; cbw->dCBWSignature = USB_BOT_CBW_SIGNATURE; cbw->dCBWTag = dev->bot_next_tag++; dev->bot_last_tag = cbw->dCBWTag; dev->bot_expected_data_len = SCSI_READ_CAPACITY10_DATA_LEN; cbw->dCBWDataTransferLength = SCSI_READ_CAPACITY10_DATA_LEN; cbw->bmCBWFlags = USB_BOT_CBW_FLAG_DATA_IN; /* READ CAPACITY(10): device -> host data stage */ cbw->bCBWLUN = USB_BOT_CBW_LUN_DEFAULT; cbw->bCBWCBLength = SCSI_CDB_LEN_READ_CAPACITY10; for (uint32_t i = 0; i < sizeof(cbw->CBWCB); i++) cbw->CBWCB[i] = 0; cbw->CBWCB[0] = SCSI_CMD_READ_CAPACITY10; /* opcode 0x25, standard "report * capacity" form -- LBA field * and PMI bit left zero, see * this constant's own doc * comment in xhci.h */ ms->transfer_purpose = XHCI_XFER_CBW_SENT; xhci_bulk_out_enqueue_and_ring(dev, slot_id, (uint64_t)(uintptr_t)cbw, USB_BOT_CBW_LENGTH, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC); log_message(LOG_DEBUG, "xhci: CBW (READ CAPACITY10) submitted"); return 0; } int xhci_bot_read_block(xhci_dev_t *dev, uint32_t slot_id, uint32_t lba, uint16_t num_blocks, uint32_t block_size) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->bulk_out_ring || !ms->bulk_in_ring) return -1; if (ms->bulk_out_ep_addr == 0 || ms->bulk_in_ep_addr == 0) return -1; if ((uint32_t)num_blocks * block_size > sizeof(dev->bot_data_buf)) return -1; dev->bot_read10_lba = lba; dev->bot_read10_num_blocks = num_blocks; dev->bot_read10_block_size = block_size; dev->bot_tur_chain_target = BOT_TUR_CHAIN_READ10; dev->bot_tur_retries = 0; return xhci_bot_send_test_unit_ready(dev, slot_id); } int xhci_bot_write_block(xhci_dev_t *dev, uint32_t slot_id, uint32_t lba, uint16_t num_blocks, uint32_t block_size) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->bulk_out_ring || !ms->bulk_in_ring) return -1; if (ms->bulk_out_ep_addr == 0 || ms->bulk_in_ep_addr == 0) return -1; if ((uint32_t)num_blocks * block_size > sizeof(dev->bot_data_buf)) return -1; dev->bot_write10_lba = lba; dev->bot_write10_num_blocks = num_blocks; dev->bot_write10_block_size = block_size; dev->bot_tur_chain_target = BOT_TUR_CHAIN_WRITE10; dev->bot_tur_retries = 0; return xhci_bot_send_test_unit_ready(dev, slot_id); } int xhci_bot_get_capacity(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->bulk_out_ring || !ms->bulk_in_ring) return -1; if (ms->bulk_out_ep_addr == 0 || ms->bulk_in_ep_addr == 0) return -1; dev->bot_tur_chain_target = BOT_TUR_CHAIN_READ_CAPACITY10; dev->bot_tur_retries = 0; return xhci_bot_send_test_unit_ready(dev, slot_id); } int xhci_bot_wait_for_idle(xhci_dev_t *dev, uint32_t max_iters) { if (!dev) return BOT_STATUS_FAILED; for (uint32_t i = 0; i < max_iters; i++) { if (dev->bot_cmd_kind == BOT_CMD_NONE) return (int)dev->bot_last_status; xhci_poll_events(); /* FABRIC-3.md, 2026-09-09: same TCG timer-starvation reasoning as * xhci_wait_bit()'s own doc comment above -- xhci_poll_events() * itself is a real MMIO-reading call, and max_iters (100000 from * blkio_usb_open_msc()) is large enough that a starved host timer * during a slow completion could stall for a very long time. */ arch_relax(); } return BOT_STATUS_TIMEOUT; } /* Enqueue one Normal TRB to the bulk IN Transfer Ring and ring its * doorbell -- same shape as xhci_bulk_out_enqueue_and_ring() (a BOT * Data-In or CSW read is, like a CBW send, always exactly one TRB), just * targeting bulk_in_ring/bulk_in_ep_addr instead of the OUT side. */ static void xhci_bulk_in_enqueue_and_ring(xhci_dev_t *dev, uint32_t slot_id, uint64_t parameter, uint32_t status, uint32_t control_bits) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!ms) return; xhci_trb_t *trb = &ms->bulk_in_ring[ms->bulk_in_ring_enq]; trb->parameter = parameter; trb->status = status; trb->control = control_bits | (ms->bulk_in_ring_cycle ? XHCI_TRB_CONTROL_CYCLE : 0); ms->bulk_in_ring_enq++; if (ms->bulk_in_ring_enq == XHCI_RING_TRB_COUNT - 1) { ms->bulk_in_ring[XHCI_RING_TRB_COUNT - 1].control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_TC | (ms->bulk_in_ring_cycle ? XHCI_TRB_CONTROL_CYCLE : 0); ms->bulk_in_ring_enq = 0; ms->bulk_in_ring_cycle ^= 1u; } dev->doorbell[slot_id] = XHCI_DB_TARGET(XHCI_EP_ADDR_TO_DCI(ms->bulk_in_ep_addr)); } int xhci_bot_read_data_in(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->bulk_in_ring) return -1; ms->transfer_purpose = XHCI_XFER_BOT_DATA_IN; xhci_bulk_in_enqueue_and_ring(dev, slot_id, (uint64_t)(uintptr_t)dev->bot_data_buf, dev->bot_expected_data_len, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC); log_message(LOG_DEBUG, "xhci: BOT Data-In read submitted"); return 0; } int xhci_bot_write_data_out(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->bulk_out_ring) return -1; ms->transfer_purpose = XHCI_XFER_BOT_DATA_OUT; xhci_bulk_out_enqueue_and_ring(dev, slot_id, (uint64_t)(uintptr_t)dev->bot_data_buf, dev->bot_expected_data_len, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC); log_message(LOG_DEBUG, "xhci: BOT Data-Out write submitted"); return 0; } int xhci_bot_receive_csw(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->bulk_in_ring) return -1; ms->transfer_purpose = XHCI_XFER_CSW_RECEIVED; /* Length is USB_BOT_CSW_LENGTH (13), not sizeof(dev->bot_csw) -- same * padding hazard as the CBW, see usb_bot_csw_t's own doc comment. */ xhci_bulk_in_enqueue_and_ring(dev, slot_id, (uint64_t)(uintptr_t)&dev->bot_csw, USB_BOT_CSW_LENGTH, (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC); log_message(LOG_DEBUG, "xhci: CSW receive submitted"); return 0; } /* Enqueue one TRB to the EP0 Transfer Ring without ringing the doorbell * -- Setup/Data/Status stage TRBs are enqueued as a group, then the * doorbell is rung once after all three are posted, matching how a real * xHCI control transfer is submitted (the controller processes queued * TRBs as a unit once notified, not one doorbell ring per TRB). Same * fixed-ring-plus-Link-TRB wraparound pattern as xhci_submit_command(), * operating on ep0_ring/ep0_ring_enq/ep0_ring_cycle instead of the * Command Ring's fields. */ static void xhci_ep0_enqueue_trb(xhci_dev_t *dev, uint32_t slot_id, uint64_t parameter, uint32_t status, uint32_t control_bits) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!ms) return; xhci_trb_t *trb = &ms->ep0_ring[ms->ep0_ring_enq]; trb->parameter = parameter; trb->status = status; trb->control = control_bits | (ms->ep0_ring_cycle ? XHCI_TRB_CONTROL_CYCLE : 0); ms->ep0_ring_enq++; if (ms->ep0_ring_enq == XHCI_RING_TRB_COUNT - 1) { ms->ep0_ring[XHCI_RING_TRB_COUNT - 1].control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_TC | (ms->ep0_ring_cycle ? XHCI_TRB_CONTROL_CYCLE : 0); ms->ep0_ring_enq = 0; ms->ep0_ring_cycle ^= 1u; } } /* Shared submission for any "device-to-host, standard, device recipient, * IN data stage" control read -- both GET_DESCRIPTOR(Device) and * GET_DESCRIPTOR(Configuration) are this same shape, differing only in * wValue/wLength/destination buffer. Does not set dev->transfer_purpose * or dev->pending_transfer_slot_id -- callers do that themselves so the * purpose is set before the doorbell rings (avoids a window where a * stray Transfer Event could be misread against a not-yet-set purpose, * even though this driver is polled and that window can't actually be * hit by anything external in practice). */ static void xhci_ep0_control_read(xhci_dev_t *dev, uint32_t slot_id, uint8_t bRequest, uint16_t wValue, uint16_t wIndex, uint8_t *buf, uint16_t len) { usb_setup_packet_t setup = { .bmRequestType = USB_DIR_DEVICE_TO_HOST, .bRequest = bRequest, .wValue = wValue, .wIndex = wIndex, .wLength = len }; uint64_t setup_bits; memcpy(&setup_bits, &setup, sizeof(setup_bits)); /* Setup Stage: IDT set (parameter IS the 8-byte packet, not a * pointer), TRT = IN Data Stage since this request reads data back. */ xhci_ep0_enqueue_trb(dev, slot_id, setup_bits, 8u, (XHCI_TRB_TYPE_SETUP_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IDT | (XHCI_SETUP_TRT_IN_DATA << XHCI_TRB_CONTROL_TRT_SHIFT)); /* Data Stage: parameter is a real pointer here (not immediate) -- * points at the caller's buffer. DIR=IN matches the Setup Stage's * TRT. */ xhci_ep0_enqueue_trb(dev, slot_id, (uint64_t)(uintptr_t)buf, len, (XHCI_TRB_TYPE_DATA_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_DIR_IN); /* Status Stage: DIR=OUT (opposite of the Data Stage's IN) -- the * status handshake always runs the reverse direction. IOC set here * only: this is the sole TRB of the three whose completion signals * "the whole control transfer is done" to xhci_poll_events(). */ xhci_ep0_enqueue_trb(dev, slot_id, 0, 0, (XHCI_TRB_TYPE_STATUS_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC); } int xhci_ep0_get_device_descriptor(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->ep0_ring) return -1; ms->transfer_purpose = XHCI_XFER_DEVICE_DESC; xhci_ep0_control_read(dev, slot_id, USB_REQ_GET_DESCRIPTOR, (uint16_t)(USB_DESC_TYPE_DEVICE << 8), 0, ms->device_descriptor, sizeof(ms->device_descriptor)); /* Doorbell Array is indexed by slot ID; target 1 = Default Control * Endpoint (EP0)'s Device Context Index, per xHCI 1.2 spec table * 6-25 -- distinct from doorbell[0], which is always the Command * Ring regardless of slot. */ dev->doorbell[slot_id] = XHCI_DB_TARGET(1); log_message(LOG_DEBUG, "xhci: get device descriptor submitted"); return 0; } int xhci_ep0_get_config_descriptor(xhci_dev_t *dev, uint32_t slot_id, uint16_t length) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->ep0_ring) return -1; /* Cap to the fixed buffer size -- a device whose real Configuration * descriptor set exceeds this would be truncated, not overflowed; * 128 bytes comfortably covers a single-interface Mass Storage * device (Config 9 + Interface 9 + 2 Endpoints * 7 = 32 bytes * typical), so this is a defensive cap, not an expected path. */ if (length > sizeof(ms->config_descriptor)) { length = (uint16_t)sizeof(ms->config_descriptor); } ms->transfer_purpose = (length <= 9) ? XHCI_XFER_CONFIG_DESC_SHORT : XHCI_XFER_CONFIG_DESC_FULL; xhci_ep0_control_read(dev, slot_id, USB_REQ_GET_DESCRIPTOR, (uint16_t)(USB_DESC_TYPE_CONFIG << 8), 0, ms->config_descriptor, length); dev->doorbell[slot_id] = XHCI_DB_TARGET(1); log_message(LOG_DEBUG, "xhci: get config descriptor submitted"); return 0; } /* Shared submission for a "no Data Stage" control transfer -- Setup Stage * only (TRT = XHCI_SETUP_TRT_NO_DATA), then Status Stage. Per USB 2.0 * spec section 8.5.3, a control transfer with no Data Stage always uses * an IN Status Stage (the reverse of a normal OUT request's OUT status), * so DIR_IN is set unconditionally here -- this helper isn't generic * across OUT-data and no-data requests, only the latter. */ static void xhci_ep0_control_write_nodata(xhci_dev_t *dev, uint32_t slot_id, uint8_t bRequest, uint16_t wValue, uint16_t wIndex) { usb_setup_packet_t setup = { .bmRequestType = USB_DIR_HOST_TO_DEVICE, .bRequest = bRequest, .wValue = wValue, .wIndex = wIndex, .wLength = 0 }; uint64_t setup_bits; memcpy(&setup_bits, &setup, sizeof(setup_bits)); xhci_ep0_enqueue_trb(dev, slot_id, setup_bits, 8u, (XHCI_TRB_TYPE_SETUP_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IDT | (XHCI_SETUP_TRT_NO_DATA << XHCI_TRB_CONTROL_TRT_SHIFT)); xhci_ep0_enqueue_trb(dev, slot_id, 0, 0, (XHCI_TRB_TYPE_STATUS_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC | XHCI_TRB_CONTROL_DIR_IN); } int xhci_ep0_set_configuration(xhci_dev_t *dev, uint32_t slot_id, uint8_t config_value) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->ep0_ring) return -1; ms->transfer_purpose = XHCI_XFER_SET_CONFIG; xhci_ep0_control_write_nodata(dev, slot_id, USB_REQ_SET_CONFIGURATION, config_value, 0); dev->doorbell[slot_id] = XHCI_DB_TARGET(1); log_message(LOG_DEBUG, "xhci: set configuration submitted"); return 0; } /* G.1 / §F.14 stall recovery: CLEAR_FEATURE(ENDPOINT_HALT) on a specific * endpoint. Reuses the existing no-data control-transfer machinery * (xhci_ep0_control_write_nodata(), the same Setup+Status shape as * SET_CONFIGURATION) -- just a different request payload: bRequest = * USB_REQ_CLEAR_FEATURE, wValue = USB_FEATURE_ENDPOINT_HALT, wIndex = the * endpoint's own bEndpointAddress (standard, host-to-device, endpoint- * recipient -- bmRequestType inferred by the helper as 0x00). This is the * USB-level clear that clears the device's halt condition and resets its * data toggle after the two xHCI command steps (Reset Endpoint, Set TR * Dequeue Pointer) have un-halted the xHC side. */ int xhci_ep0_clear_endpoint_halt(xhci_dev_t *dev, uint32_t slot_id, uint8_t ep_addr) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->ep0_ring) return -1; ms->transfer_purpose = XHCI_XFER_CLEAR_HALT; xhci_ep0_control_write_nodata(dev, slot_id, USB_REQ_CLEAR_FEATURE, USB_FEATURE_ENDPOINT_HALT, ep_addr); dev->doorbell[slot_id] = XHCI_DB_TARGET(1); log_message(LOG_DEBUG, "xhci: clear endpoint halt submitted"); return 0; } /* G.1 / §F.14 stall recovery escalation: Bulk-Only Transport Mass Storage * Reset. bmRequestType = 0x21 (class, interface recipient), bRequest = * 0xFF, no data stage -- also built on xhci_ep0_control_write_nodata(), but * that helper hardcodes bmRequestType = 0x00 (standard, host-to-device, * device recipient). A BOT Mass Storage Reset is class + interface * recipient, so it can't reuse the helper directly; the request bytes are * stamped into the Setup packet by hand here (matching the same explicit- * layout, no-padding discipline usb_setup_packet_t is documented for). */ int xhci_ep0_bot_mass_storage_reset(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->ep0_ring) return -1; usb_setup_packet_t setup = { .bmRequestType = (USB_REQ_TYPE_CLASS << 5) | USB_RECIP_INTERFACE, .bRequest = USB_BOT_MASS_STORAGE_RESET, .wValue = 0, .wIndex = 0, .wLength = 0 }; uint64_t setup_bits; memcpy(&setup_bits, &setup, sizeof(setup_bits)); xhci_ep0_enqueue_trb(dev, slot_id, setup_bits, 8u, (XHCI_TRB_TYPE_SETUP_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IDT | (XHCI_SETUP_TRT_NO_DATA << XHCI_TRB_CONTROL_TRT_SHIFT)); xhci_ep0_enqueue_trb(dev, slot_id, 0, 0, (XHCI_TRB_TYPE_STATUS_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC | XHCI_TRB_CONTROL_DIR_IN); ms->transfer_purpose = XHCI_XFER_BOT_RESET; dev->doorbell[slot_id] = XHCI_DB_TARGET(1); log_message(LOG_DEBUG, "xhci: BOT mass storage reset submitted"); return 0; } /* A boot keyboard's class requests (HID 1.11 section 7.2), on EP0, to its * interface: SET_PROTOCOL(boot) -- a keyboard starts in report protocol, * whose reports need not be the 8-byte boot ones -- and SET_REPORT for the * lock lights (one output byte). (2026-10-14, boards/BRINGUP.md A3.) */ int xhci_ep0_hid_set_protocol(xhci_dev_t *dev, uint32_t slot_id) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->ep0_ring) return -1; usb_setup_packet_t setup = { .bmRequestType = (USB_REQ_TYPE_CLASS << 5) | USB_RECIP_INTERFACE, .bRequest = USB_HID_REQ_SET_PROTOCOL, .wValue = USB_HID_PROTOCOL_BOOT, .wIndex = ms->hid_iface, .wLength = 0 }; uint64_t setup_bits; memcpy(&setup_bits, &setup, sizeof(setup_bits)); xhci_ep0_enqueue_trb(dev, slot_id, setup_bits, 8u, (XHCI_TRB_TYPE_SETUP_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IDT | (XHCI_SETUP_TRT_NO_DATA << XHCI_TRB_CONTROL_TRT_SHIFT)); xhci_ep0_enqueue_trb(dev, slot_id, 0, 0, (XHCI_TRB_TYPE_STATUS_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC | XHCI_TRB_CONTROL_DIR_IN); ms->transfer_purpose = XHCI_XFER_HID_SET_PROTOCOL; dev->doorbell[slot_id] = XHCI_DB_TARGET(1); log_message(LOG_DEBUG, "xhci: keyboard set protocol (boot) submitted"); return 0; } #define USB_HID_REQ_SET_REPORT 0x09u #define USB_HID_REPORT_OUTPUT 0x0200u /* wValue: report type Output (2), report ID 0 */ static int xhci_ep0_hid_set_leds(xhci_dev_t *dev, uint32_t slot_id, uint8_t leds) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!dev || !ms || !ms->ep0_ring || !ms->hid_buf) return -1; ms->hid_buf[64] = leds; usb_setup_packet_t setup = { .bmRequestType = (USB_REQ_TYPE_CLASS << 5) | USB_RECIP_INTERFACE, .bRequest = USB_HID_REQ_SET_REPORT, .wValue = USB_HID_REPORT_OUTPUT, .wIndex = ms->hid_iface, .wLength = 1 }; uint64_t setup_bits; memcpy(&setup_bits, &setup, sizeof(setup_bits)); xhci_ep0_enqueue_trb(dev, slot_id, setup_bits, 8u, (XHCI_TRB_TYPE_SETUP_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IDT | (XHCI_SETUP_TRT_OUT_DATA << XHCI_TRB_CONTROL_TRT_SHIFT)); xhci_ep0_enqueue_trb(dev, slot_id, (uint64_t)(uintptr_t)&ms->hid_buf[64], 1u, (XHCI_TRB_TYPE_DATA_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT)); xhci_ep0_enqueue_trb(dev, slot_id, 0, 0, (XHCI_TRB_TYPE_STATUS_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC | XHCI_TRB_CONTROL_DIR_IN); ms->transfer_purpose = XHCI_XFER_HID_SET_LEDS; ms->hid_leds_sent = leds; /* once asked: a keyboard that refuses is not asked again and again */ dev->doorbell[slot_id] = XHCI_DB_TARGET(1); return 0; } /* One Normal TRB on the keyboard's interrupt ring: the next report lands in * hid_buf. One at a time -- queued again only once the last has been read. */ static void xhci_hid_queue_report(xhci_dev_t *dev, uint32_t slot_id, xhci_msc_slot_t *ms) { uint32_t len = ms->hid_max_packet ? ms->hid_max_packet : 8u; if (len > 64u) len = 64u; xhci_trb_t *trb = &ms->hid_ring[ms->hid_ring_enq]; trb->parameter = (uint64_t)(uintptr_t)ms->hid_buf; trb->status = len; trb->control = (XHCI_TRB_TYPE_NORMAL << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_IOC | (ms->hid_ring_cycle ? XHCI_TRB_CONTROL_CYCLE : 0); ms->hid_ring_enq++; if (ms->hid_ring_enq == XHCI_RING_TRB_COUNT - 1) { ms->hid_ring[XHCI_RING_TRB_COUNT - 1].control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_CONTROL_TYPE_SHIFT) | XHCI_TRB_CONTROL_TC | (ms->hid_ring_cycle ? XHCI_TRB_CONTROL_CYCLE : 0); ms->hid_ring_enq = 0; ms->hid_ring_cycle ^= 1u; } dev->doorbell[slot_id] = XHCI_DB_TARGET(XHCI_EP_ADDR_TO_DCI(ms->hid_ep_addr)); } /* ------------------------------------------------------------------------- * Milestone 2d: Event Ring servicing, polled from sk_repl_idle(). * * Only one controller is supported (matches xhci_dev_t's own doc comment), * so xhci_poll_events() is a self-contained singleton call, no argument * needed -- it recovers the device pointer latched by xhci_bringup() above * rather than taking one. See xhci_driver.h's own doc comment for why this * is polled rather than interrupt-driven (a real, checked-live finding, * not a shortcut: the amd64 PCI INTx routing formula tried first turned * out to be simply wrong). * ------------------------------------------------------------------------- */ /* Port Register Set array lives at Operational base + 0x400 (xhci.h's own * doc comment on xhci_port_regs_t) -- not reachable through xhci_op_regs_t * itself since it isn't a contiguous struct member. port_id is 1-based, * matching XHCI_PSC_EVT_PORT_ID()'s decode and the spec's own numbering. */ static xhci_port_regs_t *xhci_port_regs(xhci_dev_t *dev, uint32_t port_id) { if (port_id < 1 || port_id > dev->max_ports) return NULL; return (xhci_port_regs_t *)((uint8_t *)dev->op + XHCI_PORT_REGS_OFFSET + (port_id - 1) * sizeof(xhci_port_regs_t)); } /* Shared "device connected" handling -- factored out of xhci_poll_events()'s * PORT_STATUS_CHANGE_EVT case so xhci_scan_ports_for_already_connected() * (called once from xhci_bringup(), see its own doc comment) can drive the * exact same Enable Slot sequence for a device that was already attached at * controller bring-up, not just one detected via a later hotplug event. */ /* FABRIC-3.md §VII item 3 (2026-09-05): queue a connect/disconnect event * that couldn't be serviced immediately (Command Ring busy with another * command) instead of dropping it -- a slot in dev->pending_events[], * keyed by port_id-1 like port_slot_id[] itself, so a second PORTSC change * on the same port before the first is drained simply overwrites its own * queue entry (a port only ever has one current state). Drained one entry * at a time by xhci_drain_pending_event() every time connect_state returns * to XHCI_CONN_IDLE. */ static void xhci_queue_pending_event(xhci_dev_t *dev, uint32_t port_id, int is_connect, uint32_t portsc, uint32_t slot_id) { if (port_id < 1 || port_id > XHCI_MAX_TRACKED_PORTS) return; dev->pending_events[port_id - 1].valid = 1; dev->pending_events[port_id - 1].is_connect = (uint8_t)(is_connect ? 1 : 0); dev->pending_events[port_id - 1].port_id = port_id; dev->pending_events[port_id - 1].portsc = portsc; dev->pending_events[port_id - 1].slot_id = slot_id; log_message(LOG_DEBUG, "xhci: command ring busy -- event queued"); } static void xhci_handle_port_connected(xhci_dev_t *dev, uint32_t port_id, uint32_t portsc) { log_message(LOG_DEBUG, "xhci: port status change -- device connected"); /* Milestone 2e prep: Address Device requires the port in Default * state (PED set). USB3 links train and enable themselves; USB2 * needs software to drive PORTSC.PR and wait for PRC/PED before the * device will respond to addressing. */ xhci_log_hex32("xhci: portsc=", portsc); if (port_id > XHCI_MAX_TRACKED_PORTS) { log_message(LOG_WARN, "xhci: port beyond tracked range -- enable slot skipped"); return; } /* FABRIC-3.md, 2026-09-09: a reset this driver itself kicked off * (below) has landed as a fresh port-status-change event -- CCS is * still set (device never left), so this is the *same* connect * continuing, not a new one. Proceed to Enable Slot now that PED * reads set; if it still doesn't, the reset is still in progress * (real hardware/QEMU can take more than one event to settle it) -- * just wait for the next port-status-change event, same as every * other outstanding-command wait in this driver (no explicit * timeout, matching this file's existing convention). */ if (dev->connect_state == XHCI_CONN_AWAIT_PORT_RESET && dev->pending_connect_port_id == port_id) { if (portsc & XHCI_PORTSC_PED) { log_message(LOG_DEBUG, "xhci: port reset complete (PED set)"); dev->pending_connect_speed = XHCI_PORTSC_SPEED(portsc); dev->connect_state = XHCI_CONN_AWAIT_ENABLE_SLOT; xhci_cmd_enable_slot(dev); } else { log_message(LOG_DEBUG, "xhci: port reset still in progress (PED clear)"); } return; } /* Only one Enable Slot in flight at a time (real xHCI Command Ring * semantics, one outstanding command) -- if another connect's slot * request is still outstanding, this one is queued (FABRIC-3.md §VII * item 3, 2026-09-05 -- used to be dropped here) rather than lost. */ if (dev->connect_state != XHCI_CONN_IDLE) { xhci_queue_pending_event(dev, port_id, 1, portsc, 0); return; } if (portsc & XHCI_PORTSC_PED) { /* Common case: the port already reads enabled -- either this is * the very first connect on this port this boot (QEMU/most real * USB3 root hubs present it already-enabled), or a USB3 link * that trained/enabled itself. Go straight to Enable Slot, same * as this driver has always done for that case. */ log_message(LOG_DEBUG, "xhci: port enabled (PED set)"); dev->pending_connect_port_id = port_id; dev->pending_connect_speed = XHCI_PORTSC_SPEED(portsc); dev->connect_state = XHCI_CONN_AWAIT_ENABLE_SLOT; xhci_cmd_enable_slot(dev); } else { /* FABRIC-3.md, 2026-09-09: closes the "reused-port hotplug * wedge" this function's own comment used to flag as an open * question ("not yet known which this driver's ports need"). * Confirmed live: a second device attached to a port whose * previous occupant just disconnected reads PED clear here, and * without a reset cycle Address Device reliably fails * ("xhci: address device failed") -- the port needs an explicit * Port Reset before it will respond to addressing again, exactly * as USB2 devices always have. Same preserve-PP-only write * discipline xhci_poll_events()'s own CSC ack already uses; * PORTSC.PR is RW1S and self-clears in hardware once the reset * completes (PRC set, observed above via the PED check on the * event this produces), so no matching "clear" write is needed. */ log_message(LOG_DEBUG, "xhci: port not yet enabled (PED clear) -- driving port reset"); dev->pending_connect_port_id = port_id; dev->connect_state = XHCI_CONN_AWAIT_PORT_RESET; xhci_port_regs_t *port = xhci_port_regs(dev, port_id); if (port) { port->portsc = (portsc & XHCI_PORTSC_PP) | XHCI_PORTSC_PR; } else { dev->connect_state = XHCI_CONN_IDLE; dev->pending_connect_port_id = 0; } } } /* FABRIC-3.md §VII item 3: called every time connect_state returns to * XHCI_CONN_IDLE (the Command Ring is free again) to service one queued * connect/disconnect that arrived while it was busy. Services at most one * per call -- if servicing it leaves connect_state busy again (a real * command was just issued), the remaining queued entries wait for the * next XHCI_CONN_IDLE transition, same one-at-a-time discipline this * driver already applies to the Command Ring itself. */ static void xhci_drain_pending_event(xhci_dev_t *dev) { for (uint32_t i = 0; i < XHCI_MAX_TRACKED_PORTS; i++) { if (!dev->pending_events[i].valid) continue; dev->pending_events[i].valid = 0; uint32_t port_id = dev->pending_events[i].port_id; if (dev->pending_events[i].is_connect) { xhci_handle_port_connected(dev, port_id, dev->pending_events[i].portsc); } else { uint32_t slot_id = dev->pending_events[i].slot_id; log_message(LOG_DEBUG, "xhci: draining queued disconnect"); dev->pending_disable_slot_id = slot_id; dev->connect_state = XHCI_CONN_AWAIT_DISABLE_SLOT; xhci_cmd_disable_slot(dev, slot_id); } return; } } /* xhci_scan_ports_for_already_connected -- called once from xhci_bringup(), * right after the controller starts running. xhci_poll_events() only reacts * to Port Status Change *events*, and a device present on the QEMU command * line before this controller reset never generates one (nothing "changed" * from the controller's perspective once it starts looking) -- confirmed by * reading the spec's event model, not assumed. Without this scan, such a * device stays invisible to the guest forever, since no later event will * ever announce it either. * * Scans every tracked port for CCS (Current Connect Status) and drives each * one found through the same Enable Slot path xhci_poll_events() uses. * FABRIC-3.md §VII item 4 (2026-09-05): this used to `break` after the * first hit, "this driver's real use case is exactly one thumbdrive already * attached at boot, not several" -- that premise is exactly what this fix * corrects; a second device already attached at boot (e.g. Zuse's drive * plus a target drive both present at QEMU launch) was previously invisible * to this driver, not merely mishandled. The first hit still starts Enable * Slot immediately (connect_state was XHCI_CONN_IDLE); every subsequent hit * found in the same scan queues via xhci_handle_port_connected()'s own * XHCI_CONN_IDLE check (item 3's queue), draining automatically as each * connect completes. */ static void xhci_scan_ports_for_already_connected(xhci_dev_t *dev) { uint32_t max_port = dev->max_ports; if (max_port > XHCI_MAX_TRACKED_PORTS) max_port = XHCI_MAX_TRACKED_PORTS; for (uint32_t port_id = 1; port_id <= max_port; port_id++) { xhci_port_regs_t *port = xhci_port_regs(dev, port_id); if (!port) continue; uint32_t portsc = port->portsc; if (!(portsc & XHCI_PORTSC_CCS)) continue; xhci_handle_port_connected(dev, port_id, portsc); /* Acknowledge CSC and PRC the same way xhci_poll_events() does, in * case the controller latched either during reset -- harmless if * neither was actually set. */ port->portsc = (portsc & XHCI_PORTSC_PP) | XHCI_PORTSC_CSC | XHCI_PORTSC_PRC; } } /* G.1 / §F.14: clean terminal failure of the BOT command currently in * flight, used by every stall-recovery bail-out path. Explicitly sets * bot_last_status = BOT_STATUS_FAILED and bot_cmd_kind = BOT_CMD_NONE -- a * clean signal to a synchronous caller blocked in * xhci_bot_wait_for_idle() (which would otherwise just time out, leaving * state ambiguous), rather than relying on the outer timeout the way the * pre-G.1 un-recovered stall did. */ static void xhci_stall_fail(xhci_dev_t *dev) { log_message(LOG_ERROR, "xhci: stall recovery exhausted -- failing command cleanly"); dev->bot_last_status = BOT_STATUS_FAILED; dev->bot_cmd_kind = BOT_CMD_NONE; dev->stall_retry_action = XHCI_NEXT_ACTION_NONE; dev->bot_reset_clear_remaining = 0; } /* G.1 / §F.14: is `purpose` a bulk (BOT) transfer rather than an EP0 * control transfer? Stall recovery is defined only for the bulk endpoints * (CBW_SENT and BOT_DATA_OUT run on bulk_out; BOT_DATA_IN and CSW_RECEIVED * on bulk_in) -- an EP0 control transfer that stalls during enumeration is * a different, out-of-scope case this driver deliberately does not attempt * to recover. */ static int xhci_bulk_purpose_stalled(xhci_dev_t *dev, uint32_t purpose) { (void)dev; switch (purpose) { case XHCI_XFER_CBW_SENT: case XHCI_XFER_BOT_DATA_IN: case XHCI_XFER_BOT_DATA_OUT: case XHCI_XFER_CSW_RECEIVED: return 1; default: return 0; } } /* G.1 / §F.14: start stall recovery for a bulk transfer that completed * with XHCI_COMPLETION_CODE_STALL_ERROR in xhci_poll_events()'s transfer- * event handler. Identifies the stalled endpoint from `purpose` (which * bulk ring it ran on) and the command-stage to retry once recovery * completes, then either (a) kicks off the basic xHCI Reset Endpoint / * Set TR Dequeue Pointer / CLEAR_FEATURE(ENDPOINT_HALT) recovery, or on a * repeated stall (b) escalates to a full BOT Mass Storage Reset. Recovery * is bounded by bot_stall_recoveries vs XHCI_BOT_STALL_MAX_RECOVERIES; * exhausting it bails out clean per the helper above. Called with purpose * already read (it is cleared by the caller immediately after this call). */ static void xhci_handle_bulk_stall(xhci_dev_t *dev, uint32_t slot_id, uint32_t purpose) { xhci_msc_slot_t *ms = xhci_msc_slot_for(dev, slot_id); if (!ms) return; switch (purpose) { case XHCI_XFER_CBW_SENT: dev->stall_ep_addr = ms->bulk_out_ep_addr; /* Re-send the whole SCSI command from its CBW -- which takes * the existing TUR/READ/WRITE/CAPACITY entry points, picked * by what's in flight. */ switch (dev->bot_cmd_kind) { case BOT_CMD_WRITE10: dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_SEND_WRITE10; break; case BOT_CMD_READ_CAPACITY10: dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_SEND_READ_CAPACITY10; break; case BOT_CMD_TEST_UNIT_READY: dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_SEND_TUR; break; default: /* BOT_CMD_READ10 (and anything unexpected) */ dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_SEND_READ10; break; } break; case XHCI_XFER_BOT_DATA_OUT: dev->stall_ep_addr = ms->bulk_out_ep_addr; dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_DATA_OUT; break; case XHCI_XFER_BOT_DATA_IN: dev->stall_ep_addr = ms->bulk_in_ep_addr; dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_DATA_IN; break; case XHCI_XFER_CSW_RECEIVED: dev->stall_ep_addr = ms->bulk_in_ep_addr; dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_CSW_RECEIVE; break; default: /* Unreachable -- xhci_bulk_purpose_stalled() gates entry. */ return; } dev->stall_dci = XHCI_EP_ADDR_TO_DCI(dev->stall_ep_addr); if (dev->bot_stall_recoveries >= XHCI_BOT_STALL_MAX_RECOVERIES) { xhci_stall_fail(dev); return; } dev->bot_stall_recoveries++; log_message(LOG_WARN, "xhci: bulk endpoint STALL -- starting recovery"); if (dev->bot_stall_recoveries == 1) { /* First recovery: the basic xHCI-level Reset Endpoint sequence * (commands are submitted synchronously -- they ring doorbell 0, * which is safe from inside event processing; only transfer-ring * doorbells are deferred). */ dev->connect_state = XHCI_CONN_AWAIT_RESET_ENDPOINT; if (xhci_cmd_reset_endpoint(dev, slot_id, dev->stall_ep_addr) != 0) { log_message(LOG_ERROR, "xhci: reset endpoint submit failed -- stalling out clean"); dev->connect_state = XHCI_CONN_IDLE; xhci_stall_fail(dev); } } else { /* Repeated stall: escalate to a full BOT Mass Storage Reset + clear * halt on both endpoints -- deferred, since BOT Mass Storage Reset * is a control transfer that rings doorbell[slot] (must not happen * inside event processing). */ ms->next_action = XHCI_NEXT_ACTION_BOT_RESET; } } /* How many boot keyboards are running (A3). While any is, the console * services this event ring on every timer tick as it waits for a key * (repl.c), not only on its once-a-second idle beat: one report at a time * is queued, so at one a second fast typing overflowed the keyboard's own * short queue and keys were lost (2026-10-14). */ static uint32_t g_xhci_keyboards; int xhci_keyboard_running(void) { return g_xhci_keyboards != 0; } static void xhci_poll_one(xhci_dev_t *dev); void xhci_poll_events(void) { uint32_t i; for (i = 0; i < g_xhci_dev_count; i++) xhci_poll_one(g_xhci_devs[i]); } /* One controller's Event Ring -- everything below, per controller. */ static void xhci_poll_one(xhci_dev_t *dev) { if (!dev) return; uint32_t evt_processed = 0; while (evt_processed < XHCI_EVT_RING_MAX_DRAIN && ((dev->evt_ring[dev->evt_ring_deq].control & XHCI_TRB_CONTROL_CYCLE) != 0) == (dev->evt_ring_cycle != 0)) { xhci_trb_t *trb = &dev->evt_ring[dev->evt_ring_deq]; uint32_t type = XHCI_TRB_TYPE(trb->control); switch (type) { case XHCI_TRB_TYPE_PORT_STATUS_CHANGE_EVT: { /* Milestone 2e: identify which port changed and whether it * now reads connected or disconnected. Slot allocation/ * addressing on connect is the next increment -- this only * detects and acknowledges the change for now. */ uint32_t port_id = XHCI_PSC_EVT_PORT_ID(trb->parameter); xhci_port_regs_t *port = xhci_port_regs(dev, port_id); if (!port) { log_message(LOG_WARN, "xhci: port status change event (bad port id)"); break; } uint32_t portsc = port->portsc; if (portsc & XHCI_PORTSC_CCS) { xhci_handle_port_connected(dev, port_id, portsc); } else { log_message(LOG_DEBUG, "xhci: port status change -- device disconnected"); if (port_id >= 1 && port_id <= XHCI_MAX_TRACKED_PORTS && dev->port_slot_id[port_id - 1] != 0) { uint32_t disconnecting_slot_id = dev->port_slot_id[port_id - 1]; /* Stop tracking the slot immediately so a future * connect on this port isn't confused for one * already in progress -- independent of whether * the Disable Slot command below can be issued * right now. */ dev->port_slot_id[port_id - 1] = 0; /* Milestone 2h hot-detach: fire on the disconnect * itself, independent of whether Disable Slot can * be sent right now -- see bot_msc_detach_pending's * own doc comment for why this point, not Disable * Slot completion. Per-slot now (FABRIC-3.md §VII, * 2026-09-05) -- this disconnect only affects its * own slot's attach state, not a single shared * flag. */ { xhci_msc_slot_t *dms = xhci_msc_slot_for(dev, disconnecting_slot_id); if (dms && dms->bot_msc_attached) { dms->bot_msc_attached = 0; dms->bot_msc_detach_pending = 1; } } if (dev->connect_state == XHCI_CONN_IDLE) { dev->pending_disable_slot_id = disconnecting_slot_id; dev->connect_state = XHCI_CONN_AWAIT_DISABLE_SLOT; xhci_cmd_disable_slot(dev, disconnecting_slot_id); } else { /* FABRIC-3.md §VII item 3 (2026-09-05): queued * instead of dropped -- the slot's DCBAA entry * used to be left stale here ("harmless, never * looked at again"), which stopped being true * once a second device can be genuinely * in-flight; the Disable Slot command still * needs to run once the ring is free. */ xhci_queue_pending_event(dev, port_id, 0, 0, disconnecting_slot_id); } } } /* Acknowledge CSC and PRC (both RW1C): preserve PP, write 0 * for PED/PR (writing 1 there disables the port / starts a * new reset -- not intended here) and for every other _C * bit (writing 0 leaves them untouched, not cleared) -- the * same discipline this driver already applies to ERDP.EHB. * PRC added FABRIC-3.md 2026-09-09 alongside * xhci_handle_port_connected()'s new port-reset drive -- * unconditional like CSC always was (RW1C: writing 1 to an * already-clear bit is a no-op, harmless when this event * wasn't reset-related). */ port->portsc = (portsc & XHCI_PORTSC_PP) | XHCI_PORTSC_CSC | XHCI_PORTSC_PRC; break; } case XHCI_TRB_TYPE_COMMAND_COMPLETION_EVT: { uint32_t code = XHCI_EVT_COMPLETION_CODE(trb->status); uint32_t slot_id = XHCI_EVT_SLOT_ID(trb->control); /* Correlates to connect_state, not to the Command TRB * Pointer in trb->parameter -- Enable Slot and Address * Device are issued sequentially for a given connect (see * xhci_dev_t's doc comment), never concurrently, so * connect_state alone identifies which command this * completion answers. A real Command TRB Pointer match * becomes necessary once commands for different connects * can overlap in flight. */ if (dev->connect_state == XHCI_CONN_AWAIT_ENABLE_SLOT) { uint32_t port_id = dev->pending_connect_port_id; if (code == XHCI_COMPLETION_CODE_SUCCESS && port_id >= 1 && port_id <= XHCI_MAX_TRACKED_PORTS) { dev->port_slot_id[port_id - 1] = slot_id; dev->pending_connect_slot_id = slot_id; log_message(LOG_DEBUG, "xhci: enable slot succeeded"); dev->connect_state = XHCI_CONN_AWAIT_ADDRESS_DEVICE; if (xhci_cmd_address_device(dev, slot_id, port_id, dev->pending_connect_speed) != 0) { log_message(LOG_ERROR, "xhci: address device setup failed"); dev->connect_state = XHCI_CONN_IDLE; dev->pending_connect_port_id = 0; } } else { log_message(LOG_ERROR, "xhci: enable slot failed"); dev->connect_state = XHCI_CONN_IDLE; dev->pending_connect_port_id = 0; } } else if (dev->connect_state == XHCI_CONN_AWAIT_ADDRESS_DEVICE) { if (code == XHCI_COMPLETION_CODE_SUCCESS) { log_message(LOG_DEBUG, "xhci: address device succeeded"); /* Milestone 2f: enumeration starts here -- the * device now has a USB address and EP0 is * usable for control transfers. Deferred (see * xhci_next_action_t's own doc comment) rather * than called directly here. */ xhci_msc_slot_t *addr_ms = xhci_msc_slot_for(dev, dev->pending_connect_slot_id); if (addr_ms) addr_ms->next_action = XHCI_NEXT_ACTION_GET_DEVICE_DESC; } else { log_message(LOG_ERROR, "xhci: address device failed"); } dev->connect_state = XHCI_CONN_IDLE; dev->pending_connect_port_id = 0; } else if (dev->connect_state == XHCI_CONN_AWAIT_DISABLE_SLOT) { if (code == XHCI_COMPLETION_CODE_SUCCESS) { /* DCBAA[slot_id] cleared on success only -- if the * controller reports failure, leave it: the slot * may still be in a state where zeroing its * context pointer out from under the controller * is unsafe, and port_slot_id[] no longer * references this slot either way, so nothing * else in this driver will look at it again. */ ((uint64_t *)dev->dcbaa)[dev->pending_disable_slot_id] = 0; /* FABRIC-3.md §VII (2026-09-05): mark the slot free * for reuse by a future, possibly different, device * -- the allocated buffers/rings themselves stay * (lazily reused, same precedent as the old * single-device fields), only in_use clears. */ { xhci_msc_slot_t *dms2 = xhci_msc_slot_for(dev, dev->pending_disable_slot_id); if (dms2) { if (dms2->hid_kbd) usb_kbd_forget(dms2->hid_prev); /* A3: no key stays held */ if (dms2->hid_running && g_xhci_keyboards) g_xhci_keyboards--; dms2->hid_kbd = 0; dms2->hid_running = 0; dms2->in_use = 0; } } log_message(LOG_DEBUG, "xhci: disable slot succeeded"); } else { log_message(LOG_ERROR, "xhci: disable slot failed"); } dev->connect_state = XHCI_CONN_IDLE; dev->pending_disable_slot_id = 0; } else if (dev->connect_state == XHCI_CONN_AWAIT_CONFIGURE_ENDPOINT) { if (code == XHCI_COMPLETION_CODE_SUCCESS) { log_message(LOG_DEBUG, "xhci: configure endpoint succeeded"); /* Chain into SET_CONFIGURATION -- this slot's own * next_action_config_value is still the value * staged when this Configure Endpoint was itself * deferred (see the XFER_CONFIG_DESC_FULL handler * above); only next_action's enum tag gets cleared * on consumption, not the payload fields, so it's * still valid to reuse here. slot_id (this * completion's own Slot ID) is the same slot that * was just configured. */ xhci_msc_slot_t *cms = xhci_msc_slot_for(dev, slot_id); if (cms) cms->next_action = XHCI_NEXT_ACTION_SET_CONFIG; } else { log_message(LOG_ERROR, "xhci: configure endpoint failed"); } dev->connect_state = XHCI_CONN_IDLE; } else if (dev->connect_state == XHCI_CONN_AWAIT_RESET_ENDPOINT) { /* G.1 / §F.14 stall recovery, command step 1 of 2: the * Reset Endpoint command has completed. On success, * the xHC endpoint is back in the Stopped state and we * immediately issue step 2, Set TR Dequeue Pointer, to * reposition the stalled Transfer Ring's dequeue past * the failed TRB (at the ring's current producer slot, * with the producer's cycle as DCS) so a freshly * enqueued retry TRB is consumed cleanly. stall_dci * tells which ring. */ if (code == XHCI_COMPLETION_CODE_SUCCESS) { log_message(LOG_DEBUG, "xhci: reset endpoint succeeded"); uintptr_t new_dequeue = 0; uint32_t dcs = 0; xhci_msc_slot_t *sms = xhci_msc_slot_for(dev, slot_id); if (sms) { if (dev->stall_ep_addr & USB_EP_ADDR_DIR_MASK) { new_dequeue = (uintptr_t)&sms->bulk_in_ring[sms->bulk_in_ring_enq]; dcs = sms->bulk_in_ring_cycle; } else { new_dequeue = (uintptr_t)&sms->bulk_out_ring[sms->bulk_out_ring_enq]; dcs = sms->bulk_out_ring_cycle; } } dev->connect_state = XHCI_CONN_AWAIT_SET_TR_DEQUEUE; if (xhci_cmd_set_tr_dequeue_pointer(dev, slot_id, dev->stall_ep_addr, new_dequeue, dcs) != 0) { log_message(LOG_ERROR, "xhci: set tr dequeue pointer submit failed"); dev->connect_state = XHCI_CONN_IDLE; /* Recovery couldn't even be issued -- clean * terminal failure so the synchronous waiter * doesn't spin forever on a wedged command. */ xhci_stall_fail(dev); } } else { log_message(LOG_ERROR, "xhci: reset endpoint failed -- stalling out clean"); dev->connect_state = XHCI_CONN_IDLE; xhci_stall_fail(dev); } } else if (dev->connect_state == XHCI_CONN_AWAIT_SET_TR_DEQUEUE) { /* G.1 / §F.14 stall recovery, command step 2 of 2: Set * TR Dequeue Pointer has completed. The xHC side is * now fully recovered; the device side still needs its * halt cleared via control transfer, so chain into the * deferred CLEAR_FEATURE(ENDPOINT_HALT). */ if (code == XHCI_COMPLETION_CODE_SUCCESS) { log_message(LOG_DEBUG, "xhci: set TR dequeue pointer succeeded"); } else { log_message(LOG_ERROR, "xhci: set TR dequeue pointer failed -- stalling out clean"); xhci_stall_fail(dev); } dev->connect_state = XHCI_CONN_IDLE; if (dev->bot_last_status != BOT_STATUS_FAILED) { dev->bot_reset_clear_remaining = 1; xhci_msc_slot_t *rms = xhci_msc_slot_for(dev, slot_id); if (rms) rms->next_action = XHCI_NEXT_ACTION_CLEAR_HALT; } } else { log_message(LOG_DEBUG, "xhci: command completion event"); } /* FABRIC-3.md §VII item 3 (2026-09-05): the Command Ring * just freed up (or never left XHCI_CONN_IDLE) -- service * one queued connect/disconnect, if any, now rather than * waiting for the next unrelated event to happen to check. */ if (dev->connect_state == XHCI_CONN_IDLE) { xhci_drain_pending_event(dev); } break; } case XHCI_TRB_TYPE_TRANSFER_EVENT: { uint32_t code = XHCI_EVT_COMPLETION_CODE(trb->status); /* FABRIC-3.md §VII (2026-09-05), the real bug that made * concurrent multi-device enumeration hang: this used to * trust a single shared dev->pending_transfer_slot_id * scalar to say which slot a Transfer Event belonged to -- * wrong once two devices' EP0/bulk rings can each complete * independently, since one device's real hardware * completion could arrive after a second device's own * transfer had already overwritten that shared scalar, * permanently misattributing (or losing) the first * device's completion. The Transfer Event TRB itself * already carries the real Slot ID (xHCI 1.2 table 6-38, * same field/bit position as Command Completion Event) -- * use that directly instead of trusting external * bookkeeping, and keep transfer_purpose per-slot * (xhci_msc_slot_t) so each device's own outstanding * transfer is independent. */ uint32_t xfer_slot_id = XHCI_EVT_SLOT_ID(trb->control); xhci_msc_slot_t *xms = xhci_msc_slot_for(dev, xfer_slot_id); /* A boot keyboard's report, on its interrupt endpoint (A3): * read it, and queue for the next -- deferred, like every * doorbell here, until ERDP is updated below. */ if (xms && xms->hid_running && xms->hid_ep_addr && XHCI_EVT_EP_ID(trb->control) == XHCI_EP_ADDR_TO_DCI(xms->hid_ep_addr)) { if (code == XHCI_COMPLETION_CODE_SUCCESS || code == XHCI_COMPLETION_CODE_SHORT_PACKET) { uint32_t asked = xms->hid_max_packet ? xms->hid_max_packet : 8u; uint32_t left = trb->status & 0xFFFFFFu; if (asked > 64u) asked = 64u; usb_kbd_report(xms->hid_prev, xms->hid_buf, left < asked ? asked - left : 0u); xms->next_action = XHCI_NEXT_ACTION_HID_POLL; } else { console_println("USB keyboard: report failed -- keyboard stopped"); xms->hid_running = 0; if (g_xhci_keyboards) g_xhci_keyboards--; } break; } if (xms && xms->transfer_purpose != XHCI_XFER_NONE) { uint32_t purpose = xms->transfer_purpose; xms->transfer_purpose = XHCI_XFER_NONE; if (code != XHCI_COMPLETION_CODE_SUCCESS) { /* G.1 / §F.14: distinguish a true STALL (the one * recoverable xHCI transfer completion) from every * other failure. A stall on a bulk endpoint enters * recovery; a plain failure (or a stall on an EP0 * control transfer, which this driver only meets * during enumeration and does not attempt to * recover) keeps today's behavior -- log and bail. */ if (code == XHCI_COMPLETION_CODE_STALL_ERROR && xhci_bulk_purpose_stalled(dev, purpose)) { xhci_handle_bulk_stall(dev, xfer_slot_id, purpose); } else { log_message(LOG_ERROR, "xhci: control transfer failed"); } break; } switch (purpose) { case XHCI_XFER_DEVICE_DESC: { log_message(LOG_DEBUG, "xhci: device descriptor received"); /* USB 2.0 spec table 9-8 layout. Logged -- * 2f's own punch list asked whether vendor/ * product IDs are even needed, or class-only * detection suffices; this surfaces the real * values, doesn't decide it. */ uint32_t id_vendor = xms->device_descriptor[8] | ((uint32_t)xms->device_descriptor[9] << 8); uint32_t id_product = xms->device_descriptor[10] | ((uint32_t)xms->device_descriptor[11] << 8); xhci_log_hex32("xhci: idVendor=", id_vendor); xhci_log_hex32("xhci: idProduct=", id_product); xhci_log_hex32("xhci: bDeviceClass=", xms->device_descriptor[4]); /* Chain: request just the Configuration * descriptor's 9-byte header first, to learn * wTotalLength before requesting everything. * Deferred (see xhci_dev_t's doc comment on * next_action) rather than called directly -- * a doorbell rung synchronously here, still * inside this event-processing loop and * before ERDP is updated, hung the guest * outright (confirmed live via checkpoint * logging, amd64 QEMU, 2026-08-22). */ xms->next_action = XHCI_NEXT_ACTION_GET_CONFIG_DESC; xms->next_action_length = 9; break; } case XHCI_XFER_CONFIG_DESC_SHORT: { uint16_t total_len = (uint16_t)(xms->config_descriptor[USB_CONFIG_OFF_TOTAL_LENGTH] | ((uint16_t)xms->config_descriptor[USB_CONFIG_OFF_TOTAL_LENGTH + 1] << 8)); xms->config_total_length = total_len; xhci_log_hex32("xhci: config wTotalLength=", total_len); xms->next_action = XHCI_NEXT_ACTION_GET_CONFIG_DESC; xms->next_action_length = total_len; break; } case XHCI_XFER_CONFIG_DESC_FULL: { log_message(LOG_DEBUG, "xhci: full config descriptor received"); /* Walk the concatenated descriptor stream * (Config + Interface + Endpoint descriptors * back to back) looking for the Interface * descriptor -- its fixed offset within the * stream isn't guaranteed, has to be found by * bDescriptorType, not assumed. */ uint16_t len = xms->config_total_length; if (len > sizeof(xms->config_descriptor)) len = (uint16_t)sizeof(xms->config_descriptor); uint16_t off = 0; int found = 0; while (off + 2 <= len) { uint8_t desc_len = xms->config_descriptor[off + USB_DESC_OFF_LENGTH]; uint8_t desc_type = xms->config_descriptor[off + USB_DESC_OFF_TYPE]; if (desc_len == 0) break; /* malformed -- avoid an infinite loop */ if (desc_type == USB_DESC_TYPE_INTERFACE && off + USB_IFACE_OFF_PROTOCOL < len) { uint8_t iface_class = xms->config_descriptor[off + USB_IFACE_OFF_CLASS]; uint8_t iface_subclass = xms->config_descriptor[off + USB_IFACE_OFF_SUBCLASS]; uint8_t iface_protocol = xms->config_descriptor[off + USB_IFACE_OFF_PROTOCOL]; xhci_log_hex32("xhci: bInterfaceClass=", iface_class); xhci_log_hex32("xhci: bInterfaceSubClass=", iface_subclass); xhci_log_hex32("xhci: bInterfaceProtocol=", iface_protocol); if (iface_class == USB_CLASS_MASS_STORAGE && iface_subclass == USB_SUBCLASS_SCSI && iface_protocol == USB_PROTOCOL_BOT) { log_message(LOG_DEBUG, "xhci: confirmed Mass Storage / SCSI / BOT device"); /* Walk the Endpoint descriptors that * follow this Interface descriptor, * stopping at the next Interface * descriptor (start of a different * interface's endpoints) or end of * the stream. Only bulk endpoints * are of interest for BOT. */ uint16_t ep_off = (uint16_t)(off + desc_len); while (ep_off + 2 <= len) { uint8_t ep_desc_len = xms->config_descriptor[ep_off + USB_DESC_OFF_LENGTH]; uint8_t ep_desc_type = xms->config_descriptor[ep_off + USB_DESC_OFF_TYPE]; if (ep_desc_len == 0) break; /* malformed -- avoid an infinite loop */ if (ep_desc_type == USB_DESC_TYPE_INTERFACE) break; if (ep_desc_type == USB_DESC_TYPE_ENDPOINT && ep_off + USB_EP_OFF_MAX_PACKET_SIZE + 1 < len) { uint8_t ep_addr = xms->config_descriptor[ep_off + USB_EP_OFF_ADDRESS]; uint8_t ep_attr = xms->config_descriptor[ep_off + USB_EP_OFF_ATTRIBUTES]; uint16_t ep_max_packet = (uint16_t)(xms->config_descriptor[ep_off + USB_EP_OFF_MAX_PACKET_SIZE] | ((uint16_t)xms->config_descriptor[ep_off + USB_EP_OFF_MAX_PACKET_SIZE + 1] << 8)); if ((ep_attr & USB_EP_ATTR_TYPE_MASK) == USB_EP_TYPE_BULK) { if (ep_addr & USB_EP_ADDR_DIR_MASK) { xms->bulk_in_ep_addr = ep_addr; xms->bulk_in_max_packet = ep_max_packet; xhci_log_hex32("xhci: bulk IN endpoint=", ep_addr); } else { xms->bulk_out_ep_addr = ep_addr; xms->bulk_out_max_packet = ep_max_packet; xhci_log_hex32("xhci: bulk OUT endpoint=", ep_addr); } } } ep_off = (uint16_t)(ep_off + ep_desc_len); } /* Deferred (see xhci_dev_t's * next_action doc comment) rather * than called directly here -- * same doorbell-ordering hazard * as the device/config descriptor * chaining above. next_action_ * config_value is staged now but * not consumed until SET_CONFIG * actually runs, after Configure * Endpoint completes below. */ xms->next_action_config_value = xms->config_descriptor[USB_CONFIG_OFF_CONFIG_VALUE]; if (xms->bulk_in_ep_addr == 0 || xms->bulk_out_ep_addr == 0) { /* Can't Configure Endpoint * without knowing both bulk * endpoints -- go straight to * SET_CONFIGURATION so the * device is still usable for * whatever doesn't need BOT * (nothing, today, but this * keeps the two concerns * separate rather than * failing enumeration * outright). */ log_message(LOG_WARN, "xhci: warning -- BOT device missing a bulk IN or OUT endpoint, skipping configure endpoint"); xms->next_action = XHCI_NEXT_ACTION_SET_CONFIG; } else { xms->next_action = XHCI_NEXT_ACTION_CONFIGURE_ENDPOINT; } } else if (iface_class == USB_CLASS_HID && iface_subclass == USB_SUBCLASS_HID_BOOT && iface_protocol == USB_PROTOCOL_HID_KEYBOARD) { /* A boot keyboard (2026-10-14, A3): its * interface number, and the first * interrupt IN endpoint after it. */ uint16_t ep_off = (uint16_t)(off + desc_len); xms->hid_iface = xms->config_descriptor[off + USB_IFACE_OFF_NUMBER]; while (ep_off + 2 <= len) { uint8_t ep_desc_len = xms->config_descriptor[ep_off + USB_DESC_OFF_LENGTH]; uint8_t ep_desc_type = xms->config_descriptor[ep_off + USB_DESC_OFF_TYPE]; if (ep_desc_len == 0) break; if (ep_desc_type == USB_DESC_TYPE_INTERFACE) break; if (ep_desc_type == USB_DESC_TYPE_ENDPOINT && ep_off + USB_EP_OFF_INTERVAL < len) { uint8_t ep_addr = xms->config_descriptor[ep_off + USB_EP_OFF_ADDRESS]; uint8_t ep_attr = xms->config_descriptor[ep_off + USB_EP_OFF_ATTRIBUTES]; if ((ep_attr & USB_EP_ATTR_TYPE_MASK) == USB_EP_TYPE_INTERRUPT && (ep_addr & USB_EP_ADDR_DIR_MASK)) { xms->hid_ep_addr = ep_addr; xms->hid_max_packet = (uint16_t)(xms->config_descriptor[ep_off + USB_EP_OFF_MAX_PACKET_SIZE] | ((uint16_t)xms->config_descriptor[ep_off + USB_EP_OFF_MAX_PACKET_SIZE + 1] << 8)) & 0x7FFu; xms->hid_interval = xms->config_descriptor[ep_off + USB_EP_OFF_INTERVAL]; break; } } ep_off = (uint16_t)(ep_off + ep_desc_len); } xms->next_action_config_value = xms->config_descriptor[USB_CONFIG_OFF_CONFIG_VALUE]; if (xms->hid_ep_addr) { xms->hid_kbd = 1; xms->next_action = XHCI_NEXT_ACTION_CONFIGURE_ENDPOINT; } else { log_message(LOG_WARN, "xhci: keyboard without an interrupt IN endpoint -- not used"); } } else { log_message(LOG_WARN, "xhci: neither a Mass Storage/SCSI/BOT drive nor a boot keyboard -- not used"); } found = 1; break; } off = (uint16_t)(off + desc_len); } if (!found) { log_message(LOG_WARN, "xhci: no Interface descriptor found in config set"); } break; } case XHCI_XFER_SET_CONFIG: { log_message(LOG_DEBUG, "xhci: device configured"); if (xms->hid_kbd) { /* A3: a keyboard, not a drive */ xms->next_action = XHCI_NEXT_ACTION_HID_SET_PROTOCOL; break; } /* Milestone 2h: hand off to sk_repl_idle(), * the only safe place to run the synchronous * capacity query + block-subsystem attach -- * see bot_msc_attach_pending's own doc * comment in xhci_driver.h. Per-slot now * (FABRIC-3.md §VII, 2026-09-05) -- xfer_slot_id * itself is the identity a separate * bot_msc_attach_slot_id field used to carry. */ xms->bot_msc_attach_pending = 1; break; } case XHCI_XFER_HID_SET_PROTOCOL: { xms->hid_running = 1; g_xhci_keyboards++; memset(xms->hid_prev, 0, sizeof(xms->hid_prev)); xms->hid_leds_sent = 0xFFu; /* tell it the lights at once */ console_println("USB keyboard ready"); xms->next_action = XHCI_NEXT_ACTION_HID_POLL; break; } case XHCI_XFER_HID_SET_LEDS: break; case XHCI_XFER_CBW_SENT: { log_message(LOG_DEBUG, "xhci: CBW send completed"); /* Deferred (see xhci_dev_t's next_action doc * comment) rather than called directly -- * same doorbell-ordering hazard as every * other chained request in this driver. * dCBWDataTransferLength == 0 (TEST UNIT * READY) means no data stage exists -- BOT * spec section 6.3 -- so skip straight to CSW * receive. Otherwise the direction depends on * which command this CBW was for: WRITE(10) * needs a Data-Out stage (bulk_out_ring), * every other data-bearing command here * (READ10, READ CAPACITY10) needs Data-In. */ if (dev->bot_expected_data_len == 0) { xms->next_action = XHCI_NEXT_ACTION_BOT_CSW_RECEIVE; } else if (dev->bot_cmd_kind == BOT_CMD_WRITE10) { xms->next_action = XHCI_NEXT_ACTION_BOT_DATA_OUT; } else { xms->next_action = XHCI_NEXT_ACTION_BOT_DATA_IN; } break; } case XHCI_XFER_BOT_DATA_IN: { log_message(LOG_DEBUG, "xhci: BOT Data-In read completed"); xms->next_action = XHCI_NEXT_ACTION_BOT_CSW_RECEIVE; break; } case XHCI_XFER_BOT_DATA_OUT: { log_message(LOG_DEBUG, "xhci: BOT Data-Out write completed"); xms->next_action = XHCI_NEXT_ACTION_BOT_CSW_RECEIVE; break; } case XHCI_XFER_CSW_RECEIVED: { /* USB Mass Storage Class BOT spec section 5.2: * a valid CSW must have the right signature * and echo the CBW's own tag -- checked before * trusting bCSWStatus at all, since a garbled * or misaligned CSW read could otherwise be * misread as a clean pass. */ if (dev->bot_csw.dCSWSignature != USB_BOT_CSW_SIGNATURE) { log_message(LOG_ERROR, "xhci: CSW signature mismatch -- discarding"); dev->bot_last_status = BOT_STATUS_FAILED; dev->bot_cmd_kind = BOT_CMD_NONE; break; } else if (dev->bot_csw.dCSWTag != dev->bot_last_tag) { log_message(LOG_ERROR, "xhci: CSW tag mismatch -- discarding"); dev->bot_last_status = BOT_STATUS_FAILED; dev->bot_cmd_kind = BOT_CMD_NONE; break; } int csw_pass = (dev->bot_csw.bCSWStatus == USB_BOT_CSW_STATUS_PASS); if (csw_pass) { log_message(LOG_DEBUG, "xhci: CSW status = PASS"); } else if (dev->bot_csw.bCSWStatus == USB_BOT_CSW_STATUS_FAILED) { /* 2026-09-22: downgraded from LOG_ERROR -- * a fresh SCSI target's standard first- * command UNIT ATTENTION behavior (see this * block's own comment a few lines down), * not a driver defect; the bounded retry * below resolves it routinely. The genuine * terminal failure ("giving up" below, * after XHCI_BOT_TUR_MAX_RETRIES) stays at * LOG_ERROR. */ log_message(LOG_INFO, "xhci: CSW status = FAILED"); } else { log_message(LOG_ERROR, "xhci: CSW status = PHASE ERROR"); } xhci_log_hex32("xhci: CSW data residue=", dev->bot_csw.dCSWDataResidue); /* TEST UNIT READY completions chain into the * actual READ(10) once PASS, or retry (bounded) * on FAILED/PHASE ERROR -- a fresh SCSI * target's standard first-command UNIT * ATTENTION behavior, not a driver defect (see * SCSI_CMD_TEST_UNIT_READY's doc comment in * xhci.h). Every other command kind is * terminal here: bot_last_status/bot_cmd_kind * reset so a synchronous caller waiting in * xhci_bot_wait_for_idle() (Milestone 2h) sees * the command as finished. */ if (dev->bot_cmd_kind == BOT_CMD_TEST_UNIT_READY) { if (csw_pass) { if (dev->bot_tur_chain_target == BOT_TUR_CHAIN_READ_CAPACITY10) { log_message(LOG_DEBUG, "xhci: unit ready -- issuing READ CAPACITY10"); xms->next_action = XHCI_NEXT_ACTION_BOT_SEND_READ_CAPACITY10; } else if (dev->bot_tur_chain_target == BOT_TUR_CHAIN_WRITE10) { log_message(LOG_DEBUG, "xhci: unit ready -- issuing WRITE10"); xms->next_action = XHCI_NEXT_ACTION_BOT_SEND_WRITE10; } else { log_message(LOG_DEBUG, "xhci: unit ready -- issuing READ10"); xms->next_action = XHCI_NEXT_ACTION_BOT_SEND_READ10; } } else if (dev->bot_tur_retries < XHCI_BOT_TUR_MAX_RETRIES) { dev->bot_tur_retries++; /* 2026-09-22: downgraded from LOG_WARN, * same reasoning as the CSW FAILED * downgrade above -- expected, routinely * self-resolving. */ log_message(LOG_INFO, "xhci: unit not ready -- retrying TEST UNIT READY"); xms->next_action = XHCI_NEXT_ACTION_BOT_SEND_TUR; } else { log_message(LOG_ERROR, "xhci: unit still not ready -- giving up"); dev->bot_last_status = BOT_STATUS_FAILED; dev->bot_cmd_kind = BOT_CMD_NONE; } } else if (dev->bot_cmd_kind == BOT_CMD_READ_CAPACITY10) { if (csw_pass) { const uint8_t *d = dev->bot_data_buf; dev->bot_cap_last_lba = ((uint32_t)d[0] << 24) | ((uint32_t)d[1] << 16) | ((uint32_t)d[2] << 8) | (uint32_t)d[3]; dev->bot_cap_block_size = ((uint32_t)d[4] << 24) | ((uint32_t)d[5] << 16) | ((uint32_t)d[6] << 8) | (uint32_t)d[7]; xhci_log_hex32("xhci: READ CAPACITY last LBA=", dev->bot_cap_last_lba); xhci_log_hex32("xhci: READ CAPACITY block size=", dev->bot_cap_block_size); } dev->bot_last_status = csw_pass ? BOT_STATUS_PASS : BOT_STATUS_FAILED; dev->bot_cmd_kind = BOT_CMD_NONE; } else { /* BOT_CMD_READ10 or BOT_CMD_WRITE10 * (BOT_CMD_NONE shouldn't reach here) -- * terminal either way. */ dev->bot_last_status = csw_pass ? BOT_STATUS_PASS : BOT_STATUS_FAILED; dev->bot_cmd_kind = BOT_CMD_NONE; } break; } case XHCI_XFER_CLEAR_HALT: { /* G.1 / §F.14: CLEAR_FEATURE(ENDPOINT_HALT) * completed. This is the device-side clear; the * xHC side was already recovered by Reset * Endpoint + Set TR Dequeue Pointer. On success * this may be one of two CLEAR_FEATUREs owed in * a BOT-reset escalation (bot_reset_clear_remaining * counts how many remain; >1 means "also clear * the other bulk endpoint"). When all clears are * done, the recovery is complete and the stalled * command stage is retried (stall_retry_action). * A CLEAR_FEATURE that itself fails or stalls * is the "step 3 itself stalls" escalation case * from §F.14 -- escalate to a full BOT Mass * Storage Reset. */ if (code != XHCI_COMPLETION_CODE_SUCCESS) { log_message(LOG_ERROR, "xhci: clear endpoint halt failed -- escalating to BOT reset"); xms->next_action = XHCI_NEXT_ACTION_BOT_RESET; break; } log_message(LOG_DEBUG, "xhci: clear endpoint halt succeeded"); if (dev->bot_reset_clear_remaining > 1) { dev->bot_reset_clear_remaining--; /* Flip to the other bulk endpoint for the * second CLEAR_FEATURE owed in a BOT-reset * escalation. */ dev->stall_ep_addr = (dev->stall_ep_addr & USB_EP_ADDR_DIR_MASK) ? xms->bulk_out_ep_addr : xms->bulk_in_ep_addr; xms->next_action = XHCI_NEXT_ACTION_CLEAR_HALT; } else { dev->bot_reset_clear_remaining = 0; /* Recovery done -- resume the rest of the * command chain at the stalled stage. */ if (dev->stall_retry_action != XHCI_NEXT_ACTION_NONE) { xms->next_action = dev->stall_retry_action; } } break; } case XHCI_XFER_BOT_RESET: { /* G.1 / §F.14: BOT Mass Storage Reset completed. * Per BOT spec 5.3.4's full procedure this is * followed by CLEAR_FEATURE(ENDPOINT_HALT) on * *both* bulk endpoints before the original * command is retried; stage that as two * chained CLEAR_HALT transfers. */ if (code != XHCI_COMPLETION_CODE_SUCCESS) { log_message(LOG_ERROR, "xhci: BOT reset failed -- stalling out clean"); xhci_stall_fail(dev); break; } log_message(LOG_DEBUG, "xhci: BOT reset succeeded -- clearing both endpoints"); dev->bot_reset_clear_remaining = 2; dev->stall_ep_addr = xms->bulk_in_ep_addr; xms->next_action = XHCI_NEXT_ACTION_CLEAR_HALT; break; } default: log_message(LOG_DEBUG, "xhci: transfer event"); break; } } else { log_message(LOG_DEBUG, "xhci: transfer event"); } break; } default: break; } dev->evt_ring_deq++; if (dev->evt_ring_deq == XHCI_RING_TRB_COUNT) { dev->evt_ring_deq = 0; dev->evt_ring_cycle ^= 1u; } evt_processed++; } /* Event Ring dequeue-pointer update (xHCI 1.2 spec §4.9.4): write the * new dequeue pointer back to ERDP with bit3 (EHB, Event Handler Busy, * RW1C) set -- writing 1 to EHB is what clears it, per spec, not a * read-modify-write of the current value. Skipping this leaves the * controller believing the event handler is still busy and it will * not post further events on this interrupter. IMAN.IP/USBSTS.EINT * are deliberately not touched here: this driver never sets * USBCMD.INTE/IMAN.IE (polled, not interrupt-driven -- see this * function's own doc comment), so those RW1C bits never latch and * have nothing to clear. */ dev->intr0->erdp = ((uint64_t)(uintptr_t)&dev->evt_ring[dev->evt_ring_deq] & XHCI_ERDP_PTR_MASK) | XHCI_ERDP_EHB; /* Deferred chained request(s), if event processing above set one -- * see xhci_next_action_t's own doc comment on why this must happen * here, after ERDP is updated, not synchronously inside the loop * above -- and why it is now a per-slot scan (FABRIC-3.md §VII, * 2026-09-05) rather than reading one scalar: a single * xhci_poll_events() call can drain completions for more than one * device's enumeration/BOT step in the same XHCI_EVT_RING_MAX_DRAIN * batch, and each needs its own doorbell ring serviced, not just * whichever one a shared scalar happened to hold last. */ for (uint32_t next_slot_id = 1; next_slot_id <= dev->max_slots; next_slot_id++) { xhci_msc_slot_t *nms = xhci_msc_slot_for(dev, next_slot_id); if (!nms || nms->next_action == XHCI_NEXT_ACTION_NONE) continue; xhci_next_action_t action = nms->next_action; nms->next_action = XHCI_NEXT_ACTION_NONE; switch (action) { case XHCI_NEXT_ACTION_GET_DEVICE_DESC: if (xhci_ep0_get_device_descriptor(dev, next_slot_id) != 0) { log_message(LOG_ERROR, "xhci: deferred device descriptor request setup failed"); } break; case XHCI_NEXT_ACTION_GET_CONFIG_DESC: if (xhci_ep0_get_config_descriptor(dev, next_slot_id, nms->next_action_length) != 0) { log_message(LOG_ERROR, "xhci: deferred config descriptor request setup failed"); } break; case XHCI_NEXT_ACTION_CONFIGURE_ENDPOINT: /* Configure Endpoint is a Command Ring op, same as Enable * Slot/Address Device/Disable Slot -- but unlike those * (gated by xhci_handle_port_connected()'s own * connect_state == XHCI_CONN_IDLE check before ever * submitting), this deferred dispatch loop used to submit * it unconditionally for every slot with the action * pending in a single pass. With 2+ devices enumerating * concurrently, that let multiple Configure Endpoint * commands sit outstanding on the Command Ring at once -- * and their completion is correlated purely by * dev->connect_state == XHCI_CONN_AWAIT_CONFIGURE_ENDPOINT * (see that branch in the Command Completion Event * handler above), a single global flag, not the event's * own Slot ID. Whichever slot's completion actually * lands while connect_state still reads * AWAIT_CONFIGURE_ENDPOINT gets correctly chained into * SET_CONFIG; every other slot's completion arrives after * connect_state has already moved on (to another slot's * own Configure Endpoint, or unrelated activity) and is * silently swallowed by the handler's generic "unrelated * command completion" catch-all -- permanently stalling * that slot's enumeration with no error ever logged. * Confirmed live 2026-09-07 (FABRIC-3.md §IX.5 follow-on): * 9 devices attached concurrently, only 1 of 9 ever * reached SET_CONFIG/attach. * * Fix: apply the same single-in-flight discipline this * driver already uses for every other Command Ring op -- * only submit if the Ring is actually free * (connect_state == XHCI_CONN_IDLE); otherwise put this * slot's action back so the very next tick's dispatch * pass retries it once the Ring frees up, rather than * racing a second command onto it. */ if (dev->connect_state != XHCI_CONN_IDLE) { nms->next_action = XHCI_NEXT_ACTION_CONFIGURE_ENDPOINT; break; } dev->connect_state = XHCI_CONN_AWAIT_CONFIGURE_ENDPOINT; if ((nms->hid_kbd ? xhci_cmd_configure_hid_endpoint(dev, next_slot_id) : xhci_cmd_configure_endpoint(dev, next_slot_id)) != 0) { log_message(LOG_ERROR, "xhci: deferred configure endpoint request setup failed"); dev->connect_state = XHCI_CONN_IDLE; } break; case XHCI_NEXT_ACTION_HID_SET_PROTOCOL: if (xhci_ep0_hid_set_protocol(dev, next_slot_id) != 0) { log_message(LOG_ERROR, "xhci: deferred keyboard set protocol setup failed"); } break; case XHCI_NEXT_ACTION_HID_POLL: { uint8_t leds; if (!nms->hid_running) break; xhci_hid_queue_report(dev, next_slot_id, nms); leds = usb_kbd_leds_wanted(); if (nms->transfer_purpose == XHCI_XFER_NONE && leds != nms->hid_leds_sent) (void)xhci_ep0_hid_set_leds(dev, next_slot_id, leds); break; } case XHCI_NEXT_ACTION_SET_CONFIG: if (xhci_ep0_set_configuration(dev, next_slot_id, nms->next_action_config_value) != 0) { log_message(LOG_ERROR, "xhci: deferred set configuration request setup failed"); } break; case XHCI_NEXT_ACTION_BOT_DATA_IN: if (xhci_bot_read_data_in(dev, next_slot_id) != 0) { log_message(LOG_ERROR, "xhci: deferred BOT Data-In read setup failed"); } break; case XHCI_NEXT_ACTION_BOT_DATA_OUT: if (xhci_bot_write_data_out(dev, next_slot_id) != 0) { log_message(LOG_ERROR, "xhci: deferred BOT Data-Out write setup failed"); } break; case XHCI_NEXT_ACTION_BOT_CSW_RECEIVE: if (xhci_bot_receive_csw(dev, next_slot_id) != 0) { log_message(LOG_ERROR, "xhci: deferred CSW receive setup failed"); } break; case XHCI_NEXT_ACTION_BOT_SEND_TUR: if (xhci_bot_send_test_unit_ready(dev, next_slot_id) != 0) { log_message(LOG_ERROR, "xhci: deferred TEST UNIT READY retry setup failed"); } break; case XHCI_NEXT_ACTION_BOT_SEND_READ10: if (xhci_bot_send_read10(dev, next_slot_id, dev->bot_read10_lba, dev->bot_read10_num_blocks, dev->bot_read10_block_size) != 0) { log_message(LOG_ERROR, "xhci: deferred READ10 setup failed"); } break; case XHCI_NEXT_ACTION_BOT_SEND_READ_CAPACITY10: if (xhci_bot_send_read_capacity10(dev, next_slot_id) != 0) { log_message(LOG_ERROR, "xhci: deferred READ CAPACITY10 setup failed"); } break; case XHCI_NEXT_ACTION_BOT_SEND_WRITE10: if (xhci_bot_send_write10(dev, next_slot_id, dev->bot_write10_lba, dev->bot_write10_num_blocks, dev->bot_write10_block_size) != 0) { log_message(LOG_ERROR, "xhci: deferred WRITE10 setup failed"); } break; case XHCI_NEXT_ACTION_CLEAR_HALT: if (xhci_ep0_clear_endpoint_halt(dev, next_slot_id, dev->stall_ep_addr) != 0) { log_message(LOG_ERROR, "xhci: deferred clear endpoint halt setup failed"); xhci_stall_fail(dev); } break; case XHCI_NEXT_ACTION_BOT_RESET: if (xhci_ep0_bot_mass_storage_reset(dev, next_slot_id) != 0) { log_message(LOG_ERROR, "xhci: deferred BOT mass storage reset setup failed"); xhci_stall_fail(dev); } break; default: break; } } }