USB: fix passthrough of bulk endpoints on Linux

The PS3 USB stack routes both bulk and interrupt requests through
usb_device_passthrough::interrupt_transfer. The old code unconditionally
called libusb_fill_interrupt_transfer regardless of the endpoint type,
which on Linux's usbfs backend is rejected with EINVAL whenever the URB
type doesn't match the endpoint's bmAttributes. Reads against bulk IN
endpoints therefore never completed and the worker thread stalled.

Look up the endpoint descriptor and dispatch to libusb_fill_bulk_transfer
when bmAttributes indicates bulk. A new usb_device::find_endpoint helper
walks the descriptor tree once per submission.

Confirmed broken on Linux and working on Windows pre-fix; macOS
untested. WinUSB silently accepts the type mismatch, which is why this
went unnoticed. Reproduces with any passthrough device that has a bulk
IN endpoint, e.g. a Bandai Namco USIO (0b9a:0910) under Taiko no Tatsujin
S111 [SCEEXE000]: the game's boot-time the I/O board check runs and then hangs.

Two related defensive fixes uncovered while debugging the above:

- Zero-length bulk/interrupt IN URBs hang in libusb until the device
  sends a ZLP. The emulated path fake-completes them immediately; mirror
  that here so drain-polls between transfers don't stall the worker.

- An unexpected libusb_submit_transfer error used to leave
  UsbTransfer::busy = true forever. Mark it as a fake completion with
  EHCI_CC_HALTED so the USB manager processes the failure cleanly
  instead of deadlocking the request.

rpcs3/Emu/Io/usb_device.cpp

@@ -24,6 +24,22 @@ void usb_device::get_location(u8* location) const
 	memcpy(location, this->location.data(), 7);
 }
 
+const UsbDeviceEndpoint* usb_device::find_endpoint(u8 endpoint_addr) const
+{
+	for (const auto& config_node : device.subnodes)
+	{
+		if (config_node.bDescriptorType != USB_DESCRIPTOR_CONFIG)
+			continue;
+
+		for (const auto& node : config_node.subnodes)
+		{
+			if (node.bDescriptorType == USB_DESCRIPTOR_ENDPOINT && node._endpoint.bEndpointAddress == endpoint_addr)
+				return &node._endpoint;
+		}
+	}
+	return nullptr;
+}
+
 void usb_device::read_descriptors()
 {
 }
@@ -88,6 +104,15 @@ void usb_device_passthrough::send_libusb_transfer(libusb_transfer* transfer)
 		default:
 		{
 			sys_usbd.error("Unexpected error from libusb_submit_transfer: %d(%s)", res, libusb_error_name(res));
+
+			// Mark as a failed fake transfer so the USB manager processes a completion
+			// instead of leaving the request stuck in the busy state forever.
+			UsbTransfer* usbd_transfer = static_cast<UsbTransfer*>(transfer->user_data);
+			usbd_transfer->busy = true;
+			usbd_transfer->fake = true;
+			usbd_transfer->expected_result = EHCI_CC_HALTED;
+			usbd_transfer->expected_count = 0;
+			usbd_transfer->expected_time = get_timestamp();
 			return;
 		}
 		}
@@ -163,7 +188,32 @@ void usb_device_passthrough::control_transfer(u8 bmRequestType, u8 bRequest, u16
 
 void usb_device_passthrough::interrupt_transfer(u32 buf_size, u8* buf, u32 endpoint, UsbTransfer* transfer)
 {
-	libusb_fill_interrupt_transfer(transfer->transfer, lusb_handle, endpoint, buf, buf_size, callback_transfer, transfer, 0);
+	// Zero-length bulk/interrupt IN URBs hang in libusb until the device sends a ZLP.
+	// The emulated path fake-completes these immediately with count=0; mirror that here
+	// so games that do drain-polls between transfers don't stall the worker thread.
+	if (buf_size == 0 && (endpoint & 0x80))
+	{
+		transfer->fake = true;
+		transfer->expected_count = 0;
+		transfer->expected_result = HC_CC_NOERR;
+		transfer->expected_time = get_timestamp() + 1'000;
+		return;
+	}
+
+	// Pick the libusb helper matching the endpoint's actual transfer type. The PS3 USB
+	// stack routes both bulk and interrupt transfers through this method, but submitting
+	// an interrupt URB to a bulk endpoint fails with EINVAL on Linux.
+	const UsbDeviceEndpoint* ep_desc = find_endpoint(static_cast<u8>(endpoint));
+	const bool is_bulk = ep_desc && (ep_desc->bmAttributes & 0x03) == 0x02;
+
+	if (is_bulk)
+	{
+		libusb_fill_bulk_transfer(transfer->transfer, lusb_handle, endpoint, buf, buf_size, callback_transfer, transfer, 0);
+	}
+	else
+	{
+		libusb_fill_interrupt_transfer(transfer->transfer, lusb_handle, endpoint, buf, buf_size, callback_transfer, transfer, 0);
+	}
 	send_libusb_transfer(transfer->transfer);
 }
 

rpcs3/Emu/Io/usb_device.h

@@ -238,6 +238,9 @@ public:
 	virtual void isochronous_transfer(UsbTransfer* transfer)                                                                                        = 0;
 
 public:
+	// Look up an endpoint descriptor by address in the descriptor tree
+	const UsbDeviceEndpoint* find_endpoint(u8 endpoint_addr) const;
+
 	// device ID if the device has been ldded(0 otherwise)
 	u32 assigned_number = 0;
 	// base device descriptor, every other descriptor is a subnode