From b28cc29c5d1e4e006f56fd17f278ff64eacad923 Mon Sep 17 00:00:00 2001 From: "liping.zhang" Date: Mon, 11 Jan 2016 13:31:01 +0800 Subject: xt_qtaguid: fix a race condition in if_tag_stat_update Miss a lock protection in if_tag_stat_update while doing get_iface_entry. So if one CPU is doing iface_stat_create while another CPU is doing if_tag_stat_update, race will happened. Change-Id: Ib8d98e542f4e385685499f5b7bb7354f08654a75 Signed-off-by: Liping Zhang --- net/netfilter/xt_qtaguid.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/net/netfilter/xt_qtaguid.c b/net/netfilter/xt_qtaguid.c index 90b2c6aac7d8..c9b5ff123a8d 100644 --- a/net/netfilter/xt_qtaguid.c +++ b/net/netfilter/xt_qtaguid.c @@ -1291,11 +1291,12 @@ static void if_tag_stat_update(const char *ifname, uid_t uid, "uid=%u sk=%p dir=%d proto=%d bytes=%d)\n", ifname, uid, sk, direction, proto, bytes); - + spin_lock_bh(&iface_stat_list_lock); iface_entry = get_iface_entry(ifname); if (!iface_entry) { pr_err_ratelimited("qtaguid: iface_stat: stat_update() " "%s not found\n", ifname); + spin_unlock_bh(&iface_stat_list_lock); return; } /* It is ok to process data when an iface_entry is inactive */ @@ -1331,8 +1332,7 @@ static void if_tag_stat_update(const char *ifname, uid_t uid, * {0, uid_tag} will also get updated. */ tag_stat_update(tag_stat_entry, direction, proto, bytes); - spin_unlock_bh(&iface_entry->tag_stat_list_lock); - return; + goto unlock; } /* Loop over tag list under this interface for {0,uid_tag} */ @@ -1372,6 +1372,7 @@ static void if_tag_stat_update(const char *ifname, uid_t uid, tag_stat_update(new_tag_stat, direction, proto, bytes); unlock: spin_unlock_bh(&iface_entry->tag_stat_list_lock); + spin_unlock_bh(&iface_stat_list_lock); } static int iface_netdev_event_handler(struct notifier_block *nb, -- cgit v1.2.3 From 5cea098039af0ff16caedfb5627ff0861197fd43 Mon Sep 17 00:00:00 2001 From: "Pierre-Loup A. Griffais" Date: Mon, 22 Jun 2015 14:10:36 -0700 Subject: Input: xpad - set the LEDs properly on XBox Wireless controllers Based on Patch by Pierre-Loup A. Griffais : Add the logic to set the LEDs on XBox Wireless controllers. Command sequence found by sniffing the Windows data stream when plugging the device in. Updated based on comments on linux-input: unify codepaths in xpad_send_led_command for wired/ wireless controller. Also document command values for clarification. All values tested on Xbox 360 Wireless Controller. Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 50 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 61c761156371..6f8755e2c491 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -881,17 +881,57 @@ struct xpad_led { struct usb_xpad *xpad; }; +/** + * @param command + * 0: off + * 1: all blink, then previous setting + * 2: 1/top-left blink, then on + * 3: 2/top-right blink, then on + * 4: 3/bottom-left blink, then on + * 5: 4/bottom-right blink, then on + * 6: 1/top-left on + * 7: 2/top-right on + * 8: 3/bottom-left on + * 9: 4/bottom-right on + * 10: rotate + * 11: blink, based on previous setting + * 12: slow blink, based on previous setting + * 13: rotate with two lights + * 14: persistent slow all blink + * 15: blink once, then previous setting + */ static void xpad_send_led_command(struct usb_xpad *xpad, int command) { - if (command >= 0 && command < 14) { - mutex_lock(&xpad->odata_mutex); + command %= 16; + + mutex_lock(&xpad->odata_mutex); + + switch (xpad->xtype) { + case XTYPE_XBOX360: xpad->odata[0] = 0x01; xpad->odata[1] = 0x03; xpad->odata[2] = command; xpad->irq_out->transfer_buffer_length = 3; - usb_submit_urb(xpad->irq_out, GFP_KERNEL); - mutex_unlock(&xpad->odata_mutex); + break; + case XTYPE_XBOX360W: + xpad->odata[0] = 0x00; + xpad->odata[1] = 0x00; + xpad->odata[2] = 0x08; + xpad->odata[3] = 0x40 + command; + xpad->odata[4] = 0x00; + xpad->odata[5] = 0x00; + xpad->odata[6] = 0x00; + xpad->odata[7] = 0x00; + xpad->odata[8] = 0x00; + xpad->odata[9] = 0x00; + xpad->odata[10] = 0x00; + xpad->odata[11] = 0x00; + xpad->irq_out->transfer_buffer_length = 12; + break; } + + usb_submit_urb(xpad->irq_out, GFP_KERNEL); + mutex_unlock(&xpad->odata_mutex); } static void xpad_led_set(struct led_classdev *led_cdev, @@ -911,7 +951,7 @@ static int xpad_led_probe(struct usb_xpad *xpad) struct led_classdev *led_cdev; int error; - if (xpad->xtype != XTYPE_XBOX360) + if (xpad->xtype != XTYPE_XBOX360 && xpad->xtype != XTYPE_XBOX360W) return 0; xpad->led = led = kzalloc(sizeof(struct xpad_led), GFP_KERNEL); -- cgit v1.2.3 From 37cdf041b6dc73d890314be701345107e427230b Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Mon, 22 Jun 2015 14:11:30 -0700 Subject: Input: xpad - re-send LED command on present event The controller only receives commands when its present. So for the correct LED to be lit the LED command has to be sent on the present event. Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 6f8755e2c491..f8850f9cb331 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -344,6 +344,7 @@ struct usb_xpad { int mapping; /* map d-pad to buttons or to axes */ int xtype; /* type of xbox device */ + unsigned long led_no; /* led to lit on xbox360 controllers */ }; /* @@ -488,6 +489,8 @@ static void xpad360_process_packet(struct usb_xpad *xpad, input_sync(dev); } +static void xpad_identify_controller(struct usb_xpad *xpad); + /* * xpad360w_process_packet * @@ -510,6 +513,11 @@ static void xpad360w_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned cha if (data[1] & 0x80) { xpad->pad_present = 1; usb_submit_urb(xpad->bulk_out, GFP_ATOMIC); + /* + * Light up the segment corresponding to + * controller number. + */ + xpad_identify_controller(xpad); } else xpad->pad_present = 0; } @@ -934,6 +942,12 @@ static void xpad_send_led_command(struct usb_xpad *xpad, int command) mutex_unlock(&xpad->odata_mutex); } +static void xpad_identify_controller(struct usb_xpad *xpad) +{ + /* Light up the segment corresponding to controller number */ + xpad_send_led_command(xpad, (xpad->led_no % 4) + 2); +} + static void xpad_led_set(struct led_classdev *led_cdev, enum led_brightness value) { @@ -945,8 +959,7 @@ static void xpad_led_set(struct led_classdev *led_cdev, static int xpad_led_probe(struct usb_xpad *xpad) { - static atomic_t led_seq = ATOMIC_INIT(-1); - unsigned long led_no; + static atomic_t led_seq = ATOMIC_INIT(-1); struct xpad_led *led; struct led_classdev *led_cdev; int error; @@ -958,9 +971,9 @@ static int xpad_led_probe(struct usb_xpad *xpad) if (!led) return -ENOMEM; - led_no = atomic_inc_return(&led_seq); + xpad->led_no = atomic_inc_return(&led_seq); - snprintf(led->name, sizeof(led->name), "xpad%lu", led_no); + snprintf(led->name, sizeof(led->name), "xpad%lu", xpad->led_no); led->xpad = xpad; led_cdev = &led->led_cdev; @@ -974,10 +987,8 @@ static int xpad_led_probe(struct usb_xpad *xpad) return error; } - /* - * Light up the segment corresponding to controller number - */ - xpad_send_led_command(xpad, (led_no % 4) + 2); + /* Light up the segment corresponding to controller number */ + xpad_identify_controller(xpad); return 0; } @@ -994,6 +1005,7 @@ static void xpad_led_disconnect(struct usb_xpad *xpad) #else static int xpad_led_probe(struct usb_xpad *xpad) { return 0; } static void xpad_led_disconnect(struct usb_xpad *xpad) { } +static void xpad_identify_controller(struct usb_xpad *xpad) { } #endif -- cgit v1.2.3 From 10eac698adbbec90fc55888024c308cf8d562439 Mon Sep 17 00:00:00 2001 From: Erik Lundgren Date: Tue, 6 Oct 2015 17:04:11 -0700 Subject: Input: xpad - add Covert Forces edition of the Xbox One controller It is identical to the Xbox One controller but has a different product ID. Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index f8850f9cb331..c3ba5bee1582 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -125,6 +125,7 @@ static const struct xpad_device { { 0x045e, 0x0289, "Microsoft X-Box pad v2 (US)", 0, XTYPE_XBOX }, { 0x045e, 0x028e, "Microsoft X-Box 360 pad", 0, XTYPE_XBOX360 }, { 0x045e, 0x02d1, "Microsoft X-Box One pad", 0, XTYPE_XBOXONE }, + { 0x045e, 0x02dd, "Microsoft X-Box One pad (Covert Forces)", 0, XTYPE_XBOXONE }, { 0x045e, 0x0291, "Xbox 360 Wireless Receiver (XBOX)", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360W }, { 0x045e, 0x0719, "Xbox 360 Wireless Receiver", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360W }, { 0x044f, 0x0f07, "Thrustmaster, Inc. Controller", 0, XTYPE_XBOX }, -- cgit v1.2.3 From 6db4ce3f5f715382347480c6f9803c41f88467cb Mon Sep 17 00:00:00 2001 From: Dario Scarpa Date: Tue, 6 Oct 2015 17:04:36 -0700 Subject: Input: xpad - fix Razer Atrox Arcade Stick button mapping The "Razer Atrox Arcade Stick" features 10 buttons, and two of them (LT/RT) don't work properly. Change its definition in xpad_device[] (mapping field) to fix. Signed-off-by: Dario Scarpa Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index c3ba5bee1582..e3b393c3e696 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -205,7 +205,7 @@ static const struct xpad_device { { 0x1bad, 0xf900, "Harmonix Xbox 360 Controller", 0, XTYPE_XBOX360 }, { 0x1bad, 0xf901, "Gamestop Xbox 360 Controller", 0, XTYPE_XBOX360 }, { 0x1bad, 0xf903, "Tron Xbox 360 controller", 0, XTYPE_XBOX360 }, - { 0x24c6, 0x5000, "Razer Atrox Arcade Stick", 0, XTYPE_XBOX360 }, + { 0x24c6, 0x5000, "Razer Atrox Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 }, { 0x24c6, 0x5300, "PowerA MINI PROEX Controller", 0, XTYPE_XBOX360 }, { 0x24c6, 0x5303, "Xbox Airflo wired controller", 0, XTYPE_XBOX360 }, { 0x24c6, 0x5500, "Hori XBOX 360 EX 2 with Turbo", 0, XTYPE_XBOX360 }, -- cgit v1.2.3 From d94258a447f62b1263799747277805adda97c110 Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Tue, 6 Oct 2015 17:06:16 -0700 Subject: Input: xpad - clarify LED enumeration Rename led_no -> pad_nr: the number stored there is not the LED number - it gets translated later on to a LED number in xpad_identify_controller; Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index e3b393c3e696..5257194ef0c0 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -345,7 +345,7 @@ struct usb_xpad { int mapping; /* map d-pad to buttons or to axes */ int xtype; /* type of xbox device */ - unsigned long led_no; /* led to lit on xbox360 controllers */ + unsigned long pad_nr; /* the order x360 pads were attached */ }; /* @@ -357,7 +357,6 @@ struct usb_xpad { * The used report descriptor was taken from ITO Takayukis website: * http://euc.jp/periphs/xbox-controller.ja.html */ - static void xpad_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *data) { struct input_dev *dev = xpad->dev; @@ -506,7 +505,6 @@ static void xpad_identify_controller(struct usb_xpad *xpad); * 01.1 - Pad state (Bytes 4+) valid * */ - static void xpad360w_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *data) { /* Presence change */ @@ -891,6 +889,7 @@ struct xpad_led { }; /** + * set the LEDs on Xbox360 / Wireless Controllers * @param command * 0: off * 1: all blink, then previous setting @@ -943,10 +942,13 @@ static void xpad_send_led_command(struct usb_xpad *xpad, int command) mutex_unlock(&xpad->odata_mutex); } +/* + * Light up the segment corresponding to the pad number on + * Xbox 360 Controllers. + */ static void xpad_identify_controller(struct usb_xpad *xpad) { - /* Light up the segment corresponding to controller number */ - xpad_send_led_command(xpad, (xpad->led_no % 4) + 2); + xpad_send_led_command(xpad, (xpad->pad_nr % 4) + 2); } static void xpad_led_set(struct led_classdev *led_cdev, @@ -972,9 +974,9 @@ static int xpad_led_probe(struct usb_xpad *xpad) if (!led) return -ENOMEM; - xpad->led_no = atomic_inc_return(&led_seq); + xpad->pad_nr = atomic_inc_return(&led_seq); - snprintf(led->name, sizeof(led->name), "xpad%lu", xpad->led_no); + snprintf(led->name, sizeof(led->name), "xpad%lu", xpad->pad_nr); led->xpad = xpad; led_cdev = &led->led_cdev; -- cgit v1.2.3 From 23b08fb3423029a962debf22be2cfb88a91cf8a1 Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Sat, 10 Oct 2015 10:00:49 -0700 Subject: Input: xpad - use ida() for finding the pad_nr The pad_nr corresponds to the lit up LED on the controller. Therefore there should be no gaps when enumerating. Currently a LED is only re-assigned after a controller is re-connected 4 times. This patch uses ida to track connected pads - this way we can re-assign freed up pad number immediately. Consider the following case: 1. pad A is connected and gets pad_nr = 0 2. pad B is connected and gets pad_nr = 1 3. pad A is disconnected 4. pad A is connected again using ida_simple_get() controller A now correctly gets pad_nr = 0 again. Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 5257194ef0c0..9e85a234d5f1 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -243,7 +243,6 @@ static const signed short xpad_btn_triggers[] = { -1 }; - static const signed short xpad360_btn[] = { /* buttons for x360 controller */ BTN_TL, BTN_TR, /* Button LB/RB */ BTN_MODE, /* The big X button */ @@ -345,7 +344,7 @@ struct usb_xpad { int mapping; /* map d-pad to buttons or to axes */ int xtype; /* type of xbox device */ - unsigned long pad_nr; /* the order x360 pads were attached */ + int pad_nr; /* the order x360 pads were attached */ }; /* @@ -881,6 +880,9 @@ static int xpad_init_ff(struct usb_xpad *xpad) { return 0; } #if defined(CONFIG_JOYSTICK_XPAD_LEDS) #include +#include + +static DEFINE_IDA(xpad_pad_seq); struct xpad_led { char name[16]; @@ -962,7 +964,6 @@ static void xpad_led_set(struct led_classdev *led_cdev, static int xpad_led_probe(struct usb_xpad *xpad) { - static atomic_t led_seq = ATOMIC_INIT(-1); struct xpad_led *led; struct led_classdev *led_cdev; int error; @@ -974,9 +975,13 @@ static int xpad_led_probe(struct usb_xpad *xpad) if (!led) return -ENOMEM; - xpad->pad_nr = atomic_inc_return(&led_seq); + xpad->pad_nr = ida_simple_get(&xpad_pad_seq, 0, 0, GFP_KERNEL); + if (xpad->pad_nr < 0) { + error = xpad->pad_nr; + goto err_free_mem; + } - snprintf(led->name, sizeof(led->name), "xpad%lu", xpad->pad_nr); + snprintf(led->name, sizeof(led->name), "xpad%d", xpad->pad_nr); led->xpad = xpad; led_cdev = &led->led_cdev; @@ -984,16 +989,19 @@ static int xpad_led_probe(struct usb_xpad *xpad) led_cdev->brightness_set = xpad_led_set; error = led_classdev_register(&xpad->udev->dev, led_cdev); - if (error) { - kfree(led); - xpad->led = NULL; - return error; - } + if (error) + goto err_free_id; /* Light up the segment corresponding to controller number */ xpad_identify_controller(xpad); - return 0; + +err_free_id: + ida_simple_remove(&xpad_pad_seq, xpad->pad_nr); +err_free_mem: + kfree(led); + xpad->led = NULL; + return error; } static void xpad_led_disconnect(struct usb_xpad *xpad) @@ -1002,6 +1010,7 @@ static void xpad_led_disconnect(struct usb_xpad *xpad) if (xpad_led) { led_classdev_unregister(&xpad_led->led_cdev); + ida_simple_remove(&xpad_pad_seq, xpad->pad_nr); kfree(xpad_led); } } @@ -1011,7 +1020,6 @@ static void xpad_led_disconnect(struct usb_xpad *xpad) { } static void xpad_identify_controller(struct usb_xpad *xpad) { } #endif - static int xpad_open(struct input_dev *dev) { struct usb_xpad *xpad = input_get_drvdata(dev); -- cgit v1.2.3 From 2b85031882517a36d70bd6f8e329b5b32c8f8e43 Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Tue, 6 Oct 2015 17:07:31 -0700 Subject: Input: xpad - remove needless bulk out URB used for LED setup This code was probably wrong ever since and is redundant with xpad_send_led_command. Both try to send a similar command to the xbox360 controller. However xpad_send_led_command correctly uses the pad_nr instead of bInterfaceNumber to select the led and re-uses the irq_out URB instead of creating a new one. Note that this change only affects the two supported wireless controllers. Tested using the xbox360 wireless controller (PC). Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 79 +------------------------------------------ 1 file changed, 1 insertion(+), 78 deletions(-) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 9e85a234d5f1..946bf3c74834 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -328,9 +328,6 @@ struct usb_xpad { unsigned char *idata; /* input data */ dma_addr_t idata_dma; - struct urb *bulk_out; - unsigned char *bdata; - struct urb *irq_out; /* urb for interrupt out report */ unsigned char *odata; /* output data */ dma_addr_t odata_dma; @@ -510,7 +507,6 @@ static void xpad360w_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned cha if (data[0] & 0x08) { if (data[1] & 0x80) { xpad->pad_present = 1; - usb_submit_urb(xpad->bulk_out, GFP_ATOMIC); /* * Light up the segment corresponding to * controller number. @@ -672,28 +668,6 @@ exit: __func__, retval); } -static void xpad_bulk_out(struct urb *urb) -{ - struct usb_xpad *xpad = urb->context; - struct device *dev = &xpad->intf->dev; - - switch (urb->status) { - case 0: - /* success */ - break; - case -ECONNRESET: - case -ENOENT: - case -ESHUTDOWN: - /* this urb is terminated, clean up */ - dev_dbg(dev, "%s - urb shutting down with status: %d\n", - __func__, urb->status); - break; - default: - dev_dbg(dev, "%s - nonzero urb status received: %d\n", - __func__, urb->status); - } -} - static void xpad_irq_out(struct urb *urb) { struct usb_xpad *xpad = urb->context; @@ -1229,52 +1203,6 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id usb_set_intfdata(intf, xpad); if (xpad->xtype == XTYPE_XBOX360W) { - /* - * Setup the message to set the LEDs on the - * controller when it shows up - */ - xpad->bulk_out = usb_alloc_urb(0, GFP_KERNEL); - if (!xpad->bulk_out) { - error = -ENOMEM; - goto fail7; - } - - xpad->bdata = kzalloc(XPAD_PKT_LEN, GFP_KERNEL); - if (!xpad->bdata) { - error = -ENOMEM; - goto fail8; - } - - xpad->bdata[2] = 0x08; - switch (intf->cur_altsetting->desc.bInterfaceNumber) { - case 0: - xpad->bdata[3] = 0x42; - break; - case 2: - xpad->bdata[3] = 0x43; - break; - case 4: - xpad->bdata[3] = 0x44; - break; - case 6: - xpad->bdata[3] = 0x45; - } - - ep_irq_in = &intf->cur_altsetting->endpoint[1].desc; - if (usb_endpoint_is_bulk_out(ep_irq_in)) { - usb_fill_bulk_urb(xpad->bulk_out, udev, - usb_sndbulkpipe(udev, - ep_irq_in->bEndpointAddress), - xpad->bdata, XPAD_PKT_LEN, - xpad_bulk_out, xpad); - } else { - usb_fill_int_urb(xpad->bulk_out, udev, - usb_sndintpipe(udev, - ep_irq_in->bEndpointAddress), - xpad->bdata, XPAD_PKT_LEN, - xpad_bulk_out, xpad, 0); - } - /* * Submit the int URB immediately rather than waiting for open * because we get status messages from the device whether @@ -1285,13 +1213,11 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id xpad->irq_in->dev = xpad->udev; error = usb_submit_urb(xpad->irq_in, GFP_KERNEL); if (error) - goto fail9; + goto fail7; } return 0; - fail9: kfree(xpad->bdata); - fail8: usb_free_urb(xpad->bulk_out); fail7: input_unregister_device(input_dev); input_dev = NULL; fail6: xpad_led_disconnect(xpad); @@ -1315,8 +1241,6 @@ static void xpad_disconnect(struct usb_interface *intf) xpad_deinit_output(xpad); if (xpad->xtype == XTYPE_XBOX360W) { - usb_kill_urb(xpad->bulk_out); - usb_free_urb(xpad->bulk_out); usb_kill_urb(xpad->irq_in); } @@ -1324,7 +1248,6 @@ static void xpad_disconnect(struct usb_interface *intf) usb_free_coherent(xpad->udev, XPAD_PKT_LEN, xpad->idata, xpad->idata_dma); - kfree(xpad->bdata); kfree(xpad); usb_set_intfdata(intf, NULL); -- cgit v1.2.3 From 2d3f9bde3e222fd5ac7e7b5bb980d1cd43c62017 Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Sat, 10 Oct 2015 09:32:55 -0700 Subject: Input: xpad - factor out URB submission in xpad_play_effect Move submission logic to a single point at the end of the function. This makes it easy to add locking/ queuing code later on. Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 140 +++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 71 deletions(-) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 946bf3c74834..67ab85a4fd0e 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -762,80 +762,78 @@ static void xpad_deinit_output(struct usb_xpad *xpad) static int xpad_play_effect(struct input_dev *dev, void *data, struct ff_effect *effect) { struct usb_xpad *xpad = input_get_drvdata(dev); + __u16 strong; + __u16 weak; - if (effect->type == FF_RUMBLE) { - __u16 strong = effect->u.rumble.strong_magnitude; - __u16 weak = effect->u.rumble.weak_magnitude; - - switch (xpad->xtype) { - - case XTYPE_XBOX: - xpad->odata[0] = 0x00; - xpad->odata[1] = 0x06; - xpad->odata[2] = 0x00; - xpad->odata[3] = strong / 256; /* left actuator */ - xpad->odata[4] = 0x00; - xpad->odata[5] = weak / 256; /* right actuator */ - xpad->irq_out->transfer_buffer_length = 6; - - return usb_submit_urb(xpad->irq_out, GFP_ATOMIC); - - case XTYPE_XBOX360: - xpad->odata[0] = 0x00; - xpad->odata[1] = 0x08; - xpad->odata[2] = 0x00; - xpad->odata[3] = strong / 256; /* left actuator? */ - xpad->odata[4] = weak / 256; /* right actuator? */ - xpad->odata[5] = 0x00; - xpad->odata[6] = 0x00; - xpad->odata[7] = 0x00; - xpad->irq_out->transfer_buffer_length = 8; - - return usb_submit_urb(xpad->irq_out, GFP_ATOMIC); - - case XTYPE_XBOX360W: - xpad->odata[0] = 0x00; - xpad->odata[1] = 0x01; - xpad->odata[2] = 0x0F; - xpad->odata[3] = 0xC0; - xpad->odata[4] = 0x00; - xpad->odata[5] = strong / 256; - xpad->odata[6] = weak / 256; - xpad->odata[7] = 0x00; - xpad->odata[8] = 0x00; - xpad->odata[9] = 0x00; - xpad->odata[10] = 0x00; - xpad->odata[11] = 0x00; - xpad->irq_out->transfer_buffer_length = 12; - - return usb_submit_urb(xpad->irq_out, GFP_ATOMIC); - - case XTYPE_XBOXONE: - xpad->odata[0] = 0x09; /* activate rumble */ - xpad->odata[1] = 0x08; - xpad->odata[2] = 0x00; - xpad->odata[3] = 0x08; /* continuous effect */ - xpad->odata[4] = 0x00; /* simple rumble mode */ - xpad->odata[5] = 0x03; /* L and R actuator only */ - xpad->odata[6] = 0x00; /* TODO: LT actuator */ - xpad->odata[7] = 0x00; /* TODO: RT actuator */ - xpad->odata[8] = strong / 256; /* left actuator */ - xpad->odata[9] = weak / 256; /* right actuator */ - xpad->odata[10] = 0x80; /* length of pulse */ - xpad->odata[11] = 0x00; /* stop period of pulse */ - xpad->irq_out->transfer_buffer_length = 12; - - return usb_submit_urb(xpad->irq_out, GFP_ATOMIC); - - default: - dev_dbg(&xpad->dev->dev, - "%s - rumble command sent to unsupported xpad type: %d\n", - __func__, xpad->xtype); - return -1; - } + if (effect->type != FF_RUMBLE) + return 0; + + strong = effect->u.rumble.strong_magnitude; + weak = effect->u.rumble.weak_magnitude; + + switch (xpad->xtype) { + case XTYPE_XBOX: + xpad->odata[0] = 0x00; + xpad->odata[1] = 0x06; + xpad->odata[2] = 0x00; + xpad->odata[3] = strong / 256; /* left actuator */ + xpad->odata[4] = 0x00; + xpad->odata[5] = weak / 256; /* right actuator */ + xpad->irq_out->transfer_buffer_length = 6; + break; + + case XTYPE_XBOX360: + xpad->odata[0] = 0x00; + xpad->odata[1] = 0x08; + xpad->odata[2] = 0x00; + xpad->odata[3] = strong / 256; /* left actuator? */ + xpad->odata[4] = weak / 256; /* right actuator? */ + xpad->odata[5] = 0x00; + xpad->odata[6] = 0x00; + xpad->odata[7] = 0x00; + xpad->irq_out->transfer_buffer_length = 8; + break; + + case XTYPE_XBOX360W: + xpad->odata[0] = 0x00; + xpad->odata[1] = 0x01; + xpad->odata[2] = 0x0F; + xpad->odata[3] = 0xC0; + xpad->odata[4] = 0x00; + xpad->odata[5] = strong / 256; + xpad->odata[6] = weak / 256; + xpad->odata[7] = 0x00; + xpad->odata[8] = 0x00; + xpad->odata[9] = 0x00; + xpad->odata[10] = 0x00; + xpad->odata[11] = 0x00; + xpad->irq_out->transfer_buffer_length = 12; + break; + + case XTYPE_XBOXONE: + xpad->odata[0] = 0x09; /* activate rumble */ + xpad->odata[1] = 0x08; + xpad->odata[2] = 0x00; + xpad->odata[3] = 0x08; /* continuous effect */ + xpad->odata[4] = 0x00; /* simple rumble mode */ + xpad->odata[5] = 0x03; /* L and R actuator only */ + xpad->odata[6] = 0x00; /* TODO: LT actuator */ + xpad->odata[7] = 0x00; /* TODO: RT actuator */ + xpad->odata[8] = strong / 256; /* left actuator */ + xpad->odata[9] = weak / 256; /* right actuator */ + xpad->odata[10] = 0x80; /* length of pulse */ + xpad->odata[11] = 0x00; /* stop period of pulse */ + xpad->irq_out->transfer_buffer_length = 12; + break; + + default: + dev_dbg(&xpad->dev->dev, + "%s - rumble command sent to unsupported xpad type: %d\n", + __func__, xpad->xtype); + return -EINVAL; } - return 0; + return usb_submit_urb(xpad->irq_out, GFP_ATOMIC); } static int xpad_init_ff(struct usb_xpad *xpad) -- cgit v1.2.3 From 18915b8685a8346d0a923499ffe12bb099521a77 Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Sat, 10 Oct 2015 09:33:52 -0700 Subject: Input: xpad - x360w: report dpad as buttons and axes as discussed here[0], x360w is the only pad that maps dpad_to_button. This is bad for downstream developers as they have to differ between x360 and x360w which is not intuitive. This patch implements the suggested solution of exposing the dpad both as axes and as buttons. This retains backward compatibility with software already dealing with the difference while makes new software work as expected across x360/ x360w pads. [0] http://www.spinics.net/lists/linux-input/msg34421.html Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 67ab85a4fd0e..bd669c1ff380 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -435,7 +435,16 @@ static void xpad360_process_packet(struct usb_xpad *xpad, input_report_key(dev, BTN_TRIGGER_HAPPY2, data[2] & 0x08); input_report_key(dev, BTN_TRIGGER_HAPPY3, data[2] & 0x01); input_report_key(dev, BTN_TRIGGER_HAPPY4, data[2] & 0x02); - } else { + } + + /* + * This should be a simple else block. However historically + * xbox360w has mapped DPAD to buttons while xbox360 did not. This + * made no sense, but now we can not just switch back and have to + * support both behaviors. + */ + if (!(xpad->mapping & MAP_DPAD_TO_BUTTONS) || + xpad->xtype == XTYPE_XBOX360W) { input_report_abs(dev, ABS_HAT0X, !!(data[2] & 0x08) - !!(data[2] & 0x04)); input_report_abs(dev, ABS_HAT0Y, @@ -1158,7 +1167,16 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id if (xpad->mapping & MAP_DPAD_TO_BUTTONS) { for (i = 0; xpad_btn_pad[i] >= 0; i++) __set_bit(xpad_btn_pad[i], input_dev->keybit); - } else { + } + + /* + * This should be a simple else block. However historically + * xbox360w has mapped DPAD to buttons while xbox360 did not. This + * made no sense, but now we can not just switch back and have to + * support both behaviors. + */ + if (!(xpad->mapping & MAP_DPAD_TO_BUTTONS) || + xpad->xtype == XTYPE_XBOX360W) { for (i = 0; xpad_abs_pad[i] >= 0; i++) xpad_set_up_abs(input_dev, xpad_abs_pad[i]); } -- cgit v1.2.3 From d4d3daf5bdb756e35eb6b418c945016e427f10d9 Mon Sep 17 00:00:00 2001 From: "Pierre-Loup A. Griffais" Date: Sat, 10 Oct 2015 09:34:17 -0700 Subject: Input: xpad - move the input device creation to a new function To allow us to later create / destroy the input device from the urb callback, we need to initialize/ deinitialize the input device from a separate function. So pull that logic out now to make later patches more "obvious" as to what they do. Signed-off-by: "Pierre-Loup A. Griffais" Signed-off-by: Greg Kroah-Hartman Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 211 ++++++++++++++++++++++++------------------ 1 file changed, 119 insertions(+), 92 deletions(-) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index bd669c1ff380..486db7a1d99a 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -342,6 +342,7 @@ struct usb_xpad { int mapping; /* map d-pad to buttons or to axes */ int xtype; /* type of xbox device */ int pad_nr; /* the order x360 pads were attached */ + const char *name; /* name of the device */ }; /* @@ -1060,91 +1061,36 @@ static void xpad_set_up_abs(struct input_dev *input_dev, signed short abs) } } -static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id) +static void xpad_deinit_input(struct usb_xpad *xpad) +{ + xpad_led_disconnect(xpad); + input_unregister_device(xpad->dev); +} + +static int xpad_init_input(struct usb_xpad *xpad) { - struct usb_device *udev = interface_to_usbdev(intf); - struct usb_xpad *xpad; struct input_dev *input_dev; - struct usb_endpoint_descriptor *ep_irq_in; - int ep_irq_in_idx; int i, error; - for (i = 0; xpad_device[i].idVendor; i++) { - if ((le16_to_cpu(udev->descriptor.idVendor) == xpad_device[i].idVendor) && - (le16_to_cpu(udev->descriptor.idProduct) == xpad_device[i].idProduct)) - break; - } - - if (xpad_device[i].xtype == XTYPE_XBOXONE && - intf->cur_altsetting->desc.bInterfaceNumber != 0) { - /* - * The Xbox One controller lists three interfaces all with the - * same interface class, subclass and protocol. Differentiate by - * interface number. - */ - return -ENODEV; - } - - xpad = kzalloc(sizeof(struct usb_xpad), GFP_KERNEL); input_dev = input_allocate_device(); - if (!xpad || !input_dev) { - error = -ENOMEM; - goto fail1; - } - - xpad->idata = usb_alloc_coherent(udev, XPAD_PKT_LEN, - GFP_KERNEL, &xpad->idata_dma); - if (!xpad->idata) { - error = -ENOMEM; - goto fail1; - } - - xpad->irq_in = usb_alloc_urb(0, GFP_KERNEL); - if (!xpad->irq_in) { - error = -ENOMEM; - goto fail2; - } - - xpad->udev = udev; - xpad->intf = intf; - xpad->mapping = xpad_device[i].mapping; - xpad->xtype = xpad_device[i].xtype; - - if (xpad->xtype == XTYPE_UNKNOWN) { - if (intf->cur_altsetting->desc.bInterfaceClass == USB_CLASS_VENDOR_SPEC) { - if (intf->cur_altsetting->desc.bInterfaceProtocol == 129) - xpad->xtype = XTYPE_XBOX360W; - else - xpad->xtype = XTYPE_XBOX360; - } else - xpad->xtype = XTYPE_XBOX; - - if (dpad_to_buttons) - xpad->mapping |= MAP_DPAD_TO_BUTTONS; - if (triggers_to_buttons) - xpad->mapping |= MAP_TRIGGERS_TO_BUTTONS; - if (sticks_to_null) - xpad->mapping |= MAP_STICKS_TO_NULL; - } + if (!input_dev) + return -ENOMEM; xpad->dev = input_dev; - usb_make_path(udev, xpad->phys, sizeof(xpad->phys)); - strlcat(xpad->phys, "/input0", sizeof(xpad->phys)); - - input_dev->name = xpad_device[i].name; + input_dev->name = xpad->name; input_dev->phys = xpad->phys; - usb_to_input_id(udev, &input_dev->id); - input_dev->dev.parent = &intf->dev; + usb_to_input_id(xpad->udev, &input_dev->id); + input_dev->dev.parent = &xpad->intf->dev; input_set_drvdata(input_dev, xpad); input_dev->open = xpad_open; input_dev->close = xpad_close; - input_dev->evbit[0] = BIT_MASK(EV_KEY); + __set_bit(EV_KEY, input_dev->evbit); if (!(xpad->mapping & MAP_STICKS_TO_NULL)) { - input_dev->evbit[0] |= BIT_MASK(EV_ABS); + __set_bit(EV_ABS, input_dev->evbit); /* set up axes */ for (i = 0; xpad_abs[i] >= 0; i++) xpad_set_up_abs(input_dev, xpad_abs[i]); @@ -1189,17 +1135,100 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id xpad_set_up_abs(input_dev, xpad_abs_triggers[i]); } - error = xpad_init_output(intf, xpad); - if (error) - goto fail3; - error = xpad_init_ff(xpad); if (error) - goto fail4; + goto err_free_input; error = xpad_led_probe(xpad); if (error) - goto fail5; + goto err_destroy_ff; + + error = input_register_device(xpad->dev); + if (error) + goto err_disconnect_led; + + return 0; + +err_disconnect_led: + xpad_led_disconnect(xpad); +err_destroy_ff: + input_ff_destroy(input_dev); +err_free_input: + input_free_device(input_dev); + return error; +} + +static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id) +{ + struct usb_device *udev = interface_to_usbdev(intf); + struct usb_xpad *xpad; + struct usb_endpoint_descriptor *ep_irq_in; + int ep_irq_in_idx; + int i, error; + + for (i = 0; xpad_device[i].idVendor; i++) { + if ((le16_to_cpu(udev->descriptor.idVendor) == xpad_device[i].idVendor) && + (le16_to_cpu(udev->descriptor.idProduct) == xpad_device[i].idProduct)) + break; + } + + if (xpad_device[i].xtype == XTYPE_XBOXONE && + intf->cur_altsetting->desc.bInterfaceNumber != 0) { + /* + * The Xbox One controller lists three interfaces all with the + * same interface class, subclass and protocol. Differentiate by + * interface number. + */ + return -ENODEV; + } + + xpad = kzalloc(sizeof(struct usb_xpad), GFP_KERNEL); + if (!xpad) + return -ENOMEM; + + usb_make_path(udev, xpad->phys, sizeof(xpad->phys)); + strlcat(xpad->phys, "/input0", sizeof(xpad->phys)); + + xpad->idata = usb_alloc_coherent(udev, XPAD_PKT_LEN, + GFP_KERNEL, &xpad->idata_dma); + if (!xpad->idata) { + error = -ENOMEM; + goto err_free_mem; + } + + xpad->irq_in = usb_alloc_urb(0, GFP_KERNEL); + if (!xpad->irq_in) { + error = -ENOMEM; + goto err_free_idata; + } + + xpad->udev = udev; + xpad->intf = intf; + xpad->mapping = xpad_device[i].mapping; + xpad->xtype = xpad_device[i].xtype; + xpad->name = xpad_device[i].name; + + if (xpad->xtype == XTYPE_UNKNOWN) { + if (intf->cur_altsetting->desc.bInterfaceClass == USB_CLASS_VENDOR_SPEC) { + if (intf->cur_altsetting->desc.bInterfaceProtocol == 129) + xpad->xtype = XTYPE_XBOX360W; + else + xpad->xtype = XTYPE_XBOX360; + } else { + xpad->xtype = XTYPE_XBOX; + } + + if (dpad_to_buttons) + xpad->mapping |= MAP_DPAD_TO_BUTTONS; + if (triggers_to_buttons) + xpad->mapping |= MAP_TRIGGERS_TO_BUTTONS; + if (sticks_to_null) + xpad->mapping |= MAP_STICKS_TO_NULL; + } + + error = xpad_init_output(intf, xpad); + if (error) + goto err_free_in_urb; /* Xbox One controller has in/out endpoints swapped. */ ep_irq_in_idx = xpad->xtype == XTYPE_XBOXONE ? 1 : 0; @@ -1212,12 +1241,12 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id xpad->irq_in->transfer_dma = xpad->idata_dma; xpad->irq_in->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; - error = input_register_device(xpad->dev); - if (error) - goto fail6; - usb_set_intfdata(intf, xpad); + error = xpad_init_input(xpad); + if (error) + goto err_deinit_output; + if (xpad->xtype == XTYPE_XBOX360W) { /* * Submit the int URB immediately rather than waiting for open @@ -1229,20 +1258,19 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id xpad->irq_in->dev = xpad->udev; error = usb_submit_urb(xpad->irq_in, GFP_KERNEL); if (error) - goto fail7; + goto err_deinit_input; } - return 0; - fail7: input_unregister_device(input_dev); - input_dev = NULL; - fail6: xpad_led_disconnect(xpad); - fail5: if (input_dev) - input_ff_destroy(input_dev); - fail4: xpad_deinit_output(xpad); - fail3: usb_free_urb(xpad->irq_in); - fail2: usb_free_coherent(udev, XPAD_PKT_LEN, xpad->idata, xpad->idata_dma); - fail1: input_free_device(input_dev); +err_deinit_input: + xpad_deinit_input(xpad); +err_deinit_output: + xpad_deinit_output(xpad); +err_free_in_urb: + usb_free_urb(xpad->irq_in); +err_free_idata: + usb_free_coherent(udev, XPAD_PKT_LEN, xpad->idata, xpad->idata_dma); +err_free_mem: kfree(xpad); return error; @@ -1252,8 +1280,7 @@ static void xpad_disconnect(struct usb_interface *intf) { struct usb_xpad *xpad = usb_get_intfdata (intf); - xpad_led_disconnect(xpad); - input_unregister_device(xpad->dev); + xpad_deinit_input(xpad); xpad_deinit_output(xpad); if (xpad->xtype == XTYPE_XBOX360W) { -- cgit v1.2.3 From ba525f83100f79129e46a790e4de442170af02e3 Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Sat, 10 Oct 2015 09:36:17 -0700 Subject: Input: xpad - query wireless controller state at init When we initialize the driver/device, we really don't know how many controllers are connected. So send a "query presence" command to the base-station. (Command discovered by Zachary Lund) Presence packet taken from: https://github.com/computerquip/xpad5 Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 486db7a1d99a..d382d48d6750 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -768,6 +768,33 @@ static void xpad_deinit_output(struct usb_xpad *xpad) } } +static int xpad_inquiry_pad_presence(struct usb_xpad *xpad) +{ + int retval; + + mutex_lock(&xpad->odata_mutex); + + xpad->odata[0] = 0x08; + xpad->odata[1] = 0x00; + xpad->odata[2] = 0x0F; + xpad->odata[3] = 0xC0; + xpad->odata[4] = 0x00; + xpad->odata[5] = 0x00; + xpad->odata[6] = 0x00; + xpad->odata[7] = 0x00; + xpad->odata[8] = 0x00; + xpad->odata[9] = 0x00; + xpad->odata[10] = 0x00; + xpad->odata[11] = 0x00; + xpad->irq_out->transfer_buffer_length = 12; + + retval = usb_submit_urb(xpad->irq_out, GFP_KERNEL); + + mutex_unlock(&xpad->odata_mutex); + + return retval; +} + #ifdef CONFIG_JOYSTICK_XPAD_FF static int xpad_play_effect(struct input_dev *dev, void *data, struct ff_effect *effect) { @@ -1259,9 +1286,22 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id error = usb_submit_urb(xpad->irq_in, GFP_KERNEL); if (error) goto err_deinit_input; + + /* + * Send presence packet. + * This will force the controller to resend connection packets. + * This is useful in the case we activate the module after the + * adapter has been plugged in, as it won't automatically + * send us info about the controllers. + */ + error = xpad_inquiry_pad_presence(xpad); + if (error) + goto err_kill_in_urb; } return 0; +err_kill_in_urb: + usb_kill_urb(xpad->irq_in); err_deinit_input: xpad_deinit_input(xpad); err_deinit_output: -- cgit v1.2.3 From 6aeebbacc0ce77337cb1e82f219ade77350562eb Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Mon, 19 Oct 2015 00:06:58 -0700 Subject: Input: xpad - fix clash of presence handling with LED setting Do not call xpad_identify_controller at init with wireless devices: it conflicts with the already sent presence packet and will be called by xpad360w_process_packet as needed anyway. Signed-off-by: Pavel Rojtberg Signed-off-by: Dmitry Torokhov Signed-off-by: Tim Clark --- drivers/input/joystick/xpad.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index d382d48d6750..fd4100d56d8c 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -1001,8 +1001,15 @@ static int xpad_led_probe(struct usb_xpad *xpad) if (error) goto err_free_id; - /* Light up the segment corresponding to controller number */ - xpad_identify_controller(xpad); + if (xpad->xtype == XTYPE_XBOX360) { + /* + * Light up the segment corresponding to controller + * number on wired devices. On wireless we'll do that + * when they respond to "presence" packet. + */ + xpad_identify_controller(xpad); + } + return 0; err_free_id: -- cgit v1.2.3 From 7a3c57bad676bd7655a00c20ceb721dad6abf5cb Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Thu, 14 Jan 2016 14:33:52 +0100 Subject: UPSTREAM: memcg: Only free spare array when readers are done A spare array holding mem cgroup threshold events is kept around to make sure we can always safely deregister an event and have an array to store the new set of events in. In the scenario where we're going from 1 to 0 registered events, the pointer to the primary array containing 1 event is copied to the spare slot, and then the spare slot is freed because no events are left. However, it is freed before calling synchronize_rcu(), which means readers may still be accessing threshold->primary after it is freed. Fixed by only freeing after synchronize_rcu(). Signed-off-by: Martijn Coenen --- mm/memcontrol.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 4f89f6732624..21a028aa85b7 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3823,16 +3823,17 @@ static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, swap_buffers: /* Swap primary and spare array */ thresholds->spare = thresholds->primary; - /* If all events are unregistered, free the spare array */ - if (!new) { - kfree(thresholds->spare); - thresholds->spare = NULL; - } rcu_assign_pointer(thresholds->primary, new); /* To be sure that nobody uses thresholds */ synchronize_rcu(); + + /* If all events are unregistered, free the spare array */ + if (!new) { + kfree(thresholds->spare); + thresholds->spare = NULL; + } unlock: mutex_unlock(&memcg->thresholds_lock); } -- cgit v1.2.3 From 11510e146b89e521ad4a5c7abf34b51778468f8e Mon Sep 17 00:00:00 2001 From: Amit Pundir Date: Tue, 19 Jan 2016 01:17:30 +0530 Subject: ion: fix page pool cache policy Fix redundant "buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE" checks in if(!cached ...) condition block. AOSP Change-Id: I98ee8902df0c80135dddfa998c4ca4c2bb44e40e, "ion: Handle the memory mapping correctly on x86", is broken on android-3.18+ kernels. It conflicts with upstream commit 53a91c68fa7b, "staging: ion: Add private buffer flag to skip page pooling on free", and break the ION_PRIV_FLAG_SHRINKER_FREE private flag check logic. Change-Id: I9cee4bcc3545cf92e07c21c2b42d27cf88da3316 Reported-by: chenfeng Signed-off-by: Amit Pundir --- drivers/staging/android/ion/ion_system_heap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/staging/android/ion/ion_system_heap.c b/drivers/staging/android/ion/ion_system_heap.c index 1f9feb7480b2..9adfd6de8f8c 100644 --- a/drivers/staging/android/ion/ion_system_heap.c +++ b/drivers/staging/android/ion/ion_system_heap.c @@ -83,7 +83,7 @@ static void free_buffer_page(struct ion_system_heap *heap, unsigned int order = compound_order(page); bool cached = ion_buffer_cached(buffer); - if (!cached && !(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) { + if (!cached) { struct ion_page_pool *pool = heap->pools[order_to_index(order)]; if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) ion_page_pool_free_immediate(pool, page); -- cgit v1.2.3 From 8a8431507f8f5910db5ac85b72dbdc4ed8f6b308 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats Date: Tue, 19 Jan 2016 22:09:04 +0000 Subject: UPSTREAM: KEYS: Fix keyring ref leak in join_session_keyring() (cherry pick from commit 23567fd052a9abb6d67fe8e7a9ccdd9800a540f2) This fixes CVE-2016-0728. If a thread is asked to join as a session keyring the keyring that's already set as its session, we leak a keyring reference. This can be tested with the following program: #include #include #include #include int main(int argc, const char *argv[]) { int i = 0; key_serial_t serial; serial = keyctl(KEYCTL_JOIN_SESSION_KEYRING, "leaked-keyring"); if (serial < 0) { perror("keyctl"); return -1; } if (keyctl(KEYCTL_SETPERM, serial, KEY_POS_ALL | KEY_USR_ALL) < 0) { perror("keyctl"); return -1; } for (i = 0; i < 100; i++) { serial = keyctl(KEYCTL_JOIN_SESSION_KEYRING, "leaked-keyring"); if (serial < 0) { perror("keyctl"); return -1; } } return 0; } If, after the program has run, there something like the following line in /proc/keys: 3f3d898f I--Q--- 100 perm 3f3f0000 0 0 keyring leaked-keyring: empty with a usage count of 100 * the number of times the program has been run, then the kernel is malfunctioning. If leaked-keyring has zero usages or has been garbage collected, then the problem is fixed. Reported-by: Yevgeny Pats Signed-off-by: David Howells Acked-by: Don Zickus Acked-by: Prarit Bhargava Acked-by: Jarod Wilson Signed-off-by: James Morris Change-Id: I6052fa14ab54e32878ac9895356f1049bb96a138 --- security/keys/process_keys.c | 1 + 1 file changed, 1 insertion(+) diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index bd536cb221e2..db91639c81e3 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -794,6 +794,7 @@ long join_session_keyring(const char *name) ret = PTR_ERR(keyring); goto error2; } else if (keyring == new->session_keyring) { + key_put(keyring); ret = 0; goto error2; } -- cgit v1.2.3 From 3ab57a4f6ef0f92ab9da1c2497249ee870e4e102 Mon Sep 17 00:00:00 2001 From: Stephen Smalley Date: Mon, 23 Nov 2015 16:07:41 -0500 Subject: UPSTREAM: selinux: fix bug in conditional rules handling (cherry picked from commit commit f3bef67992e8698897b584616535803887c4a73e) commit fa1aa143ac4a ("selinux: extended permissions for ioctls") introduced a bug into the handling of conditional rules, skipping the processing entirely when the caller does not provide an extended permissions (xperms) structure. Access checks from userspace using /sys/fs/selinux/access do not include such a structure since that interface does not presently expose extended permission information. As a result, conditional rules were being ignored entirely on userspace access requests, producing denials when access was allowed by conditional rules in the policy. Fix the bug by only skipping computation of extended permissions in this situation, not the entire conditional rules processing. Change-Id: I8182999ee217bb9ca5f2ab4bbeb1806daef3b98a Reported-by: Laurent Bigonville Signed-off-by: Stephen Smalley [PM: fixed long lines in patch description] Cc: stable@vger.kernel.org # 4.3 Signed-off-by: Paul Moore --- security/selinux/ss/conditional.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/selinux/ss/conditional.c b/security/selinux/ss/conditional.c index 18643bf9894d..456e1a9bcfde 100644 --- a/security/selinux/ss/conditional.c +++ b/security/selinux/ss/conditional.c @@ -638,7 +638,7 @@ void cond_compute_av(struct avtab *ctab, struct avtab_key *key, { struct avtab_node *node; - if (!ctab || !key || !avd || !xperms) + if (!ctab || !key || !avd) return; for (node = avtab_search_node(ctab, key); node; @@ -657,7 +657,7 @@ void cond_compute_av(struct avtab *ctab, struct avtab_key *key, if ((u16)(AVTAB_AUDITALLOW|AVTAB_ENABLED) == (node->key.specified & (AVTAB_AUDITALLOW|AVTAB_ENABLED))) avd->auditallow |= node->datum.u.data; - if ((node->key.specified & AVTAB_ENABLED) && + if (xperms && (node->key.specified & AVTAB_ENABLED) && (node->key.specified & AVTAB_XPERMS)) services_compute_xperms_drivers(xperms, node); } -- cgit v1.2.3 From d4820ee1c7071966fa95543e60f9ed34ad7d47cf Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 25 Aug 2015 14:59:15 +0100 Subject: UPSTREAM: ARM: mm: improve do_ldrd_abort macro Improve the do_ldrd_abort macro code - firstly, it inefficiently checks for the LDRD encoding by doing a multi-stage test of various bits. This can be simplified by generating a mask, bitmasking the instruction and then comparing the result. Secondly, we want to be able to test the result rather than branching to do_DataAbort, so remove the branch at the end and rename the macro to 'teq_ldrd' to reflect it's new usage. teq_ldrd macro returns 'eq' if the instruction was a LDRD. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 08446b129bbde34665c423d882f857a45b8c3aed) Signed-off-by: Kees Cook Change-Id: If7b0db1bbdc7b4749c29346cad80baa8c1cf614f --- arch/arm/mm/abort-ev5t.S | 3 ++- arch/arm/mm/abort-ev5tj.S | 3 ++- arch/arm/mm/abort-ev6.S | 3 ++- arch/arm/mm/abort-macro.S | 13 +++++-------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/arch/arm/mm/abort-ev5t.S b/arch/arm/mm/abort-ev5t.S index a0908d4653a3..c913031b79cc 100644 --- a/arch/arm/mm/abort-ev5t.S +++ b/arch/arm/mm/abort-ev5t.S @@ -22,7 +22,8 @@ ENTRY(v5t_early_abort) do_thumb_abort fsr=r1, pc=r4, psr=r5, tmp=r3 ldreq r3, [r4] @ read aborted ARM instruction bic r1, r1, #1 << 11 @ clear bits 11 of FSR - do_ldrd_abort tmp=ip, insn=r3 + teq_ldrd tmp=ip, insn=r3 @ insn was LDRD? + beq do_DataAbort @ yes tst r3, #1 << 20 @ check write orreq r1, r1, #1 << 11 b do_DataAbort diff --git a/arch/arm/mm/abort-ev5tj.S b/arch/arm/mm/abort-ev5tj.S index 4006b7a61264..1b80d71adb0f 100644 --- a/arch/arm/mm/abort-ev5tj.S +++ b/arch/arm/mm/abort-ev5tj.S @@ -24,7 +24,8 @@ ENTRY(v5tj_early_abort) bne do_DataAbort do_thumb_abort fsr=r1, pc=r4, psr=r5, tmp=r3 ldreq r3, [r4] @ read aborted ARM instruction - do_ldrd_abort tmp=ip, insn=r3 + teq_ldrd tmp=ip, insn=r3 @ insn was LDRD? + beq do_DataAbort @ yes tst r3, #1 << 20 @ L = 0 -> write orreq r1, r1, #1 << 11 @ yes. b do_DataAbort diff --git a/arch/arm/mm/abort-ev6.S b/arch/arm/mm/abort-ev6.S index 8c48c5c22a33..113704f30e9f 100644 --- a/arch/arm/mm/abort-ev6.S +++ b/arch/arm/mm/abort-ev6.S @@ -34,7 +34,8 @@ ENTRY(v6_early_abort) ldr r3, [r4] @ read aborted ARM instruction ARM_BE8(rev r3, r3) - do_ldrd_abort tmp=ip, insn=r3 + teq_ldrd tmp=ip, insn=r3 @ insn was LDRD? + beq do_DataAbort @ yes tst r3, #1 << 20 @ L = 0 -> write orreq r1, r1, #1 << 11 @ yes. #endif diff --git a/arch/arm/mm/abort-macro.S b/arch/arm/mm/abort-macro.S index 2cbf68ef0e83..50d6c0a900b1 100644 --- a/arch/arm/mm/abort-macro.S +++ b/arch/arm/mm/abort-macro.S @@ -29,12 +29,9 @@ not_thumb: * [7:4] == 1101 * [20] == 0 */ - .macro do_ldrd_abort, tmp, insn - tst \insn, #0x0e100000 @ [27:25,20] == 0 - bne not_ldrd - and \tmp, \insn, #0x000000f0 @ [7:4] == 1101 - cmp \tmp, #0x000000d0 - beq do_DataAbort -not_ldrd: + .macro teq_ldrd, tmp, insn + mov \tmp, #0x0e100000 + orr \tmp, #0x000000f0 + and \tmp, \insn, \tmp + teq \tmp, #0x000000d0 .endm - -- cgit v1.2.3 From 278aaf4ed89b77d7c9fb2201bddda6fb987c1ebf Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 26 Aug 2015 20:07:25 +0100 Subject: UPSTREAM: ARM: entry: get rid of multiple macro definitions The following structure is just asking for trouble: #ifdef CONFIG_symbol .macro foo ... .endm .macro bar ... .endm .macro baz ... .endm #else .macro foo ... .endm .macro bar ... .endm #ifdef CONFIG_symbol2 .macro baz ... .endm #else .macro baz ... .endm #endif #endif such as one defintion being updated, but the other definitions miss out. Where the contents of a macro needs to be conditional, the hint is in the first clause of this very sentence. "contents" "conditional". Not multiple separate definitions, especially not when much of the macro is the same between different configs. This patch fixes this bad style, which had caused the Thumb2 code to miss-out on the uaccess updates. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit aa06e5c1f9c2b466712be904cc5b56a813e24cfd) Signed-off-by: Kees Cook Change-Id: Ie9d35de00dbae1451157259f7551af388cd37d3b --- arch/arm/kernel/entry-header.S | 109 +++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 65 deletions(-) diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 1a0045abead7..d47b5161b029 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -196,7 +196,7 @@ msr cpsr_c, \rtemp @ switch back to the SVC mode .endm -#ifndef CONFIG_THUMB2_KERNEL + .macro svc_exit, rpsr, irq = 0 .if \irq != 0 @ IRQs already off @@ -215,6 +215,9 @@ blne trace_hardirqs_off #endif .endif + +#ifndef CONFIG_THUMB2_KERNEL + @ ARM mode SVC restore msr spsr_cxsf, \rpsr #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K) @ We must avoid clrex due to Cortex-A15 erratum #830321 @@ -222,6 +225,20 @@ strex r1, r2, [r0] @ clear the exclusive monitor #endif ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr +#else + @ Thumb mode SVC restore + ldr lr, [sp, #S_SP] @ top of the stack + ldrd r0, r1, [sp, #S_LR] @ calling lr and pc + + @ We must avoid clrex due to Cortex-A15 erratum #830321 + strex r2, r1, [sp, #S_LR] @ clear the exclusive monitor + + stmdb lr!, {r0, r1, \rpsr} @ calling lr and rfe context + ldmia sp, {r0 - r12} + mov sp, lr + ldr lr, [sp], #4 + rfeia sp! +#endif .endm @ @@ -241,6 +258,8 @@ @ on the stack remains correct). @ .macro svc_exit_via_fiq +#ifndef CONFIG_THUMB2_KERNEL + @ ARM mode restore mov r0, sp ldmib r0, {r1 - r14} @ abort is deadly from here onward (it will @ clobber state restored below) @@ -250,9 +269,26 @@ msr spsr_cxsf, r9 ldr r0, [r0, #S_R0] ldmia r8, {pc}^ +#else + @ Thumb mode restore + add r0, sp, #S_R2 + ldr lr, [sp, #S_LR] + ldr sp, [sp, #S_SP] @ abort is deadly from here onward (it will + @ clobber state restored below) + ldmia r0, {r2 - r12} + mov r1, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT + msr cpsr_c, r1 + sub r0, #S_R2 + add r8, r0, #S_PC + ldmia r0, {r0 - r1} + rfeia r8 +#endif .endm + .macro restore_user_regs, fast = 0, offset = 0 +#ifndef CONFIG_THUMB2_KERNEL + @ ARM mode restore mov r2, sp ldr r1, [r2, #\offset + S_PSR] @ get calling cpsr ldr lr, [r2, #\offset + S_PC]! @ get pc @@ -270,72 +306,16 @@ @ after ldm {}^ add sp, sp, #\offset + S_FRAME_SIZE movs pc, lr @ return & move spsr_svc into cpsr - .endm - -#else /* CONFIG_THUMB2_KERNEL */ - .macro svc_exit, rpsr, irq = 0 - .if \irq != 0 - @ IRQs already off -#ifdef CONFIG_TRACE_IRQFLAGS - @ The parent context IRQs must have been enabled to get here in - @ the first place, so there's no point checking the PSR I bit. - bl trace_hardirqs_on -#endif - .else - @ IRQs off again before pulling preserved data off the stack - disable_irq_notrace -#ifdef CONFIG_TRACE_IRQFLAGS - tst \rpsr, #PSR_I_BIT - bleq trace_hardirqs_on - tst \rpsr, #PSR_I_BIT - blne trace_hardirqs_off -#endif - .endif - ldr lr, [sp, #S_SP] @ top of the stack - ldrd r0, r1, [sp, #S_LR] @ calling lr and pc - - @ We must avoid clrex due to Cortex-A15 erratum #830321 - strex r2, r1, [sp, #S_LR] @ clear the exclusive monitor - - stmdb lr!, {r0, r1, \rpsr} @ calling lr and rfe context - ldmia sp, {r0 - r12} - mov sp, lr - ldr lr, [sp], #4 - rfeia sp! - .endm - - @ - @ svc_exit_via_fiq - like svc_exit but switches to FIQ mode before exit - @ - @ For full details see non-Thumb implementation above. - @ - .macro svc_exit_via_fiq - add r0, sp, #S_R2 - ldr lr, [sp, #S_LR] - ldr sp, [sp, #S_SP] @ abort is deadly from here onward (it will - @ clobber state restored below) - ldmia r0, {r2 - r12} - mov r1, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT - msr cpsr_c, r1 - sub r0, #S_R2 - add r8, r0, #S_PC - ldmia r0, {r0 - r1} - rfeia r8 - .endm - -#ifdef CONFIG_CPU_V7M - /* - * Note we don't need to do clrex here as clearing the local monitor is - * part of each exception entry and exit sequence. - */ - .macro restore_user_regs, fast = 0, offset = 0 +#elif defined(CONFIG_CPU_V7M) + @ V7M restore. + @ Note that we don't need to do clrex here as clearing the local + @ monitor is part of the exception entry and exit sequence. .if \offset add sp, #\offset .endif v7m_exception_slow_exit ret_r0 = \fast - .endm -#else /* ifdef CONFIG_CPU_V7M */ - .macro restore_user_regs, fast = 0, offset = 0 +#else + @ Thumb mode restore mov r2, sp load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr @@ -353,9 +333,8 @@ .endif add sp, sp, #S_FRAME_SIZE - S_SP movs pc, lr @ return & move spsr_svc into cpsr - .endm -#endif /* ifdef CONFIG_CPU_V7M / else */ #endif /* !CONFIG_THUMB2_KERNEL */ + .endm /* * Context tracking subsystem. Used to instrument transitions -- cgit v1.2.3 From 7e2ab218b8a652cb364e0954c728fdcc33e09a53 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 20 Aug 2015 10:32:02 +0100 Subject: UPSTREAM: ARM: entry: provide uaccess assembly macro hooks Provide hooks into the kernel entry and exit paths to permit control of userspace visibility to the kernel. The intended use is: - on entry to kernel from user, uaccess_disable will be called to disable userspace visibility - on exit from kernel to user, uaccess_enable will be called to enable userspace visibility - on entry from a kernel exception, uaccess_save_and_disable will be called to save the current userspace visibility setting, and disable access - on exit from a kernel exception, uaccess_restore will be called to restore the userspace visibility as it was before the exception occurred. These hooks allows us to keep userspace visibility disabled for the vast majority of the kernel, except for localised regions where we want to explicitly access userspace. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 2190fed67ba6f3e8129513929f2395843645e928) Signed-off-by: Kees Cook Change-Id: I167ccd396ff29f87d15ea910f55f9d0b65c9a1b2 --- arch/arm/include/asm/assembler.h | 17 +++++++++++++++++ arch/arm/kernel/entry-armv.S | 30 ++++++++++++++++++++++-------- arch/arm/kernel/entry-common.S | 2 ++ arch/arm/kernel/entry-header.S | 3 +++ arch/arm/mm/abort-ev4.S | 1 + arch/arm/mm/abort-ev5t.S | 1 + arch/arm/mm/abort-ev5tj.S | 1 + arch/arm/mm/abort-ev6.S | 7 ++++--- arch/arm/mm/abort-ev7.S | 1 + arch/arm/mm/abort-lv4t.S | 2 ++ arch/arm/mm/abort-macro.S | 1 + 11 files changed, 55 insertions(+), 11 deletions(-) diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 186270b3e194..4fdb15e898ad 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -430,6 +430,23 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) #endif .endm + .macro uaccess_disable, tmp, isb=1 + .endm + + .macro uaccess_enable, tmp, isb=1 + .endm + + .macro uaccess_save, tmp + .endm + + .macro uaccess_restore + .endm + + .macro uaccess_save_and_disable, tmp + uaccess_save \tmp + uaccess_disable \tmp + .endm + .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo .macro ret\c, reg #if __LINUX_ARM_ARCH__ < 6 diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 570306c49406..b953d72ef514 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -147,10 +147,10 @@ ENDPROC(__und_invalid) #define SPFIX(code...) #endif - .macro svc_entry, stack_hole=0, trace=1 + .macro svc_entry, stack_hole=0, trace=1, uaccess=1 UNWIND(.fnstart ) UNWIND(.save {r0 - pc} ) - sub sp, sp, #(S_FRAME_SIZE + \stack_hole - 4) + sub sp, sp, #(S_FRAME_SIZE + 8 + \stack_hole - 4) #ifdef CONFIG_THUMB2_KERNEL SPFIX( str r0, [sp] ) @ temporarily saved SPFIX( mov r0, sp ) @@ -165,7 +165,7 @@ ENDPROC(__und_invalid) ldmia r0, {r3 - r5} add r7, sp, #S_SP - 4 @ here for interlock avoidance mov r6, #-1 @ "" "" "" "" - add r2, sp, #(S_FRAME_SIZE + \stack_hole - 4) + add r2, sp, #(S_FRAME_SIZE + 8 + \stack_hole - 4) SPFIX( addeq r2, r2, #4 ) str r3, [sp, #-4]! @ save the "real" r0 copied @ from the exception stack @@ -183,6 +183,11 @@ ENDPROC(__und_invalid) @ stmia r7, {r2 - r6} + uaccess_save r0 + .if \uaccess + uaccess_disable r0 + .endif + .if \trace #ifdef CONFIG_TRACE_IRQFLAGS bl trace_hardirqs_off @@ -192,7 +197,7 @@ ENDPROC(__und_invalid) .align 5 __dabt_svc: - svc_entry + svc_entry uaccess=0 mov r2, sp dabt_helper THUMB( ldr r5, [sp, #S_PSR] ) @ potentially updated CPSR @@ -366,7 +371,7 @@ ENDPROC(__fiq_abt) #error "sizeof(struct pt_regs) must be a multiple of 8" #endif - .macro usr_entry, trace=1 + .macro usr_entry, trace=1, uaccess=1 UNWIND(.fnstart ) UNWIND(.cantunwind ) @ don't unwind the user space sub sp, sp, #S_FRAME_SIZE @@ -398,6 +403,10 @@ ENDPROC(__fiq_abt) ARM( stmdb r0, {sp, lr}^ ) THUMB( store_user_sp_lr r0, r1, S_SP - S_PC ) + .if \uaccess + uaccess_disable ip + .endif + @ Enable the alignment trap while in kernel mode ATRAP( teq r8, r7) ATRAP( mcrne p15, 0, r8, c1, c0, 0) @@ -433,7 +442,7 @@ ENDPROC(__fiq_abt) .align 5 __dabt_usr: - usr_entry + usr_entry uaccess=0 kuser_cmpxchg_check mov r2, sp dabt_helper @@ -456,7 +465,7 @@ ENDPROC(__irq_usr) .align 5 __und_usr: - usr_entry + usr_entry uaccess=0 mov r2, r4 mov r3, r5 @@ -482,6 +491,8 @@ __und_usr: 1: ldrt r0, [r4] ARM_BE8(rev r0, r0) @ little endian instruction + uaccess_disable ip + @ r0 = 32-bit ARM instruction which caused the exception @ r2 = PC value for the following instruction (:= regs->ARM_pc) @ r4 = PC value for the faulting instruction @@ -516,9 +527,10 @@ __und_usr_thumb: 2: ldrht r5, [r4] ARM_BE8(rev16 r5, r5) @ little endian instruction cmp r5, #0xe800 @ 32bit instruction if xx != 0 - blo __und_usr_fault_16 @ 16bit undefined instruction + blo __und_usr_fault_16_pan @ 16bit undefined instruction 3: ldrht r0, [r2] ARM_BE8(rev16 r0, r0) @ little endian instruction + uaccess_disable ip add r2, r2, #2 @ r2 is PC + 2, make it PC + 4 str r2, [sp, #S_PC] @ it's a 2x16bit instr, update orr r0, r0, r5, lsl #16 @@ -713,6 +725,8 @@ ENDPROC(no_fp) __und_usr_fault_32: mov r1, #4 b 1f +__und_usr_fault_16_pan: + uaccess_disable ip __und_usr_fault_16: mov r1, #2 1: mov r0, sp diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 4e7f40c577e6..617b8de8b747 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -173,6 +173,8 @@ ENTRY(vector_swi) USER( ldr scno, [lr, #-4] ) @ get SWI instruction #endif + uaccess_disable tbl + adr tbl, sys_call_table @ load syscall table pointer #if defined(CONFIG_OABI_COMPAT) diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index d47b5161b029..0d22ad206d52 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -215,6 +215,7 @@ blne trace_hardirqs_off #endif .endif + uaccess_restore #ifndef CONFIG_THUMB2_KERNEL @ ARM mode SVC restore @@ -258,6 +259,7 @@ @ on the stack remains correct). @ .macro svc_exit_via_fiq + uaccess_restore #ifndef CONFIG_THUMB2_KERNEL @ ARM mode restore mov r0, sp @@ -287,6 +289,7 @@ .macro restore_user_regs, fast = 0, offset = 0 + uaccess_enable r1, isb=0 #ifndef CONFIG_THUMB2_KERNEL @ ARM mode restore mov r2, sp diff --git a/arch/arm/mm/abort-ev4.S b/arch/arm/mm/abort-ev4.S index 54473cd4aba9..b3b31e30cadd 100644 --- a/arch/arm/mm/abort-ev4.S +++ b/arch/arm/mm/abort-ev4.S @@ -19,6 +19,7 @@ ENTRY(v4_early_abort) mrc p15, 0, r1, c5, c0, 0 @ get FSR mrc p15, 0, r0, c6, c0, 0 @ get FAR ldr r3, [r4] @ read aborted ARM instruction + uaccess_disable ip @ disable userspace access bic r1, r1, #1 << 11 | 1 << 10 @ clear bits 11 and 10 of FSR tst r3, #1 << 20 @ L = 1 -> write? orreq r1, r1, #1 << 11 @ yes. diff --git a/arch/arm/mm/abort-ev5t.S b/arch/arm/mm/abort-ev5t.S index c913031b79cc..a6a381a6caa5 100644 --- a/arch/arm/mm/abort-ev5t.S +++ b/arch/arm/mm/abort-ev5t.S @@ -21,6 +21,7 @@ ENTRY(v5t_early_abort) mrc p15, 0, r0, c6, c0, 0 @ get FAR do_thumb_abort fsr=r1, pc=r4, psr=r5, tmp=r3 ldreq r3, [r4] @ read aborted ARM instruction + uaccess_disable ip @ disable user access bic r1, r1, #1 << 11 @ clear bits 11 of FSR teq_ldrd tmp=ip, insn=r3 @ insn was LDRD? beq do_DataAbort @ yes diff --git a/arch/arm/mm/abort-ev5tj.S b/arch/arm/mm/abort-ev5tj.S index 1b80d71adb0f..00ab011bef58 100644 --- a/arch/arm/mm/abort-ev5tj.S +++ b/arch/arm/mm/abort-ev5tj.S @@ -24,6 +24,7 @@ ENTRY(v5tj_early_abort) bne do_DataAbort do_thumb_abort fsr=r1, pc=r4, psr=r5, tmp=r3 ldreq r3, [r4] @ read aborted ARM instruction + uaccess_disable ip @ disable userspace access teq_ldrd tmp=ip, insn=r3 @ insn was LDRD? beq do_DataAbort @ yes tst r3, #1 << 20 @ L = 0 -> write diff --git a/arch/arm/mm/abort-ev6.S b/arch/arm/mm/abort-ev6.S index 113704f30e9f..8801a15aa105 100644 --- a/arch/arm/mm/abort-ev6.S +++ b/arch/arm/mm/abort-ev6.S @@ -26,17 +26,18 @@ ENTRY(v6_early_abort) ldr ip, =0x4107b36 mrc p15, 0, r3, c0, c0, 0 @ get processor id teq ip, r3, lsr #4 @ r0 ARM1136? - bne do_DataAbort + bne 1f tst r5, #PSR_J_BIT @ Java? tsteq r5, #PSR_T_BIT @ Thumb? - bne do_DataAbort + bne 1f bic r1, r1, #1 << 11 @ clear bit 11 of FSR ldr r3, [r4] @ read aborted ARM instruction ARM_BE8(rev r3, r3) teq_ldrd tmp=ip, insn=r3 @ insn was LDRD? - beq do_DataAbort @ yes + beq 1f @ yes tst r3, #1 << 20 @ L = 0 -> write orreq r1, r1, #1 << 11 @ yes. #endif +1: uaccess_disable ip @ disable userspace access b do_DataAbort diff --git a/arch/arm/mm/abort-ev7.S b/arch/arm/mm/abort-ev7.S index 4812ad054214..e8d0e08c227f 100644 --- a/arch/arm/mm/abort-ev7.S +++ b/arch/arm/mm/abort-ev7.S @@ -15,6 +15,7 @@ ENTRY(v7_early_abort) mrc p15, 0, r1, c5, c0, 0 @ get FSR mrc p15, 0, r0, c6, c0, 0 @ get FAR + uaccess_disable ip @ disable userspace access /* * V6 code adjusts the returned DFSR. diff --git a/arch/arm/mm/abort-lv4t.S b/arch/arm/mm/abort-lv4t.S index f3982580c273..6d8e8e3365d1 100644 --- a/arch/arm/mm/abort-lv4t.S +++ b/arch/arm/mm/abort-lv4t.S @@ -26,6 +26,7 @@ ENTRY(v4t_late_abort) #endif bne .data_thumb_abort ldr r8, [r4] @ read arm instruction + uaccess_disable ip @ disable userspace access tst r8, #1 << 20 @ L = 1 -> write? orreq r1, r1, #1 << 11 @ yes. and r7, r8, #15 << 24 @@ -155,6 +156,7 @@ ENTRY(v4t_late_abort) .data_thumb_abort: ldrh r8, [r4] @ read instruction + uaccess_disable ip @ disable userspace access tst r8, #1 << 11 @ L = 1 -> write? orreq r1, r1, #1 << 8 @ yes and r7, r8, #15 << 12 diff --git a/arch/arm/mm/abort-macro.S b/arch/arm/mm/abort-macro.S index 50d6c0a900b1..4509bee4e081 100644 --- a/arch/arm/mm/abort-macro.S +++ b/arch/arm/mm/abort-macro.S @@ -13,6 +13,7 @@ tst \psr, #PSR_T_BIT beq not_thumb ldrh \tmp, [\pc] @ Read aborted Thumb instruction + uaccess_disable ip @ disable userspace access and \tmp, \tmp, # 0xfe00 @ Mask opcode field cmp \tmp, # 0x5600 @ Is it ldrsb? orreq \tmp, \tmp, #1 << 11 @ Set L-bit if yes -- cgit v1.2.3 From fdc83cef58776960501c800dac1cd07dd0c7d52d Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 18 Aug 2015 23:06:25 +0100 Subject: UPSTREAM: ARM: uaccess: simplify user access assembly The user assembly for byte and word accesses was virtually identical. Rather than duplicating this, use a macro instead. Acked-by: Will Deacon Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit b64d1f66517a89b9b0f6bd0bca86b05a55a5e742) Signed-off-by: Kees Cook Change-Id: Ica8078048d18285a7adaffc7825a0ef5ab817ac8 --- arch/arm/include/asm/uaccess.h | 47 +++++++++++------------------------------- 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 74b17d09ef7a..4cf54ebe408a 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -311,9 +311,9 @@ do { \ (x) = (__typeof__(*(ptr)))__gu_val; \ } while (0) -#define __get_user_asm_byte(x, addr, err) \ +#define __get_user_asm(x, addr, err, instr) \ __asm__ __volatile__( \ - "1: " TUSER(ldrb) " %1,[%2],#0\n" \ + "1: " TUSER(instr) " %1, [%2], #0\n" \ "2:\n" \ " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ @@ -329,6 +329,9 @@ do { \ : "r" (addr), "i" (-EFAULT) \ : "cc") +#define __get_user_asm_byte(x, addr, err) \ + __get_user_asm(x, addr, err, ldrb) + #ifndef __ARMEB__ #define __get_user_asm_half(x, __gu_addr, err) \ ({ \ @@ -348,22 +351,7 @@ do { \ #endif #define __get_user_asm_word(x, addr, err) \ - __asm__ __volatile__( \ - "1: " TUSER(ldr) " %1,[%2],#0\n" \ - "2:\n" \ - " .pushsection .text.fixup,\"ax\"\n" \ - " .align 2\n" \ - "3: mov %0, %3\n" \ - " mov %1, #0\n" \ - " b 2b\n" \ - " .popsection\n" \ - " .pushsection __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 3b\n" \ - " .popsection" \ - : "+r" (err), "=&r" (x) \ - : "r" (addr), "i" (-EFAULT) \ - : "cc") + __get_user_asm(x, addr, err, ldr) #define __put_user(x, ptr) \ ({ \ @@ -393,9 +381,9 @@ do { \ } \ } while (0) -#define __put_user_asm_byte(x, __pu_addr, err) \ +#define __put_user_asm(x, __pu_addr, err, instr) \ __asm__ __volatile__( \ - "1: " TUSER(strb) " %1,[%2],#0\n" \ + "1: " TUSER(instr) " %1, [%2], #0\n" \ "2:\n" \ " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ @@ -410,6 +398,9 @@ do { \ : "r" (x), "r" (__pu_addr), "i" (-EFAULT) \ : "cc") +#define __put_user_asm_byte(x, __pu_addr, err) \ + __put_user_asm(x, __pu_addr, err, strb) + #ifndef __ARMEB__ #define __put_user_asm_half(x, __pu_addr, err) \ ({ \ @@ -427,21 +418,7 @@ do { \ #endif #define __put_user_asm_word(x, __pu_addr, err) \ - __asm__ __volatile__( \ - "1: " TUSER(str) " %1,[%2],#0\n" \ - "2:\n" \ - " .pushsection .text.fixup,\"ax\"\n" \ - " .align 2\n" \ - "3: mov %0, %3\n" \ - " b 2b\n" \ - " .popsection\n" \ - " .pushsection __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 3b\n" \ - " .popsection" \ - : "+r" (err) \ - : "r" (x), "r" (__pu_addr), "i" (-EFAULT) \ - : "cc") + __put_user_asm(x, __pu_addr, err, str) #ifndef __ARMEB__ #define __reg_oper0 "%R2" -- cgit v1.2.3 From 1d2d8b218624c388338aff4fd2feadb4bcb2c7c8 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 11 May 2015 17:52:15 +0200 Subject: UPSTREAM: sched/preempt, arm/futex: Disable preemption in UP futex_atomic_cmpxchg_inatomic() explicitly The !CONFIG_SMP implementation of futex_atomic_cmpxchg_inatomic() requires preemption to be disabled to guarantee mutual exclusion. Let's make this explicit. This patch is based on a patch by Sebastian Andrzej Siewior on the -rt branch. Reviewed-and-tested-by: Thomas Gleixner Signed-off-by: David Hildenbrand Signed-off-by: Peter Zijlstra (Intel) Cc: David.Laight@ACULAB.COM Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: airlied@linux.ie Cc: akpm@linux-foundation.org Cc: benh@kernel.crashing.org Cc: bigeasy@linutronix.de Cc: borntraeger@de.ibm.com Cc: daniel.vetter@intel.com Cc: heiko.carstens@de.ibm.com Cc: herbert@gondor.apana.org.au Cc: hocko@suse.cz Cc: hughd@google.com Cc: mst@redhat.com Cc: paulus@samba.org Cc: ralf@linux-mips.org Cc: schwidefsky@de.ibm.com Cc: yang.shi@windriver.com Link: http://lkml.kernel.org/r/1431359540-32227-11-git-send-email-dahi@linux.vnet.ibm.com Signed-off-by: Ingo Molnar Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 39919b01ae4c1949736b40b79e27178d0c0bc406) Signed-off-by: Kees Cook Change-Id: Iaf1f4cb9b81607be9835ee8bb53d11cec36089d7 --- arch/arm/include/asm/futex.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index 4e78065a16aa..255bfd15eda9 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -93,6 +93,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; + preempt_disable(); __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" "1: " TUSER(ldr) " %1, [%4]\n" " teq %1, %2\n" @@ -104,6 +105,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, : "cc", "memory"); *uval = val; + preempt_enable(); + return ret; } -- cgit v1.2.3 From 9b980da7c20bf852483f7d1324fa95fa4e9d19c8 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Mon, 11 May 2015 17:52:16 +0200 Subject: UPSTREAM: sched/preempt, arm/futex: Disable preemption in UP futex_atomic_op_inuser() explicitly The !CONFIG_SMP implementation of futex_atomic_op_inuser() seems to rely on disabled preemption to guarantee mutual exclusion. From commit e589ed23dd27 ("[ARM] 5218/1: arm: improved futex support"): "For UP it's enough to disable preemption to ensure mutual exclusion..." From the code itself: "!SMP, we can work around lack of atomic ops by disabling preemption" Let's make this explicit, to prepare for pagefault_disable() not touching preemption anymore. Reviewed-and-tested-by: Thomas Gleixner Signed-off-by: David Hildenbrand Signed-off-by: Peter Zijlstra (Intel) Cc: David.Laight@ACULAB.COM Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: airlied@linux.ie Cc: akpm@linux-foundation.org Cc: benh@kernel.crashing.org Cc: bigeasy@linutronix.de Cc: borntraeger@de.ibm.com Cc: daniel.vetter@intel.com Cc: heiko.carstens@de.ibm.com Cc: herbert@gondor.apana.org.au Cc: hocko@suse.cz Cc: hughd@google.com Cc: mst@redhat.com Cc: paulus@samba.org Cc: ralf@linux-mips.org Cc: schwidefsky@de.ibm.com Cc: yang.shi@windriver.com Link: http://lkml.kernel.org/r/1431359540-32227-12-git-send-email-dahi@linux.vnet.ibm.com Signed-off-by: Ingo Molnar Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 388b0e0adbc98a1b12a077dc92851a3ce016db42) Signed-off-by: Kees Cook Change-Id: I6a50b38852b18d90315040fa8fd66ea8cc0f3aa7 --- arch/arm/include/asm/futex.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index 255bfd15eda9..5eed82809d82 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -127,7 +127,10 @@ futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; - pagefault_disable(); /* implies preempt_disable() */ +#ifndef CONFIG_SMP + preempt_disable(); +#endif + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -149,7 +152,10 @@ futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); /* subsumes preempt_enable() */ + pagefault_enable(); +#ifndef CONFIG_SMP + preempt_enable(); +#endif if (!ret) { switch (cmp) { -- cgit v1.2.3 From 591eba9c503f592b4b3467bd65391d6bc4676c7e Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 19 Aug 2015 11:02:28 +0100 Subject: UPSTREAM: ARM: uaccess: provide uaccess_save_and_enable() and uaccess_restore() Provide uaccess_save_and_enable() and uaccess_restore() to permit control of userspace visibility to the kernel, and hook these into the appropriate places in the kernel where we need to access userspace. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 3fba7e23f754a9a6e639b640fa2a393712ffe1b8) Signed-off-by: Kees Cook Change-Id: I0a43fd4f18edb95b47dc1ec622ec37bea8018663 --- arch/arm/include/asm/futex.h | 19 ++++++++-- arch/arm/include/asm/uaccess.h | 71 +++++++++++++++++++++++++++++++++++--- arch/arm/kernel/armksyms.c | 6 ++-- arch/arm/lib/clear_user.S | 6 ++-- arch/arm/lib/copy_from_user.S | 6 ++-- arch/arm/lib/copy_to_user.S | 6 ++-- arch/arm/lib/uaccess_with_memcpy.c | 4 +-- 7 files changed, 97 insertions(+), 21 deletions(-) diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index 5eed82809d82..6795368ad023 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -22,8 +22,11 @@ #ifdef CONFIG_SMP #define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg) \ +({ \ + unsigned int __ua_flags; \ smp_mb(); \ prefetchw(uaddr); \ + __ua_flags = uaccess_save_and_enable(); \ __asm__ __volatile__( \ "1: ldrex %1, [%3]\n" \ " " insn "\n" \ @@ -34,12 +37,15 @@ __futex_atomic_ex_table("%5") \ : "=&r" (ret), "=&r" (oldval), "=&r" (tmp) \ : "r" (uaddr), "r" (oparg), "Ir" (-EFAULT) \ - : "cc", "memory") + : "cc", "memory"); \ + uaccess_restore(__ua_flags); \ +}) static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { + unsigned int __ua_flags; int ret; u32 val; @@ -49,6 +55,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, smp_mb(); /* Prefetching cannot fault */ prefetchw(uaddr); + __ua_flags = uaccess_save_and_enable(); __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" "1: ldrex %1, [%4]\n" " teq %1, %2\n" @@ -61,6 +68,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, : "=&r" (ret), "=&r" (val) : "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT) : "cc", "memory"); + uaccess_restore(__ua_flags); smp_mb(); *uval = val; @@ -73,6 +81,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, #include #define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg) \ +({ \ + unsigned int __ua_flags = uaccess_save_and_enable(); \ __asm__ __volatile__( \ "1: " TUSER(ldr) " %1, [%3]\n" \ " " insn "\n" \ @@ -81,12 +91,15 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, __futex_atomic_ex_table("%5") \ : "=&r" (ret), "=&r" (oldval), "=&r" (tmp) \ : "r" (uaddr), "r" (oparg), "Ir" (-EFAULT) \ - : "cc", "memory") + : "cc", "memory"); \ + uaccess_restore(__ua_flags); \ +}) static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { + unsigned int __ua_flags; int ret = 0; u32 val; @@ -94,6 +107,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, return -EFAULT; preempt_disable(); + __ua_flags = uaccess_save_and_enable(); __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" "1: " TUSER(ldr) " %1, [%4]\n" " teq %1, %2\n" @@ -103,6 +117,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, : "+r" (ret), "=&r" (val) : "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT) : "cc", "memory"); + uaccess_restore(__ua_flags); *uval = val; preempt_enable(); diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 4cf54ebe408a..2580eabf7706 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -49,6 +49,21 @@ struct exception_table_entry extern int fixup_exception(struct pt_regs *regs); +/* + * These two functions allow hooking accesses to userspace to increase + * system integrity by ensuring that the kernel can not inadvertantly + * perform such accesses (eg, via list poison values) which could then + * be exploited for priviledge escalation. + */ +static inline unsigned int uaccess_save_and_enable(void) +{ + return 0; +} + +static inline void uaccess_restore(unsigned int flags) +{ +} + /* * These two are intentionally not defined anywhere - if the kernel * code generates any references to them, that's a bug. @@ -165,6 +180,7 @@ extern int __get_user_64t_4(void *); register typeof(x) __r2 asm("r2"); \ register unsigned long __l asm("r1") = __limit; \ register int __e asm("r0"); \ + unsigned int __ua_flags = uaccess_save_and_enable(); \ switch (sizeof(*(__p))) { \ case 1: \ if (sizeof((x)) >= 8) \ @@ -192,6 +208,7 @@ extern int __get_user_64t_4(void *); break; \ default: __e = __get_user_bad(); break; \ } \ + uaccess_restore(__ua_flags); \ x = (typeof(*(p))) __r2; \ __e; \ }) @@ -224,6 +241,7 @@ extern int __put_user_8(void *, unsigned long long); register const typeof(*(p)) __user *__p asm("r0") = __tmp_p; \ register unsigned long __l asm("r1") = __limit; \ register int __e asm("r0"); \ + unsigned int __ua_flags = uaccess_save_and_enable(); \ switch (sizeof(*(__p))) { \ case 1: \ __put_user_x(__r2, __p, __e, __l, 1); \ @@ -239,6 +257,7 @@ extern int __put_user_8(void *, unsigned long long); break; \ default: __e = __put_user_bad(); break; \ } \ + uaccess_restore(__ua_flags); \ __e; \ }) @@ -300,14 +319,17 @@ static inline void set_fs(mm_segment_t fs) do { \ unsigned long __gu_addr = (unsigned long)(ptr); \ unsigned long __gu_val; \ + unsigned int __ua_flags; \ __chk_user_ptr(ptr); \ might_fault(); \ + __ua_flags = uaccess_save_and_enable(); \ switch (sizeof(*(ptr))) { \ case 1: __get_user_asm_byte(__gu_val, __gu_addr, err); break; \ case 2: __get_user_asm_half(__gu_val, __gu_addr, err); break; \ case 4: __get_user_asm_word(__gu_val, __gu_addr, err); break; \ default: (__gu_val) = __get_user_bad(); \ } \ + uaccess_restore(__ua_flags); \ (x) = (__typeof__(*(ptr)))__gu_val; \ } while (0) @@ -369,9 +391,11 @@ do { \ #define __put_user_err(x, ptr, err) \ do { \ unsigned long __pu_addr = (unsigned long)(ptr); \ + unsigned int __ua_flags; \ __typeof__(*(ptr)) __pu_val = (x); \ __chk_user_ptr(ptr); \ might_fault(); \ + __ua_flags = uaccess_save_and_enable(); \ switch (sizeof(*(ptr))) { \ case 1: __put_user_asm_byte(__pu_val, __pu_addr, err); break; \ case 2: __put_user_asm_half(__pu_val, __pu_addr, err); break; \ @@ -379,6 +403,7 @@ do { \ case 8: __put_user_asm_dword(__pu_val, __pu_addr, err); break; \ default: __put_user_bad(); \ } \ + uaccess_restore(__ua_flags); \ } while (0) #define __put_user_asm(x, __pu_addr, err, instr) \ @@ -451,11 +476,46 @@ do { \ #ifdef CONFIG_MMU -extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n); -extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n); -extern unsigned long __must_check __copy_to_user_std(void __user *to, const void *from, unsigned long n); -extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n); -extern unsigned long __must_check __clear_user_std(void __user *addr, unsigned long n); +extern unsigned long __must_check +arm_copy_from_user(void *to, const void __user *from, unsigned long n); + +static inline unsigned long __must_check +__copy_from_user(void *to, const void __user *from, unsigned long n) +{ + unsigned int __ua_flags = uaccess_save_and_enable(); + n = arm_copy_from_user(to, from, n); + uaccess_restore(__ua_flags); + return n; +} + +extern unsigned long __must_check +arm_copy_to_user(void __user *to, const void *from, unsigned long n); +extern unsigned long __must_check +__copy_to_user_std(void __user *to, const void *from, unsigned long n); + +static inline unsigned long __must_check +__copy_to_user(void __user *to, const void *from, unsigned long n) +{ + unsigned int __ua_flags = uaccess_save_and_enable(); + n = arm_copy_to_user(to, from, n); + uaccess_restore(__ua_flags); + return n; +} + +extern unsigned long __must_check +arm_clear_user(void __user *addr, unsigned long n); +extern unsigned long __must_check +__clear_user_std(void __user *addr, unsigned long n); + +static inline unsigned long __must_check +__clear_user(void __user *addr, unsigned long n) +{ + unsigned int __ua_flags = uaccess_save_and_enable(); + n = arm_clear_user(addr, n); + uaccess_restore(__ua_flags); + return n; +} + #else #define __copy_from_user(to, from, n) (memcpy(to, (void __force *)from, n), 0) #define __copy_to_user(to, from, n) (memcpy((void __force *)to, from, n), 0) @@ -488,6 +548,7 @@ static inline unsigned long __must_check clear_user(void __user *to, unsigned lo return n; } +/* These are from lib/ code, and use __get_user() and friends */ extern long strncpy_from_user(char *dest, const char __user *src, long count); extern __must_check long strlen_user(const char __user *str); diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c index a88671cfe1ff..a35d72d30b56 100644 --- a/arch/arm/kernel/armksyms.c +++ b/arch/arm/kernel/armksyms.c @@ -91,9 +91,9 @@ EXPORT_SYMBOL(__memzero); #ifdef CONFIG_MMU EXPORT_SYMBOL(copy_page); -EXPORT_SYMBOL(__copy_from_user); -EXPORT_SYMBOL(__copy_to_user); -EXPORT_SYMBOL(__clear_user); +EXPORT_SYMBOL(arm_copy_from_user); +EXPORT_SYMBOL(arm_copy_to_user); +EXPORT_SYMBOL(arm_clear_user); EXPORT_SYMBOL(__get_user_1); EXPORT_SYMBOL(__get_user_2); diff --git a/arch/arm/lib/clear_user.S b/arch/arm/lib/clear_user.S index 1710fd7db2d5..970d6c043774 100644 --- a/arch/arm/lib/clear_user.S +++ b/arch/arm/lib/clear_user.S @@ -12,14 +12,14 @@ .text -/* Prototype: int __clear_user(void *addr, size_t sz) +/* Prototype: unsigned long arm_clear_user(void *addr, size_t sz) * Purpose : clear some user memory * Params : addr - user memory address to clear * : sz - number of bytes to clear * Returns : number of bytes NOT cleared */ ENTRY(__clear_user_std) -WEAK(__clear_user) +WEAK(arm_clear_user) stmfd sp!, {r1, lr} mov r2, #0 cmp r1, #4 @@ -44,7 +44,7 @@ WEAK(__clear_user) USER( strnebt r2, [r0]) mov r0, #0 ldmfd sp!, {r1, pc} -ENDPROC(__clear_user) +ENDPROC(arm_clear_user) ENDPROC(__clear_user_std) .pushsection .text.fixup,"ax" diff --git a/arch/arm/lib/copy_from_user.S b/arch/arm/lib/copy_from_user.S index 7a235b9952be..1512bebfbf1b 100644 --- a/arch/arm/lib/copy_from_user.S +++ b/arch/arm/lib/copy_from_user.S @@ -17,7 +17,7 @@ /* * Prototype: * - * size_t __copy_from_user(void *to, const void *from, size_t n) + * size_t arm_copy_from_user(void *to, const void *from, size_t n) * * Purpose: * @@ -89,11 +89,11 @@ .text -ENTRY(__copy_from_user) +ENTRY(arm_copy_from_user) #include "copy_template.S" -ENDPROC(__copy_from_user) +ENDPROC(arm_copy_from_user) .pushsection .fixup,"ax" .align 0 diff --git a/arch/arm/lib/copy_to_user.S b/arch/arm/lib/copy_to_user.S index 9648b0675a3e..caf5019d8161 100644 --- a/arch/arm/lib/copy_to_user.S +++ b/arch/arm/lib/copy_to_user.S @@ -17,7 +17,7 @@ /* * Prototype: * - * size_t __copy_to_user(void *to, const void *from, size_t n) + * size_t arm_copy_to_user(void *to, const void *from, size_t n) * * Purpose: * @@ -93,11 +93,11 @@ .text ENTRY(__copy_to_user_std) -WEAK(__copy_to_user) +WEAK(arm_copy_to_user) #include "copy_template.S" -ENDPROC(__copy_to_user) +ENDPROC(arm_copy_to_user) ENDPROC(__copy_to_user_std) .pushsection .text.fixup,"ax" diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c index 3e58d710013c..77f020e75ccd 100644 --- a/arch/arm/lib/uaccess_with_memcpy.c +++ b/arch/arm/lib/uaccess_with_memcpy.c @@ -136,7 +136,7 @@ out: } unsigned long -__copy_to_user(void __user *to, const void *from, unsigned long n) +arm_copy_to_user(void __user *to, const void *from, unsigned long n) { /* * This test is stubbed out of the main function above to keep @@ -190,7 +190,7 @@ out: return n; } -unsigned long __clear_user(void __user *addr, unsigned long n) +unsigned long arm_clear_user(void __user *addr, unsigned long n) { /* See rational for this in __copy_to_user() above. */ if (n < 64) -- cgit v1.2.3 From f1806e08194739e8c13ec39d8c2a38e9ec2a2ea8 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 19 Aug 2015 21:23:48 +0100 Subject: UPSTREAM: ARM: domains: switch to keeping domain value in register Rather than modifying both the domain access control register and our per-thread copy, modify only the domain access control register, and use the per-thread copy to save and restore the register over context switches. We can also avoid the explicit initialisation of the init thread_info structure. This allows us to avoid needing to gain access to the thread information at the uaccess control sites. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 1eef5d2f1b461c120bcd82077edee5ec706ac53b) Signed-off-by: Kees Cook Change-Id: If903ec86006c4de1a440bf918c5d2491e34ade05 --- arch/arm/include/asm/domain.h | 20 +++++++++++++++----- arch/arm/include/asm/thread_info.h | 3 --- arch/arm/kernel/entry-armv.S | 2 ++ arch/arm/kernel/process.c | 13 ++++++++++--- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index 6ddbe446425e..7f2941905714 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -59,6 +59,17 @@ #ifndef __ASSEMBLY__ +static inline unsigned int get_domain(void) +{ + unsigned int domain; + + asm( + "mrc p15, 0, %0, c3, c0 @ get domain" + : "=r" (domain)); + + return domain; +} + #ifdef CONFIG_CPU_USE_DOMAINS static inline void set_domain(unsigned val) { @@ -70,11 +81,10 @@ static inline void set_domain(unsigned val) #define modify_domain(dom,type) \ do { \ - struct thread_info *thread = current_thread_info(); \ - unsigned int domain = thread->cpu_domain; \ - domain &= ~domain_val(dom, DOMAIN_MANAGER); \ - thread->cpu_domain = domain | domain_val(dom, type); \ - set_domain(thread->cpu_domain); \ + unsigned int domain = get_domain(); \ + domain &= ~domain_val(dom, DOMAIN_MANAGER); \ + domain = domain | domain_val(dom, type); \ + set_domain(domain); \ } while (0) #else diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index bd32eded3e50..0a0aec410d8c 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -74,9 +74,6 @@ struct thread_info { .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ - .cpu_domain = domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_IO, DOMAIN_CLIENT), \ } #define init_thread_info (init_thread_union.thread_info) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index b953d72ef514..a7f6e2a84465 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -782,6 +782,8 @@ ENTRY(__switch_to) ldr r4, [r2, #TI_TP_VALUE] ldr r5, [r2, #TI_TP_VALUE + 4] #ifdef CONFIG_CPU_USE_DOMAINS + mrc p15, 0, r6, c3, c0, 0 @ Get domain register + str r6, [r1, #TI_CPU_DOMAIN] @ Save old domain register ldr r6, [r2, #TI_CPU_DOMAIN] #endif switch_tls r1, r4, r5, r3, r7 diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index c49a915ef811..6cc5ba29ad47 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -231,10 +231,9 @@ void __show_regs(struct pt_regs *regs) buf[0] = '\0'; #ifdef CONFIG_CPU_CP15_MMU { - unsigned int transbase, dac; + unsigned int transbase, dac = get_domain(); asm("mrc p15, 0, %0, c2, c0\n\t" - "mrc p15, 0, %1, c3, c0\n" - : "=r" (transbase), "=r" (dac)); + : "=r" (transbase)); snprintf(buf, sizeof(buf), " Table: %08x DAC: %08x", transbase, dac); } @@ -297,6 +296,14 @@ copy_thread(unsigned long clone_flags, unsigned long stack_start, memset(&thread->cpu_context, 0, sizeof(struct cpu_context_save)); + /* + * Copy the initial value of the domain access control register + * from the current thread: thread->addr_limit will have been + * copied from the current thread via setup_thread_stack() in + * kernel/fork.c + */ + thread->cpu_domain = get_domain(); + if (likely(!(p->flags & PF_KTHREAD))) { *childregs = *current_pt_regs(); childregs->ARM_r0 = 0; -- cgit v1.2.3 From 07127c1717bfc572a259e5ccdc9d452e4da5bb08 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 19 Aug 2015 22:36:24 +0100 Subject: UPSTREAM: ARM: domains: provide domain_mask() Provide a macro to generate the mask for a domain, rather than using domain_val(, DOMAIN_MANAGER) which won't work when CPU_USE_DOMAINS is turned off. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 8e798706f7e9cd7f096aa194de90269dde83773e) Signed-off-by: Kees Cook Change-Id: I3f732beb6f8d094648636c6030c47115382c6dd7 --- arch/arm/include/asm/domain.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index 7f2941905714..045b9b453bcd 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -55,7 +55,8 @@ #define DOMAIN_MANAGER 1 #endif -#define domain_val(dom,type) ((type) << (2*(dom))) +#define domain_mask(dom) ((3) << (2 * (dom))) +#define domain_val(dom,type) ((type) << (2 * (dom))) #ifndef __ASSEMBLY__ @@ -82,7 +83,7 @@ static inline void set_domain(unsigned val) #define modify_domain(dom,type) \ do { \ unsigned int domain = get_domain(); \ - domain &= ~domain_val(dom, DOMAIN_MANAGER); \ + domain &= ~domain_mask(dom); \ domain = domain | domain_val(dom, type); \ set_domain(domain); \ } while (0) -- cgit v1.2.3 From ab4e6d754114b8bfe3e26766b3eeb226fd951b69 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 21 Aug 2015 09:23:26 +0100 Subject: BACKPORT: ARM: domains: move initial domain setting value to asm/domains.h Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 0171356a7708af01ad3224702b7f0aaa5b7a1399) Signed-off-by: Kees Cook Conflicts: arch/arm/kernel/head.S Change-Id: If8fb66d8887e88c2e8a576b9855225df6ddb3471 --- arch/arm/include/asm/domain.h | 6 ++++++ arch/arm/kernel/head.S | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index 045b9b453bcd..4218f88e8f7e 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -58,6 +58,12 @@ #define domain_mask(dom) ((3) << (2 * (dom))) #define domain_val(dom,type) ((type) << (2 * (dom))) +#define DACR_INIT \ + (domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ + domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ + domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ + domain_val(DOMAIN_IO, DOMAIN_CLIENT)) + #ifndef __ASSEMBLY__ static inline unsigned int get_domain(void) diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index 3637973a9708..b96475bdef86 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -441,10 +441,7 @@ __enable_mmu: bic r0, r0, #CR_I #endif #ifndef CONFIG_ARM_LPAE - mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_IO, DOMAIN_CLIENT)) + mov r5, #DACR_INIT mcr p15, 0, r5, c3, c0, 0 @ load domain access register mcr p15, 0, r4, c2, c0, 0 @ load page table pointer #endif -- cgit v1.2.3 From b2cfef831e74fe04838e29c6d254b6d99a3dc614 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 21 Aug 2015 09:30:16 +0100 Subject: UPSTREAM: ARM: domains: get rid of manager mode for user domain Since we switched to early trap initialisation in 94e5a85b3be0 ("ARM: earlier initialization of vectors page") we haven't been writing directly to the vectors page, and so there's no need for this domain to be in manager mode. Switch it to client mode. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 3c2aed5b28819564e1a07b4686bd89802bcc4d6b) Signed-off-by: Kees Cook Change-Id: I6a31f15cd3cc8d431ee50fcc041e95becf2ed7df --- arch/arm/include/asm/domain.h | 2 +- arch/arm/kernel/traps.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index 4218f88e8f7e..08b601e69ddc 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -59,7 +59,7 @@ #define domain_val(dom,type) ((type) << (2 * (dom))) #define DACR_INIT \ - (domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ + (domain_val(DOMAIN_USER, DOMAIN_CLIENT) | \ domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ domain_val(DOMAIN_IO, DOMAIN_CLIENT)) diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 3dce1a342030..4837003aad6a 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -878,7 +878,6 @@ void __init early_trap_init(void *vectors_base) kuser_init(vectors_base); flush_icache_range(vectors, vectors + PAGE_SIZE * 2); - modify_domain(DOMAIN_USER, DOMAIN_CLIENT); #else /* ifndef CONFIG_CPU_V7M */ /* * on V7-M there is no need to copy the vector table to a dedicated -- cgit v1.2.3 From 9a1f6e83b6a0197f0c4cdc6e289865a8ab6ed4eb Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 21 Aug 2015 09:38:31 +0100 Subject: UPSTREAM: ARM: domains: keep vectors in separate domain Keep the machine vectors in its own domain to avoid software based user access control from making the vector code inaccessible, and thereby deadlocking the machine. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit a02d8dfd54cdf3b1b0464ccc2c1c4afe2c003a35) Signed-off-by: Kees Cook Change-Id: I2f912a55c606a08b9e123de2819bc77f8dac955b --- arch/arm/include/asm/domain.h | 4 +++- arch/arm/include/asm/pgtable-2level-hwdef.h | 1 + arch/arm/mm/mmu.c | 4 ++-- arch/arm/mm/pgd.c | 10 ++++++++++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index 08b601e69ddc..396a12e486fe 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -43,6 +43,7 @@ #define DOMAIN_USER 1 #define DOMAIN_IO 0 #endif +#define DOMAIN_VECTORS 3 /* * Domain types @@ -62,7 +63,8 @@ (domain_val(DOMAIN_USER, DOMAIN_CLIENT) | \ domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_IO, DOMAIN_CLIENT)) + domain_val(DOMAIN_IO, DOMAIN_CLIENT) | \ + domain_val(DOMAIN_VECTORS, DOMAIN_CLIENT)) #ifndef __ASSEMBLY__ diff --git a/arch/arm/include/asm/pgtable-2level-hwdef.h b/arch/arm/include/asm/pgtable-2level-hwdef.h index 5e68278e953e..d0131ee6f6af 100644 --- a/arch/arm/include/asm/pgtable-2level-hwdef.h +++ b/arch/arm/include/asm/pgtable-2level-hwdef.h @@ -23,6 +23,7 @@ #define PMD_PXNTABLE (_AT(pmdval_t, 1) << 2) /* v7 */ #define PMD_BIT4 (_AT(pmdval_t, 1) << 4) #define PMD_DOMAIN(x) (_AT(pmdval_t, (x)) << 5) +#define PMD_DOMAIN_MASK PMD_DOMAIN(0x0f) #define PMD_PROTECTION (_AT(pmdval_t, 1) << 9) /* v5 */ /* * - section diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 7186382672b5..ddb5a7b38b56 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -291,13 +291,13 @@ static struct mem_type mem_types[] = { .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | L_PTE_RDONLY, .prot_l1 = PMD_TYPE_TABLE, - .domain = DOMAIN_USER, + .domain = DOMAIN_VECTORS, }, [MT_HIGH_VECTORS] = { .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | L_PTE_USER | L_PTE_RDONLY, .prot_l1 = PMD_TYPE_TABLE, - .domain = DOMAIN_USER, + .domain = DOMAIN_VECTORS, }, [MT_MEMORY_RWX] = { .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY, diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index a3681f11dd9f..e683db1b90a3 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -84,6 +84,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm) if (!new_pte) goto no_pte; +#ifndef CONFIG_ARM_LPAE + /* + * Modify the PTE pointer to have the correct domain. This + * needs to be the vectors domain to avoid the low vectors + * being unmapped. + */ + pmd_val(*new_pmd) &= ~PMD_DOMAIN_MASK; + pmd_val(*new_pmd) |= PMD_DOMAIN(DOMAIN_VECTORS); +#endif + init_pud = pud_offset(init_pgd, 0); init_pmd = pmd_offset(init_pud, 0); init_pte = pte_offset_map(init_pmd, 0); -- cgit v1.2.3 From 7d8fb5cf801ae6f92dca85bb32cfe29a707faece Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 21 Aug 2015 09:42:10 +0100 Subject: UPSTREAM: ARM: domains: remove DOMAIN_TABLE DOMAIN_TABLE is not used; in any case, it aliases to the kernel domain. Remove this definition. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 1fb6755f16872ad256c18cce2830f9087502dffd) Signed-off-by: Kees Cook Change-Id: I87e5aabbd57de2f6b6d96ac1ed6d1e85ceda4ed6 --- arch/arm/include/asm/domain.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index 396a12e486fe..2be929549938 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -34,12 +34,10 @@ */ #ifndef CONFIG_IO_36 #define DOMAIN_KERNEL 0 -#define DOMAIN_TABLE 0 #define DOMAIN_USER 1 #define DOMAIN_IO 2 #else #define DOMAIN_KERNEL 2 -#define DOMAIN_TABLE 2 #define DOMAIN_USER 1 #define DOMAIN_IO 0 #endif @@ -62,7 +60,6 @@ #define DACR_INIT \ (domain_val(DOMAIN_USER, DOMAIN_CLIENT) | \ domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ domain_val(DOMAIN_IO, DOMAIN_CLIENT) | \ domain_val(DOMAIN_VECTORS, DOMAIN_CLIENT)) -- cgit v1.2.3 From 6cc9b9815f2f100499f7934aca1a214fedc7de68 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 7 Sep 2015 00:29:15 +0100 Subject: UPSTREAM: ARM: swpan: fix nwfpe for uaccess changes NWFPE needs to access userspace to check whether the next instruction is another FP instruction. Allow userspace access for this read. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 39dc53deff30d9b239ac36cfeb0ef2022d03a449) Signed-off-by: Kees Cook Change-Id: Ia9a0fd707d14f8a8f4b181a210a8e27fabd9060d --- arch/arm/nwfpe/entry.S | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm/nwfpe/entry.S b/arch/arm/nwfpe/entry.S index 71df43547659..39c20afad7ed 100644 --- a/arch/arm/nwfpe/entry.S +++ b/arch/arm/nwfpe/entry.S @@ -95,9 +95,10 @@ emulate: reteq r4 @ no, return failure next: + uaccess_enable r3 .Lx1: ldrt r6, [r5], #4 @ get the next instruction and @ increment PC - + uaccess_disable r3 and r2, r6, #0x0F000000 @ test for FP insns teq r2, #0x0C000000 teqne r2, #0x0D000000 -- cgit v1.2.3 From 0e18e5355bb66e54159bb685e2766779aa75ac74 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 7 Sep 2015 00:30:06 +0100 Subject: UPSTREAM: ARM: uaccess: remove unneeded uaccess_save_and_disable macro This macro is never referenced, remove it. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 296254f3223d201f2aa53f5f717eedfdc63f3db8) Signed-off-by: Kees Cook Change-Id: I647251a8c51c81c2b89cc0cd35a5f60eba1a4855 --- arch/arm/include/asm/assembler.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 4fdb15e898ad..00558fff4deb 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -442,11 +442,6 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .macro uaccess_restore .endm - .macro uaccess_save_and_disable, tmp - uaccess_save \tmp - uaccess_disable \tmp - .endm - .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo .macro ret\c, reg #if __LINUX_ARM_ARCH__ < 6 -- cgit v1.2.3 From b59db83c6edc5e9c48ea4ddaaaa7cd7f174df3a5 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 9 Sep 2015 21:19:49 +0100 Subject: UPSTREAM: ARM: uaccess: fix undefined instruction on ARMv7M/noMMU The use of get_domain() in copy_thread() results in an oops on ARMv7M/noMMU systems. The thread cpu_domain value is only used when CONFIG_CPU_USE_DOMAINS is enabled, so there's no need to save the value in copy_thread() except when this is enabled, and this option will never be enabled on these platforms. Unhandled exception: IPSR = 00000006 LR = fffffff1 CPU: 0 PID: 0 Comm: swapper Not tainted 4.2.0-next-20150909-00001-gb8ec5ad #41 Hardware name: NXP LPC18xx/43xx (Device Tree) task: 2823fbe0 ti: 2823c000 task.ti: 2823c000 PC is at copy_thread+0x18/0x92 LR is at copy_thread+0x19/0x92 pc : [<2800a46e>] lr : [<2800a46f>] psr: 4100000b sp : 2823df00 ip : 00000000 fp : 287c81c0 r10: 00000000 r9 : 00800300 r8 : 287c8000 r7 : 287c8000 r6 : 2818908d r5 : 00000000 r4 : 287ca000 r3 : 00000000 r2 : 00000000 r1 : fffffff0 r0 : 287ca048 xPSR: 4100000b Reported-by: Ariel D'Alessandro Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit af4cb25df93d2e7a97d65db2bfacaa4400988dea) Signed-off-by: Kees Cook Change-Id: I2aa2bf6c1153732c68919ff465ada7878d327310 --- arch/arm/kernel/process.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 6cc5ba29ad47..21a80630c9cb 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -296,6 +296,7 @@ copy_thread(unsigned long clone_flags, unsigned long stack_start, memset(&thread->cpu_context, 0, sizeof(struct cpu_context_save)); +#ifdef CONFIG_CPU_USE_DOMAINS /* * Copy the initial value of the domain access control register * from the current thread: thread->addr_limit will have been @@ -303,6 +304,7 @@ copy_thread(unsigned long clone_flags, unsigned long stack_start, * kernel/fork.c */ thread->cpu_domain = get_domain(); +#endif if (likely(!(p->flags & PF_KTHREAD))) { *childregs = *current_pt_regs(); -- cgit v1.2.3 From 4fe92225a5024052f095e9e24dce9242a5ad3007 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 11 Sep 2015 08:17:39 +0100 Subject: UPSTREAM: ARM: domains: thread_info.h no longer needs asm/domains.h As of 1eef5d2f1b46 ("ARM: domains: switch to keeping domain value in register") we no longer need to include asm/domains.h into asm/thread_info.h. Remove it. Tested-by: Robert Jarzmik Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 716ff1921a86c637b8875c7bb312fc6755fa9300) Signed-off-by: Kees Cook Change-Id: I35ba6851bd5d1e688e8d16771efd3f738dbd5436 --- arch/arm/include/asm/thread_info.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 0a0aec410d8c..ae02e68b61fc 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -25,7 +25,6 @@ struct task_struct; #include -#include typedef unsigned long mm_segment_t; -- cgit v1.2.3 From bad6c993d9ebd088c16f3249e4c8bc2af1d1fc7e Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 11 Sep 2015 08:34:52 +0100 Subject: UPSTREAM: ARM: domains: add memory dependencies to get_domain/set_domain We need to have memory dependencies on get_domain/set_domain to avoid the compiler over-optimising these inline assembly instructions. Loads/stores must not be reordered across a set_domain(), so introduce a compiler barrier for that assembly. The value of get_domain() must not be cached across a set_domain(), but we still want to allow the compiler to optimise it away. Introduce a dependency on current_thread_info()->cpu_domain to avoid this; the new memory clobber in set_domain() should therefore cause the compiler to re-load this. The other advantage of using this is we should have its address in the register set already, or very soon after at most call sites. Tested-by: Robert Jarzmik Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 6e8f580d1fcc18e290713984c379cb97131c015a) Signed-off-by: Kees Cook Change-Id: I44021c2e0af72a2f20bab4d5af7d67d20ca0a211 --- arch/arm/include/asm/domain.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index 2be929549938..bc42d4cb17fe 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -12,6 +12,7 @@ #ifndef __ASSEMBLY__ #include +#include #endif /* @@ -71,7 +72,8 @@ static inline unsigned int get_domain(void) asm( "mrc p15, 0, %0, c3, c0 @ get domain" - : "=r" (domain)); + : "=r" (domain) + : "m" (current_thread_info()->cpu_domain)); return domain; } @@ -81,7 +83,7 @@ static inline void set_domain(unsigned val) { asm volatile( "mcr p15, 0, %0, c3, c0 @ set domain" - : : "r" (val)); + : : "r" (val) : "memory"); isb(); } -- cgit v1.2.3 From a410153a08c71144474d76c7a2d3bae1b295ebf1 Mon Sep 17 00:00:00 2001 From: Robert Jarzmik Date: Fri, 11 Sep 2015 17:12:27 +0100 Subject: UPSTREAM: ARM: 8431/1: fix alignement of __bug_table section entries On old ARM chips, unaligned accesses to memory are not trapped and fixed. On module load, symbols are relocated, and the relocation of __bug_table symbols is done on a u32 basis. Yet the section is not aligned to a multiple of 4 address, but to a multiple of 2. This triggers an Oops on pxa architecture, where address 0xbf0021ea is the first relocation in the __bug_table section : apply_relocate(): pxa3xx_nand: section 13 reloc 0 sym '' Unable to handle kernel paging request at virtual address bf0021ea pgd = e1cd0000 [bf0021ea] *pgd=c1cce851, *pte=c1cde04f, *ppte=c1cde01f Internal error: Oops: 23 [#1] ARM Modules linked in: CPU: 0 PID: 606 Comm: insmod Not tainted 4.2.0-rc8-next-20150828-cm-x300+ #887 Hardware name: CM-X300 module task: e1c68700 ti: e1c3e000 task.ti: e1c3e000 PC is at apply_relocate+0x2f4/0x3d4 LR is at 0xbf0021ea pc : [] lr : [] psr: 80000013 sp : e1c3fe30 ip : 60000013 fp : e49e8c60 r10: e49e8fa8 r9 : 00000000 r8 : e49e7c58 r7 : e49e8c38 r6 : e49e8a58 r5 : e49e8920 r4 : e49e8918 r3 : bf0021ea r2 : bf007034 r1 : 00000000 r0 : bf000000 Flags: Nzcv IRQs on FIQs on Mode SVC_32 ISA ARM Segment none Control: 0000397f Table: c1cd0018 DAC: 00000051 Process insmod (pid: 606, stack limit = 0xe1c3e198) [] (apply_relocate) from [] (load_module+0x1248/0x1f5c) [] (load_module) from [] (SyS_init_module+0xe4/0x170) [] (SyS_init_module) from [] (ret_fast_syscall+0x0/0x38) Fix this by ensuring entries in __bug_table are all aligned to at least of multiple of 4. This transforms a module section __bug_table as : - [12] __bug_table PROGBITS 00000000 002232 000018 00 A 0 0 1 + [12] __bug_table PROGBITS 00000000 002232 000018 00 A 0 0 4 Signed-off-by: Robert Jarzmik Reviewed-by: Dave Martin Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit a4a5a7379e4ca03c192b732d61e446994eb67bbc) Signed-off-by: Kees Cook Change-Id: I0ea8e41d6f71e177b37160487c8b26ff8bea37c5 --- arch/arm/include/asm/bug.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/include/asm/bug.h b/arch/arm/include/asm/bug.h index b274bde24905..e7335a92144e 100644 --- a/arch/arm/include/asm/bug.h +++ b/arch/arm/include/asm/bug.h @@ -40,6 +40,7 @@ do { \ "2:\t.asciz " #__file "\n" \ ".popsection\n" \ ".pushsection __bug_table,\"a\"\n" \ + ".align 2\n" \ "3:\t.word 1b, 2b\n" \ "\t.hword " #__line ", 0\n" \ ".popsection"); \ -- cgit v1.2.3 From 6588b7fe09f583f92407ca90630f9f9a5e4a32f2 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 19 Aug 2015 20:40:41 +0100 Subject: UPSTREAM: ARM: software-based priviledged-no-access support Provide a software-based implementation of the priviledged no access support found in ARMv8.1. Userspace pages are mapped using a different domain number from the kernel and IO mappings. If we switch the user domain to "no access" when we enter the kernel, we can prevent the kernel from touching userspace. However, the kernel needs to be able to access userspace via the various user accessor functions. With the wrapping in the previous patch, we can temporarily enable access when the kernel needs user access, and re-disable it afterwards. This allows us to trap non-intended accesses to userspace, eg, caused by an inadvertent dereference of the LIST_POISON* values, which, with appropriate user mappings setup, can be made to succeed. This in turn can allow use-after-free bugs to be further exploited than would otherwise be possible. Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit a5e090acbf545c0a3b04080f8a488b17ec41fe02) Signed-off-by: Kees Cook Change-Id: Ic57a2969ba8199fab04fc657f15777422be3b2fa --- arch/arm/Kconfig | 15 +++++++++++++++ arch/arm/include/asm/assembler.h | 30 ++++++++++++++++++++++++++++++ arch/arm/include/asm/domain.h | 21 +++++++++++++++++++-- arch/arm/include/asm/uaccess.h | 14 ++++++++++++++ arch/arm/kernel/process.c | 36 ++++++++++++++++++++++++++++++------ arch/arm/kernel/swp_emulate.c | 3 +++ arch/arm/lib/csumpartialcopyuser.S | 14 ++++++++++++++ 7 files changed, 125 insertions(+), 8 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index cffce3e6615c..8c6982b8ff05 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1671,6 +1671,21 @@ config HIGHPTE bool "Allocate 2nd-level pagetables from highmem" depends on HIGHMEM +config CPU_SW_DOMAIN_PAN + bool "Enable use of CPU domains to implement privileged no-access" + depends on MMU && !ARM_LPAE + default y + help + Increase kernel security by ensuring that normal kernel accesses + are unable to access userspace addresses. This can help prevent + use-after-free bugs becoming an exploitable privilege escalation + by ensuring that magic values (such as LIST_POISON) will always + fault when dereferenced. + + CPUs with low-vector mappings use a best-efforts implementation. + Their lower 1MB needs to remain accessible for the vectors, but + the remainder of userspace will become appropriately inaccessible. + config HW_PERF_EVENTS bool "Enable hardware performance counter support for perf events" depends on PERF_EVENTS diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 00558fff4deb..c245f15abd44 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -431,15 +431,45 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .endm .macro uaccess_disable, tmp, isb=1 +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + /* + * Whenever we re-enter userspace, the domains should always be + * set appropriately. + */ + mov \tmp, #DACR_UACCESS_DISABLE + mcr p15, 0, \tmp, c3, c0, 0 @ Set domain register + .if \isb + instr_sync + .endif +#endif .endm .macro uaccess_enable, tmp, isb=1 +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + /* + * Whenever we re-enter userspace, the domains should always be + * set appropriately. + */ + mov \tmp, #DACR_UACCESS_ENABLE + mcr p15, 0, \tmp, c3, c0, 0 + .if \isb + instr_sync + .endif +#endif .endm .macro uaccess_save, tmp +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + mrc p15, 0, \tmp, c3, c0, 0 + str \tmp, [sp, #S_FRAME_SIZE] +#endif .endm .macro uaccess_restore +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + ldr r0, [sp, #S_FRAME_SIZE] + mcr p15, 0, r0, c3, c0, 0 +#endif .endm .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index bc42d4cb17fe..fc8ba1663601 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -58,11 +58,29 @@ #define domain_mask(dom) ((3) << (2 * (dom))) #define domain_val(dom,type) ((type) << (2 * (dom))) +#ifdef CONFIG_CPU_SW_DOMAIN_PAN +#define DACR_INIT \ + (domain_val(DOMAIN_USER, DOMAIN_NOACCESS) | \ + domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ + domain_val(DOMAIN_IO, DOMAIN_CLIENT) | \ + domain_val(DOMAIN_VECTORS, DOMAIN_CLIENT)) +#else #define DACR_INIT \ (domain_val(DOMAIN_USER, DOMAIN_CLIENT) | \ domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ domain_val(DOMAIN_IO, DOMAIN_CLIENT) | \ domain_val(DOMAIN_VECTORS, DOMAIN_CLIENT)) +#endif + +#define __DACR_DEFAULT \ + domain_val(DOMAIN_KERNEL, DOMAIN_CLIENT) | \ + domain_val(DOMAIN_IO, DOMAIN_CLIENT) | \ + domain_val(DOMAIN_VECTORS, DOMAIN_CLIENT) + +#define DACR_UACCESS_DISABLE \ + (__DACR_DEFAULT | domain_val(DOMAIN_USER, DOMAIN_NOACCESS)) +#define DACR_UACCESS_ENABLE \ + (__DACR_DEFAULT | domain_val(DOMAIN_USER, DOMAIN_CLIENT)) #ifndef __ASSEMBLY__ @@ -78,7 +96,6 @@ static inline unsigned int get_domain(void) return domain; } -#ifdef CONFIG_CPU_USE_DOMAINS static inline void set_domain(unsigned val) { asm volatile( @@ -87,6 +104,7 @@ static inline void set_domain(unsigned val) isb(); } +#ifdef CONFIG_CPU_USE_DOMAINS #define modify_domain(dom,type) \ do { \ unsigned int domain = get_domain(); \ @@ -96,7 +114,6 @@ static inline void set_domain(unsigned val) } while (0) #else -static inline void set_domain(unsigned val) { } static inline void modify_domain(unsigned dom, unsigned type) { } #endif diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 2580eabf7706..8cc85a4ebec2 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -57,11 +57,25 @@ extern int fixup_exception(struct pt_regs *regs); */ static inline unsigned int uaccess_save_and_enable(void) { +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + unsigned int old_domain = get_domain(); + + /* Set the current domain access to permit user accesses */ + set_domain((old_domain & ~domain_mask(DOMAIN_USER)) | + domain_val(DOMAIN_USER, DOMAIN_CLIENT)); + + return old_domain; +#else return 0; +#endif } static inline void uaccess_restore(unsigned int flags) { +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + /* Restore the user access mask */ + set_domain(flags); +#endif } /* diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 21a80630c9cb..eafe6b2089c9 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -214,12 +214,36 @@ void __show_regs(struct pt_regs *regs) buf[4] = '\0'; #ifndef CONFIG_CPU_V7M - printk("Flags: %s IRQs o%s FIQs o%s Mode %s ISA %s Segment %s\n", - buf, interrupts_enabled(regs) ? "n" : "ff", - fast_interrupts_enabled(regs) ? "n" : "ff", - processor_modes[processor_mode(regs)], - isa_modes[isa_mode(regs)], - get_fs() == get_ds() ? "kernel" : "user"); + { + unsigned int domain = get_domain(); + const char *segment; + +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + /* + * Get the domain register for the parent context. In user + * mode, we don't save the DACR, so lets use what it should + * be. For other modes, we place it after the pt_regs struct. + */ + if (user_mode(regs)) + domain = DACR_UACCESS_ENABLE; + else + domain = *(unsigned int *)(regs + 1); +#endif + + if ((domain & domain_mask(DOMAIN_USER)) == + domain_val(DOMAIN_USER, DOMAIN_NOACCESS)) + segment = "none"; + else if (get_fs() == get_ds()) + segment = "kernel"; + else + segment = "user"; + + printk("Flags: %s IRQs o%s FIQs o%s Mode %s ISA %s Segment %s\n", + buf, interrupts_enabled(regs) ? "n" : "ff", + fast_interrupts_enabled(regs) ? "n" : "ff", + processor_modes[processor_mode(regs)], + isa_modes[isa_mode(regs)], segment); + } #else printk("xPSR: %08lx\n", regs->ARM_cpsr); #endif diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c index 1361756782c7..5b26e7efa9ea 100644 --- a/arch/arm/kernel/swp_emulate.c +++ b/arch/arm/kernel/swp_emulate.c @@ -141,11 +141,14 @@ static int emulate_swpX(unsigned int address, unsigned int *data, while (1) { unsigned long temp; + unsigned int __ua_flags; + __ua_flags = uaccess_save_and_enable(); if (type == TYPE_SWPB) __user_swpb_asm(*data, address, res, temp); else __user_swp_asm(*data, address, res, temp); + uaccess_restore(__ua_flags); if (likely(res != -EAGAIN) || signal_pending(current)) break; diff --git a/arch/arm/lib/csumpartialcopyuser.S b/arch/arm/lib/csumpartialcopyuser.S index 1d0957e61f89..1712f132b80d 100644 --- a/arch/arm/lib/csumpartialcopyuser.S +++ b/arch/arm/lib/csumpartialcopyuser.S @@ -17,6 +17,19 @@ .text +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + .macro save_regs + mrc p15, 0, ip, c3, c0, 0 + stmfd sp!, {r1, r2, r4 - r8, ip, lr} + uaccess_enable ip + .endm + + .macro load_regs + ldmfd sp!, {r1, r2, r4 - r8, ip, lr} + mcr p15, 0, ip, c3, c0, 0 + ret lr + .endm +#else .macro save_regs stmfd sp!, {r1, r2, r4 - r8, lr} .endm @@ -24,6 +37,7 @@ .macro load_regs ldmfd sp!, {r1, r2, r4 - r8, pc} .endm +#endif .macro load1b, reg1 ldrusr \reg1, r0, 1 -- cgit v1.2.3 From 2e9d4f240193f46048a8f811684be0a036c18dd0 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 23 Sep 2015 11:06:30 +0100 Subject: UPSTREAM: ARM: alignment: fix alignment handling for uaccess changes Jonathan Liu reports that the recent addition of CPU_SW_DOMAIN_PAN causes wpa_supplicant to die due to the following kernel oops: Unhandled fault: page domain fault (0x81b) at 0x001017a2 pgd = ee1b8000 [001017a2] *pgd=6ebee831, *pte=6c35475f, *ppte=6c354c7f Internal error: : 81b [#1] SMP ARM Modules linked in: rt2800usb rt2x00usb rt2800librt2x00lib crc_ccitt mac80211 CPU: 1 PID: 202 Comm: wpa_supplicant Not tainted 4.3.0-rc2 #1 Hardware name: Allwinner sun7i (A20) Family task: ec872f80 ti: ee364000 task.ti: ee364000 PC is at do_alignment_ldmstm+0x1d4/0x238 LR is at 0x0 pc : [] lr : [<00000000>] psr: 600c0113 sp : ee365e18 ip : 00000000 fp : 00000002 r10: 001017a2 r9 : 00000002 r8 : 001017aa r7 : ee365fb0 r6 : e8820018 r5 : 001017a2 r4 : 00000003 r3 : d49e30e0 r2 : 00000000 r1 : ee365fbc r0 : 00000000 Flags: nZCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment none[ 34.393106] Control: 10c5387d Table: 6e1b806a DAC: 00000051 Process wpa_supplicant (pid: 202, stack limit = 0xee364210) Stack: (0xee365e18 to 0xee366000) ... [] (do_alignment_ldmstm) from [] (do_alignment+0x1f0/0x904) [] (do_alignment) from [] (do_DataAbort+0x38/0xb4) [] (do_DataAbort) from [] (__dabt_usr+0x3c/0x40) Exception stack(0xee365fb0 to 0xee365ff8) 5fa0: 00000000 56c728c0 001017a2 d49e30e0 5fc0: 775448d2 597d4e74 00200800 7a9e1625 00802001 00000021 b6deec84 00000100 5fe0: 08020200 be9f4f20 0c0b0d0a b6d9b3e0 600c0010 ffffffff Code: e1a0a005 e1a0000c 1affffe8 e5913000 (e4ea3001) ---[ end trace 0acd3882fcfdf9dd ]--- This is caused by the alignment handler not being fixed up for the uaccess changes, and userspace issuing an unaligned LDM instruction. So, fix the problem by adding the necessary fixups. Reported-by: Jonathan Liu Tested-by: Jonathan Liu Signed-off-by: Russell King Bug: 25672827 Patchset: PAN emulation (cherry picked from commit 274e91b81ed22957b510ad2988359584eea95dae) Signed-off-by: Kees Cook Change-Id: I804b6a6d037ef1603565f98decee9190e19b03d6 --- arch/arm/mm/alignment.c | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index 9769f1eefe3b..00b7f7de28a1 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -365,15 +365,21 @@ do_alignment_ldrhstrh(unsigned long addr, unsigned long instr, struct pt_regs *r user: if (LDST_L_BIT(instr)) { unsigned long val; + unsigned int __ua_flags = uaccess_save_and_enable(); + get16t_unaligned_check(val, addr); + uaccess_restore(__ua_flags); /* signed half-word? */ if (instr & 0x40) val = (signed long)((signed short) val); regs->uregs[rd] = val; - } else + } else { + unsigned int __ua_flags = uaccess_save_and_enable(); put16t_unaligned_check(regs->uregs[rd], addr); + uaccess_restore(__ua_flags); + } return TYPE_LDST; @@ -420,14 +426,21 @@ do_alignment_ldrdstrd(unsigned long addr, unsigned long instr, user: if (load) { - unsigned long val; + unsigned long val, val2; + unsigned int __ua_flags = uaccess_save_and_enable(); + get32t_unaligned_check(val, addr); + get32t_unaligned_check(val2, addr + 4); + + uaccess_restore(__ua_flags); + regs->uregs[rd] = val; - get32t_unaligned_check(val, addr + 4); - regs->uregs[rd2] = val; + regs->uregs[rd2] = val2; } else { + unsigned int __ua_flags = uaccess_save_and_enable(); put32t_unaligned_check(regs->uregs[rd], addr); put32t_unaligned_check(regs->uregs[rd2], addr + 4); + uaccess_restore(__ua_flags); } return TYPE_LDST; @@ -458,10 +471,15 @@ do_alignment_ldrstr(unsigned long addr, unsigned long instr, struct pt_regs *reg trans: if (LDST_L_BIT(instr)) { unsigned int val; + unsigned int __ua_flags = uaccess_save_and_enable(); get32t_unaligned_check(val, addr); + uaccess_restore(__ua_flags); regs->uregs[rd] = val; - } else + } else { + unsigned int __ua_flags = uaccess_save_and_enable(); put32t_unaligned_check(regs->uregs[rd], addr); + uaccess_restore(__ua_flags); + } return TYPE_LDST; fault: @@ -531,6 +549,7 @@ do_alignment_ldmstm(unsigned long addr, unsigned long instr, struct pt_regs *reg #endif if (user_mode(regs)) { + unsigned int __ua_flags = uaccess_save_and_enable(); for (regbits = REGMASK_BITS(instr), rd = 0; regbits; regbits >>= 1, rd += 1) if (regbits & 1) { @@ -542,6 +561,7 @@ do_alignment_ldmstm(unsigned long addr, unsigned long instr, struct pt_regs *reg put32t_unaligned_check(regs->uregs[rd], eaddr); eaddr += 4; } + uaccess_restore(__ua_flags); } else { for (regbits = REGMASK_BITS(instr), rd = 0; regbits; regbits >>= 1, rd += 1) -- cgit v1.2.3 From b43f700a268c2262d95a44b2270f5358b7cba174 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 5 Dec 2015 13:42:07 +0000 Subject: UPSTREAM: ARM: fix uaccess_with_memcpy() with SW_DOMAIN_PAN The uaccess_with_memcpy() code is currently incompatible with the SW PAN code: it takes locks within the region that we've changed the DACR, potentially sleeping as a result. As we do not save and restore the DACR across co-operative sleep events, can lead to an incorrect DACR value later in this code path. Reported-by: Peter Rosin Tested-by: Peter Rosin Signed-off-by: Russell King (cherry picked from commit c014953d84ec21a4df9a43be2378861ea6e9246e) Change-Id: I9c9d1789cb69dcb6ecd1e4055833a5dfebe2045e --- arch/arm/include/asm/uaccess.h | 4 ++++ arch/arm/lib/uaccess_with_memcpy.c | 29 +++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 8cc85a4ebec2..35c9db857ebe 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -510,10 +510,14 @@ __copy_to_user_std(void __user *to, const void *from, unsigned long n); static inline unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n) { +#ifndef CONFIG_UACCESS_WITH_MEMCPY unsigned int __ua_flags = uaccess_save_and_enable(); n = arm_copy_to_user(to, from, n); uaccess_restore(__ua_flags); return n; +#else + return arm_copy_to_user(to, from, n); +#endif } extern unsigned long __must_check diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c index 77f020e75ccd..b3e68086b347 100644 --- a/arch/arm/lib/uaccess_with_memcpy.c +++ b/arch/arm/lib/uaccess_with_memcpy.c @@ -88,6 +88,7 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp) static unsigned long noinline __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n) { + unsigned long ua_flags; int atomic; if (unlikely(segment_eq(get_fs(), KERNEL_DS))) { @@ -118,7 +119,9 @@ __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n) if (tocopy > n) tocopy = n; + ua_flags = uaccess_save_and_enable(); memcpy((void *)to, from, tocopy); + uaccess_restore(ua_flags); to += tocopy; from += tocopy; n -= tocopy; @@ -145,14 +148,21 @@ arm_copy_to_user(void __user *to, const void *from, unsigned long n) * With frame pointer disabled, tail call optimization kicks in * as well making this test almost invisible. */ - if (n < 64) - return __copy_to_user_std(to, from, n); - return __copy_to_user_memcpy(to, from, n); + if (n < 64) { + unsigned long ua_flags = uaccess_save_and_enable(); + n = __copy_to_user_std(to, from, n); + uaccess_restore(ua_flags); + } else { + n = __copy_to_user_memcpy(to, from, n); + } + return n; } static unsigned long noinline __clear_user_memset(void __user *addr, unsigned long n) { + unsigned long ua_flags; + if (unlikely(segment_eq(get_fs(), KERNEL_DS))) { memset((void *)addr, 0, n); return 0; @@ -175,7 +185,9 @@ __clear_user_memset(void __user *addr, unsigned long n) if (tocopy > n) tocopy = n; + ua_flags = uaccess_save_and_enable(); memset((void *)addr, 0, tocopy); + uaccess_restore(ua_flags); addr += tocopy; n -= tocopy; @@ -193,9 +205,14 @@ out: unsigned long arm_clear_user(void __user *addr, unsigned long n) { /* See rational for this in __copy_to_user() above. */ - if (n < 64) - return __clear_user_std(addr, n); - return __clear_user_memset(addr, n); + if (n < 64) { + unsigned long ua_flags = uaccess_save_and_enable(); + n = __clear_user_std(addr, n); + uaccess_restore(ua_flags); + } else { + n = __clear_user_memset(addr, n); + } + return n; } #if 0 -- cgit v1.2.3 From a4291fedcbb40d43c095b54033a156949a71f12b Mon Sep 17 00:00:00 2001 From: Amit Pundir Date: Thu, 28 Jan 2016 14:12:20 +0530 Subject: Revert "net: wireless: Add CONFIG_WIFI_CONTROL_FUNC option" This reverts commit ae06550aade22de89e84a69d543ade8b31dd9f10. WIFI_CONTROL_FUNC config is not used anywhere in android-3.10+ kernels. Its usage can be last tracked to BCMDHD wireless driver in android-3.4 kernel. Change-Id: I8092bf10b70dd84a9bb3aade4aca80891742d504 Signed-off-by: Amit Pundir --- drivers/net/wireless/Kconfig | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/net/wireless/Kconfig b/drivers/net/wireless/Kconfig index 710733f034fa..16604bdf5197 100644 --- a/drivers/net/wireless/Kconfig +++ b/drivers/net/wireless/Kconfig @@ -265,11 +265,6 @@ config MWL8K To compile this driver as a module, choose M here: the module will be called mwl8k. If unsure, say N. -config WIFI_CONTROL_FUNC - bool "Enable WiFi control function abstraction" - help - Enables Power/Reset/Carddetect function abstraction - source "drivers/net/wireless/ath/Kconfig" source "drivers/net/wireless/b43/Kconfig" source "drivers/net/wireless/b43legacy/Kconfig" -- cgit v1.2.3 From a884540f5a58461512475fa88d547160e48b55e2 Mon Sep 17 00:00:00 2001 From: Amit Pundir Date: Mon, 8 Feb 2016 11:47:13 +0530 Subject: Revert "usb: gadget: composite: Fix corruption when changing configuration" This reverts commit 4b749ddcb21c8cffd9989d200ef4a1bef7877e09. We do not need this fix with Android ConfigFS gadgets. Change-Id: I7dffb884a49ecb2f5d619150edfa3e51808ef8cc Signed-off-by: Amit Pundir --- drivers/usb/gadget/composite.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c index e2c4286359ab..5beec3b449cc 100644 --- a/drivers/usb/gadget/composite.c +++ b/drivers/usb/gadget/composite.c @@ -845,7 +845,7 @@ done: } EXPORT_SYMBOL_GPL(usb_add_config); -static void unbind_config(struct usb_composite_dev *cdev, +static void remove_config(struct usb_composite_dev *cdev, struct usb_configuration *config) { while (!list_empty(&config->functions)) { @@ -860,6 +860,7 @@ static void unbind_config(struct usb_composite_dev *cdev, /* may free memory for "f" */ } } + list_del(&config->list); if (config->unbind) { DBG(cdev, "unbind config '%s'/%p\n", config->label, config); config->unbind(config); @@ -886,11 +887,9 @@ void usb_remove_config(struct usb_composite_dev *cdev, if (cdev->config == config) reset_config(cdev); - list_del(&config->list); - spin_unlock_irqrestore(&cdev->lock, flags); - unbind_config(cdev, config); + remove_config(cdev, config); } /*-------------------------------------------------------------------------*/ @@ -1879,8 +1878,7 @@ static void __composite_unbind(struct usb_gadget *gadget, bool unbind_driver) struct usb_configuration *c; c = list_first_entry(&cdev->configs, struct usb_configuration, list); - list_del(&c->list); - unbind_config(cdev, c); + remove_config(cdev, c); } if (cdev->driver->unbind && unbind_driver) cdev->driver->unbind(cdev); -- cgit v1.2.3 From 7b65e308f3eea338822c1bfda8f9ac009e536b7d Mon Sep 17 00:00:00 2001 From: Mark Salyzyn Date: Tue, 9 Feb 2016 12:31:42 -0800 Subject: android: recommended.cfg: enable taskstats CONFIG_TASKSTATS=y CONFIG_TASK_DELAY_ACCT=y CONFIG_TASK_XACCT=y CONFIG_TASK_IO_ACCOUNTING=y Signed-off-by: Mark Salyzyn Bug: 21334988 Bug: 26966375 Change-Id: Id54be2aad6acdb51040ba613d5d987dd693cd591 --- android/configs/android-recommended.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/android/configs/android-recommended.cfg b/android/configs/android-recommended.cfg index 562de015f05c..e4c8aaade197 100644 --- a/android/configs/android-recommended.cfg +++ b/android/configs/android-recommended.cfg @@ -111,6 +111,10 @@ CONFIG_TABLET_USB_GTCO=y CONFIG_TABLET_USB_HANWANG=y CONFIG_TABLET_USB_KBTAB=y CONFIG_TABLET_USB_WACOM=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_TASK_XACCT=y CONFIG_TIMER_STATS=y CONFIG_TMPFS=y CONFIG_TMPFS_POSIX_ACL=y -- cgit v1.2.3 From dc582695e70da54b0907ed74f46d01fb1072cc0c Mon Sep 17 00:00:00 2001 From: Mark Salyzyn Date: Wed, 10 Feb 2016 14:44:42 -0800 Subject: android: base-cfg: Add CONFIG_IP_MULTICAST (cherry pick from commit 1d0f72986958c2bf3528cadf7d7acf0771465fd1) Signed-off-by: Mark Salyzyn Bug: 19173869 Change-Id: I4ccd6161e87df7a87f3bd990cfe1de1f7567bf4c --- android/configs/android-base.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/android/configs/android-base.cfg b/android/configs/android-base.cfg index 44fa01da65b9..a4cffb1840d7 100644 --- a/android/configs/android-base.cfg +++ b/android/configs/android-base.cfg @@ -44,6 +44,7 @@ CONFIG_IPV6_PRIVACY=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y CONFIG_IP_NF_ARPFILTER=y CONFIG_IP_NF_ARPTABLES=y -- cgit v1.2.3 From 07e3edc5d7b3b64fe83caa321f1ba7d70d5daa1c Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 11 Sep 2015 15:31:24 +0100 Subject: UPSTREAM: arm64: cpu hotplug: ensure we mask out CPU_TASKS_FROZEN in notifiers (cherry picked from commit e56d82a116176f7af9d642b560abbbd3a2b68013) We have a couple of CPU hotplug notifiers for resetting the CPU debug state to a sane value when a CPU comes online. This patch ensures that we mask out CPU_TASKS_FROZEN so that we don't miss any online events occuring due to suspend/resume. Acked-by: Lorenzo Pieralisi Signed-off-by: Will Deacon Signed-off-by: Pavel Labath Bug: 27189927 Change-Id: I72549149b9bf1f0d05cb17a1db98f9a342c580c0 --- arch/arm64/kernel/debug-monitors.c | 2 +- arch/arm64/kernel/hw_breakpoint.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index b056369fd47d..3d1ce766d570 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -134,7 +134,7 @@ static int os_lock_notify(struct notifier_block *self, unsigned long action, void *data) { int cpu = (unsigned long)data; - if (action == CPU_ONLINE) + if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE) smp_call_function_single(cpu, clear_os_lock, NULL, 1); return NOTIFY_OK; } diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index e7d934d3afe0..9f548fd7b8af 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -885,7 +885,7 @@ static int hw_breakpoint_reset_notify(struct notifier_block *self, void *hcpu) { int cpu = (long)hcpu; - if (action == CPU_ONLINE) + if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE) smp_call_function_single(cpu, hw_breakpoint_reset, NULL, 1); return NOTIFY_OK; } -- cgit v1.2.3 From 52cca7009d6dfedd490779e93fb80af4efbf55a1 Mon Sep 17 00:00:00 2001 From: Tim Murray Date: Tue, 19 Jan 2016 16:36:40 -0800 Subject: ANDROID: mmc: move to a SCHED_FIFO thread (cherry picked from commit 011e507b413393eab8279dac8b778ad9b6e9971b) Running mmcqd as a prio 120 thread forces it to compete with standard user processes for IO performance, especially when the system is under severe CPU load. Move it to a SCHED_FIFO thread to reduce the impact of load on IO performance. Signed-off-by: Tim Murray Bug: 25392275 Change-Id: I1edfe73baa25e181367c30c1f40fee886e92b60d --- drivers/mmc/card/queue.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index 8efa3684aef8..ccee8ebc55d8 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c @@ -19,6 +19,7 @@ #include #include +#include #include "queue.h" #define MMC_QUEUE_BOUNCESZ 65536 @@ -50,6 +51,11 @@ static int mmc_queue_thread(void *d) { struct mmc_queue *mq = d; struct request_queue *q = mq->queue; + struct sched_param scheduler_params = {0}; + + scheduler_params.sched_priority = 1; + + sched_setscheduler(current, SCHED_FIFO, &scheduler_params); current->flags |= PF_MEMALLOC; -- cgit v1.2.3 From 45f08da5f55fc3f85fa0bc8148e4d993f3017f2a Mon Sep 17 00:00:00 2001 From: Dmitry Shmidt Date: Tue, 23 Feb 2016 13:01:53 -0800 Subject: Revert "mmc: core: Hold a wake lock accross delayed work + mmc rescan" Patch mmc: core: Signal wakeup event at card insert/removal provides wake lock for mmc_detect_change() This reverts commit bdbc5cfe7c034175466285da10d4b2a84addd9c3. --- drivers/mmc/core/core.c | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index 5da8515b8660..c731dc4b4664 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -29,7 +29,6 @@ #include #include #include -#include #include @@ -59,7 +58,6 @@ #define MMC_BKOPS_MAX_TIMEOUT (4 * 60 * 1000) /* max time to wait in ms */ static struct workqueue_struct *workqueue; -static struct wake_lock mmc_delayed_work_wake_lock; static const unsigned freqs[] = { 400000, 300000, 200000, 100000 }; /* @@ -76,7 +74,6 @@ module_param(use_spi_crc, bool, 0); static int mmc_schedule_delayed_work(struct delayed_work *work, unsigned long delay) { - wake_lock(&mmc_delayed_work_wake_lock); return queue_delayed_work(workqueue, work, delay); } @@ -2464,7 +2461,6 @@ void mmc_rescan(struct work_struct *work) struct mmc_host *host = container_of(work, struct mmc_host, detect.work); int i; - bool extend_wakelock = false; if (host->trigger_card_event && host->ops->card_event) { host->ops->card_event(host); @@ -2526,20 +2522,14 @@ void mmc_rescan(struct work_struct *work) mmc_claim_host(host); for (i = 0; i < ARRAY_SIZE(freqs); i++) { - if (!mmc_rescan_try_freq(host, max(freqs[i], host->f_min))) { - extend_wakelock = true; + if (!mmc_rescan_try_freq(host, max(freqs[i], host->f_min))) break; - } if (freqs[i] <= host->f_min) break; } mmc_release_host(host); out: - if (extend_wakelock) - wake_lock_timeout(&mmc_delayed_work_wake_lock, HZ / 2); - else - wake_unlock(&mmc_delayed_work_wake_lock); if (host->caps & MMC_CAP_NEEDS_POLL) mmc_schedule_delayed_work(&host->detect, HZ); } @@ -2761,9 +2751,6 @@ static int __init mmc_init(void) if (!workqueue) return -ENOMEM; - wake_lock_init(&mmc_delayed_work_wake_lock, WAKE_LOCK_SUSPEND, - "mmc_delayed_work"); - ret = mmc_register_bus(); if (ret) goto destroy_workqueue; @@ -2784,7 +2771,6 @@ unregister_bus: mmc_unregister_bus(); destroy_workqueue: destroy_workqueue(workqueue); - wake_lock_destroy(&mmc_delayed_work_wake_lock); return ret; } @@ -2795,7 +2781,6 @@ static void __exit mmc_exit(void) mmc_unregister_host_class(); mmc_unregister_bus(); destroy_workqueue(workqueue); - wake_lock_destroy(&mmc_delayed_work_wake_lock); } subsys_initcall(mmc_init); -- cgit v1.2.3 From 336e299457a488dde45ca67e5e68aa1a5b6c9d4a Mon Sep 17 00:00:00 2001 From: Dmitry Shmidt Date: Wed, 24 Feb 2016 12:44:42 -0800 Subject: Revert "mmc: Extend wakelock if bus is dead" This reverts commit dde72f9e313fc52d467ef0aad41cecd2c9f9f212. --- drivers/mmc/core/core.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index c731dc4b4664..ebfb3606f7a3 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -2487,12 +2487,6 @@ void mmc_rescan(struct work_struct *work) host->detect_change = 0; - /* If the card was removed the bus will be marked - * as dead - extend the wakelock so userspace - * can respond */ - if (host->bus_dead) - extend_wakelock = 1; - /* * Let mmc_bus_put() free the bus/bus_ops if we've found that * the card is no longer present. -- cgit v1.2.3 From 4834f5d785c17889608ae930d414a22d25c745c5 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 5 Nov 2015 02:02:31 +0000 Subject: UPSTREAM: dm verity: clean up duplicate hashing code Handle dm-verity salting in one place to simplify the code. Bug: 27175947 Signed-off-by: Sami Tolvanen Reviewed-by: Kees Cook Signed-off-by: Mike Snitzer Signed-off-by: Badhri Jagan Sridharan Change-Id: Iea337445fe4f8380cf1af0e491c7935b6db2092a --- drivers/md/dm-verity.c | 262 +++++++++++++++++++++++++++---------------------- 1 file changed, 147 insertions(+), 115 deletions(-) diff --git a/drivers/md/dm-verity.c b/drivers/md/dm-verity.c index 66616db33e6f..9b0a561b22a5 100644 --- a/drivers/md/dm-verity.c +++ b/drivers/md/dm-verity.c @@ -177,6 +177,84 @@ static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, return block >> (level * v->hash_per_block_bits); } +/* + * Wrapper for crypto_shash_init, which handles verity salting. + */ +static int verity_hash_init(struct dm_verity *v, struct shash_desc *desc) +{ + int r; + + desc->tfm = v->tfm; + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; + + r = crypto_shash_init(desc); + + if (unlikely(r < 0)) { + DMERR("crypto_shash_init failed: %d", r); + return r; + } + + if (likely(v->version >= 1)) { + r = crypto_shash_update(desc, v->salt, v->salt_size); + + if (unlikely(r < 0)) { + DMERR("crypto_shash_update failed: %d", r); + return r; + } + } + + return 0; +} + +static int verity_hash_update(struct dm_verity *v, struct shash_desc *desc, + const u8 *data, size_t len) +{ + int r = crypto_shash_update(desc, data, len); + + if (unlikely(r < 0)) + DMERR("crypto_shash_update failed: %d", r); + + return r; +} + +static int verity_hash_final(struct dm_verity *v, struct shash_desc *desc, + u8 *digest) +{ + int r; + + if (unlikely(!v->version)) { + r = crypto_shash_update(desc, v->salt, v->salt_size); + + if (r < 0) { + DMERR("crypto_shash_update failed: %d", r); + return r; + } + } + + r = crypto_shash_final(desc, digest); + + if (unlikely(r < 0)) + DMERR("crypto_shash_final failed: %d", r); + + return r; +} + +static int verity_hash(struct dm_verity *v, struct shash_desc *desc, + const u8 *data, size_t len, u8 *digest) +{ + int r; + + r = verity_hash_init(v, desc); + if (unlikely(r < 0)) + return r; + + r = verity_hash_update(v, desc, data, len); + if (unlikely(r < 0)) + return r; + + return verity_hash_final(v, desc, digest); +} + static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, sector_t *hash_block, unsigned *offset) { @@ -257,10 +335,10 @@ out: * If "skip_unverified" is false, unverified buffer is hashed and verified * against current value of io_want_digest(v, io). */ -static int verity_verify_level(struct dm_verity_io *io, sector_t block, - int level, bool skip_unverified) +static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, + sector_t block, int level, bool skip_unverified, + u8 *want_digest) { - struct dm_verity *v = io->v; struct dm_buffer *buf; struct buffer_aux *aux; u8 *data; @@ -277,74 +355,71 @@ static int verity_verify_level(struct dm_verity_io *io, sector_t block, aux = dm_bufio_get_aux_data(buf); if (!aux->hash_verified) { - struct shash_desc *desc; - u8 *result; - if (skip_unverified) { r = 1; goto release_ret_r; } - desc = io_hash_desc(v, io); - desc->tfm = v->tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - r = crypto_shash_init(desc); - if (r < 0) { - DMERR("crypto_shash_init failed: %d", r); + r = verity_hash(v, io_hash_desc(v, io), + data, 1 << v->hash_dev_block_bits, + io_real_digest(v, io)); + if (unlikely(r < 0)) goto release_ret_r; - } - - if (likely(v->version >= 1)) { - r = crypto_shash_update(desc, v->salt, v->salt_size); - if (r < 0) { - DMERR("crypto_shash_update failed: %d", r); - goto release_ret_r; - } - } - r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits); - if (r < 0) { - DMERR("crypto_shash_update failed: %d", r); - goto release_ret_r; - } - - if (!v->version) { - r = crypto_shash_update(desc, v->salt, v->salt_size); - if (r < 0) { - DMERR("crypto_shash_update failed: %d", r); - goto release_ret_r; - } - } - - result = io_real_digest(v, io); - r = crypto_shash_final(desc, result); - if (r < 0) { - DMERR("crypto_shash_final failed: %d", r); + if (likely(memcmp(io_real_digest(v, io), want_digest, + v->digest_size) == 0)) + aux->hash_verified = 1; + else if (verity_handle_err(v, + DM_VERITY_BLOCK_TYPE_METADATA, + hash_block)) { + r = -EIO; goto release_ret_r; } - if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) { - if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_METADATA, - hash_block)) { - r = -EIO; - goto release_ret_r; - } - } else - aux->hash_verified = 1; } data += offset; - - memcpy(io_want_digest(v, io), data, v->digest_size); - - dm_bufio_release(buf); - return 0; + memcpy(want_digest, data, v->digest_size); + r = 0; release_ret_r: dm_bufio_release(buf); - return r; } +/* + * Find a hash for a given block, write it to digest and verify the integrity + * of the hash tree if necessary. + */ +static int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, + sector_t block, u8 *digest) +{ + int i; + int r; + + if (likely(v->levels)) { + /* + * First, we try to get the requested hash for + * the current block. If the hash block itself is + * verified, zero is returned. If it isn't, this + * function returns 1 and we fall back to whole + * chain verification. + */ + r = verity_verify_level(v, io, block, 0, true, digest); + if (likely(r <= 0)) + return r; + } + + memcpy(digest, v->root_digest, v->digest_size); + + for (i = v->levels - 1; i >= 0; i--) { + r = verity_verify_level(v, io, block, i, false, digest); + if (unlikely(r)) + return r; + } + + return 0; +} + /* * Verify one "dm_verity_io" structure. */ @@ -354,54 +429,21 @@ static int verity_verify_io(struct dm_verity_io *io) struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size); unsigned b; - int i; for (b = 0; b < io->n_blocks; b++) { - struct shash_desc *desc; - u8 *result; int r; unsigned todo; + struct shash_desc *desc = io_hash_desc(v, io); - if (likely(v->levels)) { - /* - * First, we try to get the requested hash for - * the current block. If the hash block itself is - * verified, zero is returned. If it isn't, this - * function returns 0 and we fall back to whole - * chain verification. - */ - int r = verity_verify_level(io, io->block + b, 0, true); - if (likely(!r)) - goto test_block_hash; - if (r < 0) - return r; - } - - memcpy(io_want_digest(v, io), v->root_digest, v->digest_size); - - for (i = v->levels - 1; i >= 0; i--) { - int r = verity_verify_level(io, io->block + b, i, false); - if (unlikely(r)) - return r; - } + r = verity_hash_for_block(v, io, io->block + b, + io_want_digest(v, io)); + if (unlikely(r < 0)) + return r; -test_block_hash: - desc = io_hash_desc(v, io); - desc->tfm = v->tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - r = crypto_shash_init(desc); - if (r < 0) { - DMERR("crypto_shash_init failed: %d", r); + r = verity_hash_init(v, desc); + if (unlikely(r < 0)) return r; - } - if (likely(v->version >= 1)) { - r = crypto_shash_update(desc, v->salt, v->salt_size); - if (r < 0) { - DMERR("crypto_shash_update failed: %d", r); - return r; - } - } todo = 1 << v->data_dev_block_bits; do { u8 *page; @@ -412,37 +454,27 @@ test_block_hash: len = bv.bv_len; if (likely(len >= todo)) len = todo; - r = crypto_shash_update(desc, page + bv.bv_offset, len); + r = verity_hash_update(v, desc, page + bv.bv_offset, + len); kunmap_atomic(page); - if (r < 0) { - DMERR("crypto_shash_update failed: %d", r); + if (unlikely(r < 0)) return r; - } bio_advance_iter(bio, &io->iter, len); todo -= len; } while (todo); - if (!v->version) { - r = crypto_shash_update(desc, v->salt, v->salt_size); - if (r < 0) { - DMERR("crypto_shash_update failed: %d", r); - return r; - } - } - - result = io_real_digest(v, io); - r = crypto_shash_final(desc, result); - if (r < 0) { - DMERR("crypto_shash_final failed: %d", r); + r = verity_hash_final(v, desc, io_real_digest(v, io)); + if (unlikely(r < 0)) return r; - } - if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) { - if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, - io->block + b)) - return -EIO; - } + + if (likely(memcmp(io_real_digest(v, io), + io_want_digest(v, io), v->digest_size) == 0)) + continue; + else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, + io->block + b)) + return -EIO; } return 0; -- cgit v1.2.3 From f647636ac46ec81361f13ef62ba7c92a0bf6ed7b Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 5 Nov 2015 02:02:32 +0000 Subject: UPSTREAM: dm verity: separate function for parsing opt args Move optional argument parsing into a separate function to make it easier to add more of them without making verity_ctr even longer. Bug: 27175947 Signed-off-by: Sami Tolvanen Reviewed-by: Kees Cook Signed-off-by: Mike Snitzer Signed-off-by: Badhri Jagan Sridharan Change-Id: I6426dc3ac7e95c3456205fdafa1078fbe9a26743 --- drivers/md/dm-verity.c | 71 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/drivers/md/dm-verity.c b/drivers/md/dm-verity.c index 9b0a561b22a5..e75d1dba28f9 100644 --- a/drivers/md/dm-verity.c +++ b/drivers/md/dm-verity.c @@ -36,6 +36,8 @@ #define DM_VERITY_OPT_LOGGING "ignore_corruption" #define DM_VERITY_OPT_RESTART "restart_on_corruption" +#define DM_VERITY_OPTS_MAX 1 + static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR); @@ -746,6 +748,44 @@ static void verity_dtr(struct dm_target *ti) kfree(v); } +static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v) +{ + int r; + unsigned argc; + struct dm_target *ti = v->ti; + const char *arg_name; + + static struct dm_arg _args[] = { + {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"}, + }; + + r = dm_read_arg_group(_args, as, &argc, &ti->error); + if (r) + return -EINVAL; + + if (!argc) + return 0; + + do { + arg_name = dm_shift_arg(as); + argc--; + + if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) { + v->mode = DM_VERITY_MODE_LOGGING; + continue; + + } else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) { + v->mode = DM_VERITY_MODE_RESTART; + continue; + } + + ti->error = "Unrecognized verity feature request"; + return -EINVAL; + } while (argc && !r); + + return r; +} + /* * Target parameters: * The current format is version 1. @@ -764,18 +804,13 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) { struct dm_verity *v; struct dm_arg_set as; - const char *opt_string; - unsigned int num, opt_params; + unsigned int num; unsigned long long num_ll; int r; int i; sector_t hash_position; char dummy; - static struct dm_arg _args[] = { - {0, 1, "Invalid number of feature args"}, - }; - v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL); if (!v) { ti->error = "Cannot allocate verity structure"; @@ -920,29 +955,9 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) as.argc = argc; as.argv = argv; - r = dm_read_arg_group(_args, &as, &opt_params, &ti->error); - if (r) + r = verity_parse_opt_args(&as, v); + if (r < 0) goto bad; - - while (opt_params) { - opt_params--; - opt_string = dm_shift_arg(&as); - if (!opt_string) { - ti->error = "Not enough feature arguments"; - r = -EINVAL; - goto bad; - } - - if (!strcasecmp(opt_string, DM_VERITY_OPT_LOGGING)) - v->mode = DM_VERITY_MODE_LOGGING; - else if (!strcasecmp(opt_string, DM_VERITY_OPT_RESTART)) - v->mode = DM_VERITY_MODE_RESTART; - else { - ti->error = "Invalid feature arguments"; - r = -EINVAL; - goto bad; - } - } } v->hash_per_block_bits = -- cgit v1.2.3 From 0e716b957a587fa4193623f86134ffae364628c8 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 3 Dec 2015 15:36:00 -0500 Subject: UPSTREAM: dm verity: move dm-verity.c to dm-verity-target.c Prepare for extending dm-verity with an optional object. Follows the naming convention used by other DM targets (e.g. dm-cache and dm-era). Bug: 27175947 Signed-off-by: Sami Tolvanen Signed-off-by: Mike Snitzer Signed-off-by: Badhri Jagan Sridharan Change-Id: Icc5c2802ebf2fce303d085ab5e52d825776cc436 --- drivers/md/Makefile | 1 + drivers/md/dm-verity-target.c | 1073 +++++++++++++++++++++++++++++++++++++++++ drivers/md/dm-verity.c | 1073 ----------------------------------------- 3 files changed, 1074 insertions(+), 1073 deletions(-) create mode 100644 drivers/md/dm-verity-target.c delete mode 100644 drivers/md/dm-verity.c diff --git a/drivers/md/Makefile b/drivers/md/Makefile index dba4db5985fb..452cdde2f6cb 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -15,6 +15,7 @@ dm-cache-y += dm-cache-target.o dm-cache-metadata.o dm-cache-policy.o dm-cache-mq-y += dm-cache-policy-mq.o dm-cache-cleaner-y += dm-cache-policy-cleaner.o dm-era-y += dm-era-target.o +dm-verity-y += dm-verity-target.o md-mod-y += md.o bitmap.o raid456-y += raid5.o diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c new file mode 100644 index 000000000000..e75d1dba28f9 --- /dev/null +++ b/drivers/md/dm-verity-target.c @@ -0,0 +1,1073 @@ +/* + * Copyright (C) 2012 Red Hat, Inc. + * + * Author: Mikulas Patocka + * + * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors + * + * This file is released under the GPLv2. + * + * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set + * default prefetch value. Data are read in "prefetch_cluster" chunks from the + * hash device. Setting this greatly improves performance when data and hash + * are on the same disk on different partitions on devices with poor random + * access behavior. + */ + +#include "dm-bufio.h" + +#include +#include +#include +#include + +#define DM_MSG_PREFIX "verity" + +#define DM_VERITY_ENV_LENGTH 42 +#define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR" + +#define DM_VERITY_IO_VEC_INLINE 16 +#define DM_VERITY_MEMPOOL_SIZE 4 +#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 + +#define DM_VERITY_MAX_LEVELS 63 +#define DM_VERITY_MAX_CORRUPTED_ERRS 100 + +#define DM_VERITY_OPT_LOGGING "ignore_corruption" +#define DM_VERITY_OPT_RESTART "restart_on_corruption" + +#define DM_VERITY_OPTS_MAX 1 + +static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; + +module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR); + +enum verity_mode { + DM_VERITY_MODE_EIO, + DM_VERITY_MODE_LOGGING, + DM_VERITY_MODE_RESTART +}; + +enum verity_block_type { + DM_VERITY_BLOCK_TYPE_DATA, + DM_VERITY_BLOCK_TYPE_METADATA +}; + +struct dm_verity { + struct dm_dev *data_dev; + struct dm_dev *hash_dev; + struct dm_target *ti; + struct dm_bufio_client *bufio; + char *alg_name; + struct crypto_shash *tfm; + u8 *root_digest; /* digest of the root block */ + u8 *salt; /* salt: its size is salt_size */ + unsigned salt_size; + sector_t data_start; /* data offset in 512-byte sectors */ + sector_t hash_start; /* hash start in blocks */ + sector_t data_blocks; /* the number of data blocks */ + sector_t hash_blocks; /* the number of hash blocks */ + unsigned char data_dev_block_bits; /* log2(data blocksize) */ + unsigned char hash_dev_block_bits; /* log2(hash blocksize) */ + unsigned char hash_per_block_bits; /* log2(hashes in hash block) */ + unsigned char levels; /* the number of tree levels */ + unsigned char version; + unsigned digest_size; /* digest size for the current hash algorithm */ + unsigned shash_descsize;/* the size of temporary space for crypto */ + int hash_failed; /* set to 1 if hash of any block failed */ + enum verity_mode mode; /* mode for handling verification errors */ + unsigned corrupted_errs;/* Number of errors for corrupted blocks */ + + mempool_t *vec_mempool; /* mempool of bio vector */ + + struct workqueue_struct *verify_wq; + + /* starting blocks for each tree level. 0 is the lowest level. */ + sector_t hash_level_block[DM_VERITY_MAX_LEVELS]; +}; + +struct dm_verity_io { + struct dm_verity *v; + + /* original values of bio->bi_end_io and bio->bi_private */ + bio_end_io_t *orig_bi_end_io; + void *orig_bi_private; + + sector_t block; + unsigned n_blocks; + + struct bvec_iter iter; + + struct work_struct work; + + /* + * Three variably-size fields follow this struct: + * + * u8 hash_desc[v->shash_descsize]; + * u8 real_digest[v->digest_size]; + * u8 want_digest[v->digest_size]; + * + * To access them use: io_hash_desc(), io_real_digest() and io_want_digest(). + */ +}; + +struct dm_verity_prefetch_work { + struct work_struct work; + struct dm_verity *v; + sector_t block; + unsigned n_blocks; +}; + +static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io) +{ + return (struct shash_desc *)(io + 1); +} + +static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io) +{ + return (u8 *)(io + 1) + v->shash_descsize; +} + +static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io) +{ + return (u8 *)(io + 1) + v->shash_descsize + v->digest_size; +} + +/* + * Auxiliary structure appended to each dm-bufio buffer. If the value + * hash_verified is nonzero, hash of the block has been verified. + * + * The variable hash_verified is set to 0 when allocating the buffer, then + * it can be changed to 1 and it is never reset to 0 again. + * + * There is no lock around this value, a race condition can at worst cause + * that multiple processes verify the hash of the same buffer simultaneously + * and write 1 to hash_verified simultaneously. + * This condition is harmless, so we don't need locking. + */ +struct buffer_aux { + int hash_verified; +}; + +/* + * Initialize struct buffer_aux for a freshly created buffer. + */ +static void dm_bufio_alloc_callback(struct dm_buffer *buf) +{ + struct buffer_aux *aux = dm_bufio_get_aux_data(buf); + + aux->hash_verified = 0; +} + +/* + * Translate input sector number to the sector number on the target device. + */ +static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector) +{ + return v->data_start + dm_target_offset(v->ti, bi_sector); +} + +/* + * Return hash position of a specified block at a specified tree level + * (0 is the lowest level). + * The lowest "hash_per_block_bits"-bits of the result denote hash position + * inside a hash block. The remaining bits denote location of the hash block. + */ +static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, + int level) +{ + return block >> (level * v->hash_per_block_bits); +} + +/* + * Wrapper for crypto_shash_init, which handles verity salting. + */ +static int verity_hash_init(struct dm_verity *v, struct shash_desc *desc) +{ + int r; + + desc->tfm = v->tfm; + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; + + r = crypto_shash_init(desc); + + if (unlikely(r < 0)) { + DMERR("crypto_shash_init failed: %d", r); + return r; + } + + if (likely(v->version >= 1)) { + r = crypto_shash_update(desc, v->salt, v->salt_size); + + if (unlikely(r < 0)) { + DMERR("crypto_shash_update failed: %d", r); + return r; + } + } + + return 0; +} + +static int verity_hash_update(struct dm_verity *v, struct shash_desc *desc, + const u8 *data, size_t len) +{ + int r = crypto_shash_update(desc, data, len); + + if (unlikely(r < 0)) + DMERR("crypto_shash_update failed: %d", r); + + return r; +} + +static int verity_hash_final(struct dm_verity *v, struct shash_desc *desc, + u8 *digest) +{ + int r; + + if (unlikely(!v->version)) { + r = crypto_shash_update(desc, v->salt, v->salt_size); + + if (r < 0) { + DMERR("crypto_shash_update failed: %d", r); + return r; + } + } + + r = crypto_shash_final(desc, digest); + + if (unlikely(r < 0)) + DMERR("crypto_shash_final failed: %d", r); + + return r; +} + +static int verity_hash(struct dm_verity *v, struct shash_desc *desc, + const u8 *data, size_t len, u8 *digest) +{ + int r; + + r = verity_hash_init(v, desc); + if (unlikely(r < 0)) + return r; + + r = verity_hash_update(v, desc, data, len); + if (unlikely(r < 0)) + return r; + + return verity_hash_final(v, desc, digest); +} + +static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, + sector_t *hash_block, unsigned *offset) +{ + sector_t position = verity_position_at_level(v, block, level); + unsigned idx; + + *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits); + + if (!offset) + return; + + idx = position & ((1 << v->hash_per_block_bits) - 1); + if (!v->version) + *offset = idx * v->digest_size; + else + *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits); +} + +/* + * Handle verification errors. + */ +static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, + unsigned long long block) +{ + char verity_env[DM_VERITY_ENV_LENGTH]; + char *envp[] = { verity_env, NULL }; + const char *type_str = ""; + struct mapped_device *md = dm_table_get_md(v->ti->table); + + /* Corruption should be visible in device status in all modes */ + v->hash_failed = 1; + + if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS) + goto out; + + v->corrupted_errs++; + + switch (type) { + case DM_VERITY_BLOCK_TYPE_DATA: + type_str = "data"; + break; + case DM_VERITY_BLOCK_TYPE_METADATA: + type_str = "metadata"; + break; + default: + BUG(); + } + + DMERR("%s: %s block %llu is corrupted", v->data_dev->name, type_str, + block); + + if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) + DMERR("%s: reached maximum errors", v->data_dev->name); + + snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu", + DM_VERITY_ENV_VAR_NAME, type, block); + + kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp); + +out: + if (v->mode == DM_VERITY_MODE_LOGGING) + return 0; + + if (v->mode == DM_VERITY_MODE_RESTART) + kernel_restart("dm-verity device corrupted"); + + return 1; +} + +/* + * Verify hash of a metadata block pertaining to the specified data block + * ("block" argument) at a specified level ("level" argument). + * + * On successful return, io_want_digest(v, io) contains the hash value for + * a lower tree level or for the data block (if we're at the lowest leve). + * + * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned. + * If "skip_unverified" is false, unverified buffer is hashed and verified + * against current value of io_want_digest(v, io). + */ +static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, + sector_t block, int level, bool skip_unverified, + u8 *want_digest) +{ + struct dm_buffer *buf; + struct buffer_aux *aux; + u8 *data; + int r; + sector_t hash_block; + unsigned offset; + + verity_hash_at_level(v, block, level, &hash_block, &offset); + + data = dm_bufio_read(v->bufio, hash_block, &buf); + if (unlikely(IS_ERR(data))) + return PTR_ERR(data); + + aux = dm_bufio_get_aux_data(buf); + + if (!aux->hash_verified) { + if (skip_unverified) { + r = 1; + goto release_ret_r; + } + + r = verity_hash(v, io_hash_desc(v, io), + data, 1 << v->hash_dev_block_bits, + io_real_digest(v, io)); + if (unlikely(r < 0)) + goto release_ret_r; + + if (likely(memcmp(io_real_digest(v, io), want_digest, + v->digest_size) == 0)) + aux->hash_verified = 1; + else if (verity_handle_err(v, + DM_VERITY_BLOCK_TYPE_METADATA, + hash_block)) { + r = -EIO; + goto release_ret_r; + } + } + + data += offset; + memcpy(want_digest, data, v->digest_size); + r = 0; + +release_ret_r: + dm_bufio_release(buf); + return r; +} + +/* + * Find a hash for a given block, write it to digest and verify the integrity + * of the hash tree if necessary. + */ +static int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, + sector_t block, u8 *digest) +{ + int i; + int r; + + if (likely(v->levels)) { + /* + * First, we try to get the requested hash for + * the current block. If the hash block itself is + * verified, zero is returned. If it isn't, this + * function returns 1 and we fall back to whole + * chain verification. + */ + r = verity_verify_level(v, io, block, 0, true, digest); + if (likely(r <= 0)) + return r; + } + + memcpy(digest, v->root_digest, v->digest_size); + + for (i = v->levels - 1; i >= 0; i--) { + r = verity_verify_level(v, io, block, i, false, digest); + if (unlikely(r)) + return r; + } + + return 0; +} + +/* + * Verify one "dm_verity_io" structure. + */ +static int verity_verify_io(struct dm_verity_io *io) +{ + struct dm_verity *v = io->v; + struct bio *bio = dm_bio_from_per_bio_data(io, + v->ti->per_bio_data_size); + unsigned b; + + for (b = 0; b < io->n_blocks; b++) { + int r; + unsigned todo; + struct shash_desc *desc = io_hash_desc(v, io); + + r = verity_hash_for_block(v, io, io->block + b, + io_want_digest(v, io)); + if (unlikely(r < 0)) + return r; + + r = verity_hash_init(v, desc); + if (unlikely(r < 0)) + return r; + + todo = 1 << v->data_dev_block_bits; + do { + u8 *page; + unsigned len; + struct bio_vec bv = bio_iter_iovec(bio, io->iter); + + page = kmap_atomic(bv.bv_page); + len = bv.bv_len; + if (likely(len >= todo)) + len = todo; + r = verity_hash_update(v, desc, page + bv.bv_offset, + len); + kunmap_atomic(page); + + if (unlikely(r < 0)) + return r; + + bio_advance_iter(bio, &io->iter, len); + todo -= len; + } while (todo); + + r = verity_hash_final(v, desc, io_real_digest(v, io)); + if (unlikely(r < 0)) + return r; + + if (likely(memcmp(io_real_digest(v, io), + io_want_digest(v, io), v->digest_size) == 0)) + continue; + else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, + io->block + b)) + return -EIO; + } + + return 0; +} + +/* + * End one "io" structure with a given error. + */ +static void verity_finish_io(struct dm_verity_io *io, int error) +{ + struct dm_verity *v = io->v; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size); + + bio->bi_end_io = io->orig_bi_end_io; + bio->bi_private = io->orig_bi_private; + + bio_endio_nodec(bio, error); +} + +static void verity_work(struct work_struct *w) +{ + struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); + + verity_finish_io(io, verity_verify_io(io)); +} + +static void verity_end_io(struct bio *bio, int error) +{ + struct dm_verity_io *io = bio->bi_private; + + if (error) { + verity_finish_io(io, error); + return; + } + + INIT_WORK(&io->work, verity_work); + queue_work(io->v->verify_wq, &io->work); +} + +/* + * Prefetch buffers for the specified io. + * The root buffer is not prefetched, it is assumed that it will be cached + * all the time. + */ +static void verity_prefetch_io(struct work_struct *work) +{ + struct dm_verity_prefetch_work *pw = + container_of(work, struct dm_verity_prefetch_work, work); + struct dm_verity *v = pw->v; + int i; + + for (i = v->levels - 2; i >= 0; i--) { + sector_t hash_block_start; + sector_t hash_block_end; + verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL); + verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL); + if (!i) { + unsigned cluster = ACCESS_ONCE(dm_verity_prefetch_cluster); + + cluster >>= v->data_dev_block_bits; + if (unlikely(!cluster)) + goto no_prefetch_cluster; + + if (unlikely(cluster & (cluster - 1))) + cluster = 1 << __fls(cluster); + + hash_block_start &= ~(sector_t)(cluster - 1); + hash_block_end |= cluster - 1; + if (unlikely(hash_block_end >= v->hash_blocks)) + hash_block_end = v->hash_blocks - 1; + } +no_prefetch_cluster: + dm_bufio_prefetch(v->bufio, hash_block_start, + hash_block_end - hash_block_start + 1); + } + + kfree(pw); +} + +static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) +{ + struct dm_verity_prefetch_work *pw; + + pw = kmalloc(sizeof(struct dm_verity_prefetch_work), + GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); + + if (!pw) + return; + + INIT_WORK(&pw->work, verity_prefetch_io); + pw->v = v; + pw->block = io->block; + pw->n_blocks = io->n_blocks; + queue_work(v->verify_wq, &pw->work); +} + +/* + * Bio map function. It allocates dm_verity_io structure and bio vector and + * fills them. Then it issues prefetches and the I/O. + */ +static int verity_map(struct dm_target *ti, struct bio *bio) +{ + struct dm_verity *v = ti->private; + struct dm_verity_io *io; + + bio->bi_bdev = v->data_dev->bdev; + bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector); + + if (((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) & + ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) { + DMERR_LIMIT("unaligned io"); + return -EIO; + } + + if (bio_end_sector(bio) >> + (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) { + DMERR_LIMIT("io out of range"); + return -EIO; + } + + if (bio_data_dir(bio) == WRITE) + return -EIO; + + io = dm_per_bio_data(bio, ti->per_bio_data_size); + io->v = v; + io->orig_bi_end_io = bio->bi_end_io; + io->orig_bi_private = bio->bi_private; + io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT); + io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits; + + bio->bi_end_io = verity_end_io; + bio->bi_private = io; + io->iter = bio->bi_iter; + + verity_submit_prefetch(v, io); + + generic_make_request(bio); + + return DM_MAPIO_SUBMITTED; +} + +/* + * Status: V (valid) or C (corruption found) + */ +static void verity_status(struct dm_target *ti, status_type_t type, + unsigned status_flags, char *result, unsigned maxlen) +{ + struct dm_verity *v = ti->private; + unsigned sz = 0; + unsigned x; + + switch (type) { + case STATUSTYPE_INFO: + DMEMIT("%c", v->hash_failed ? 'C' : 'V'); + break; + case STATUSTYPE_TABLE: + DMEMIT("%u %s %s %u %u %llu %llu %s ", + v->version, + v->data_dev->name, + v->hash_dev->name, + 1 << v->data_dev_block_bits, + 1 << v->hash_dev_block_bits, + (unsigned long long)v->data_blocks, + (unsigned long long)v->hash_start, + v->alg_name + ); + for (x = 0; x < v->digest_size; x++) + DMEMIT("%02x", v->root_digest[x]); + DMEMIT(" "); + if (!v->salt_size) + DMEMIT("-"); + else + for (x = 0; x < v->salt_size; x++) + DMEMIT("%02x", v->salt[x]); + if (v->mode != DM_VERITY_MODE_EIO) { + DMEMIT(" 1 "); + switch (v->mode) { + case DM_VERITY_MODE_LOGGING: + DMEMIT(DM_VERITY_OPT_LOGGING); + break; + case DM_VERITY_MODE_RESTART: + DMEMIT(DM_VERITY_OPT_RESTART); + break; + default: + BUG(); + } + } + break; + } +} + +static int verity_ioctl(struct dm_target *ti, unsigned cmd, + unsigned long arg) +{ + struct dm_verity *v = ti->private; + int r = 0; + + if (v->data_start || + ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) + r = scsi_verify_blk_ioctl(NULL, cmd); + + return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode, + cmd, arg); +} + +static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm, + struct bio_vec *biovec, int max_size) +{ + struct dm_verity *v = ti->private; + struct request_queue *q = bdev_get_queue(v->data_dev->bdev); + + if (!q->merge_bvec_fn) + return max_size; + + bvm->bi_bdev = v->data_dev->bdev; + bvm->bi_sector = verity_map_sector(v, bvm->bi_sector); + + return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); +} + +static int verity_iterate_devices(struct dm_target *ti, + iterate_devices_callout_fn fn, void *data) +{ + struct dm_verity *v = ti->private; + + return fn(ti, v->data_dev, v->data_start, ti->len, data); +} + +static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) +{ + struct dm_verity *v = ti->private; + + if (limits->logical_block_size < 1 << v->data_dev_block_bits) + limits->logical_block_size = 1 << v->data_dev_block_bits; + + if (limits->physical_block_size < 1 << v->data_dev_block_bits) + limits->physical_block_size = 1 << v->data_dev_block_bits; + + blk_limits_io_min(limits, limits->logical_block_size); +} + +static void verity_dtr(struct dm_target *ti) +{ + struct dm_verity *v = ti->private; + + if (v->verify_wq) + destroy_workqueue(v->verify_wq); + + if (v->vec_mempool) + mempool_destroy(v->vec_mempool); + + if (v->bufio) + dm_bufio_client_destroy(v->bufio); + + kfree(v->salt); + kfree(v->root_digest); + + if (v->tfm) + crypto_free_shash(v->tfm); + + kfree(v->alg_name); + + if (v->hash_dev) + dm_put_device(ti, v->hash_dev); + + if (v->data_dev) + dm_put_device(ti, v->data_dev); + + kfree(v); +} + +static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v) +{ + int r; + unsigned argc; + struct dm_target *ti = v->ti; + const char *arg_name; + + static struct dm_arg _args[] = { + {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"}, + }; + + r = dm_read_arg_group(_args, as, &argc, &ti->error); + if (r) + return -EINVAL; + + if (!argc) + return 0; + + do { + arg_name = dm_shift_arg(as); + argc--; + + if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) { + v->mode = DM_VERITY_MODE_LOGGING; + continue; + + } else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) { + v->mode = DM_VERITY_MODE_RESTART; + continue; + } + + ti->error = "Unrecognized verity feature request"; + return -EINVAL; + } while (argc && !r); + + return r; +} + +/* + * Target parameters: + * The current format is version 1. + * Vsn 0 is compatible with original Chromium OS releases. + * + * + * + * + * + * + * + * + * Hex string or "-" if no salt. + */ +static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) +{ + struct dm_verity *v; + struct dm_arg_set as; + unsigned int num; + unsigned long long num_ll; + int r; + int i; + sector_t hash_position; + char dummy; + + v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL); + if (!v) { + ti->error = "Cannot allocate verity structure"; + return -ENOMEM; + } + ti->private = v; + v->ti = ti; + + if ((dm_table_get_mode(ti->table) & ~FMODE_READ)) { + ti->error = "Device must be readonly"; + r = -EINVAL; + goto bad; + } + + if (argc < 10) { + ti->error = "Not enough arguments"; + r = -EINVAL; + goto bad; + } + + if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 || + num > 1) { + ti->error = "Invalid version"; + r = -EINVAL; + goto bad; + } + v->version = num; + + r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev); + if (r) { + ti->error = "Data device lookup failed"; + goto bad; + } + + r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev); + if (r) { + ti->error = "Data device lookup failed"; + goto bad; + } + + if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 || + !num || (num & (num - 1)) || + num < bdev_logical_block_size(v->data_dev->bdev) || + num > PAGE_SIZE) { + ti->error = "Invalid data device block size"; + r = -EINVAL; + goto bad; + } + v->data_dev_block_bits = __ffs(num); + + if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 || + !num || (num & (num - 1)) || + num < bdev_logical_block_size(v->hash_dev->bdev) || + num > INT_MAX) { + ti->error = "Invalid hash device block size"; + r = -EINVAL; + goto bad; + } + v->hash_dev_block_bits = __ffs(num); + + if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 || + (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) + >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) { + ti->error = "Invalid data blocks"; + r = -EINVAL; + goto bad; + } + v->data_blocks = num_ll; + + if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) { + ti->error = "Data device is too small"; + r = -EINVAL; + goto bad; + } + + if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 || + (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT)) + >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) { + ti->error = "Invalid hash start"; + r = -EINVAL; + goto bad; + } + v->hash_start = num_ll; + + v->alg_name = kstrdup(argv[7], GFP_KERNEL); + if (!v->alg_name) { + ti->error = "Cannot allocate algorithm name"; + r = -ENOMEM; + goto bad; + } + + v->tfm = crypto_alloc_shash(v->alg_name, 0, 0); + if (IS_ERR(v->tfm)) { + ti->error = "Cannot initialize hash function"; + r = PTR_ERR(v->tfm); + v->tfm = NULL; + goto bad; + } + v->digest_size = crypto_shash_digestsize(v->tfm); + if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) { + ti->error = "Digest size too big"; + r = -EINVAL; + goto bad; + } + v->shash_descsize = + sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm); + + v->root_digest = kmalloc(v->digest_size, GFP_KERNEL); + if (!v->root_digest) { + ti->error = "Cannot allocate root digest"; + r = -ENOMEM; + goto bad; + } + if (strlen(argv[8]) != v->digest_size * 2 || + hex2bin(v->root_digest, argv[8], v->digest_size)) { + ti->error = "Invalid root digest"; + r = -EINVAL; + goto bad; + } + + if (strcmp(argv[9], "-")) { + v->salt_size = strlen(argv[9]) / 2; + v->salt = kmalloc(v->salt_size, GFP_KERNEL); + if (!v->salt) { + ti->error = "Cannot allocate salt"; + r = -ENOMEM; + goto bad; + } + if (strlen(argv[9]) != v->salt_size * 2 || + hex2bin(v->salt, argv[9], v->salt_size)) { + ti->error = "Invalid salt"; + r = -EINVAL; + goto bad; + } + } + + argv += 10; + argc -= 10; + + /* Optional parameters */ + if (argc) { + as.argc = argc; + as.argv = argv; + + r = verity_parse_opt_args(&as, v); + if (r < 0) + goto bad; + } + + v->hash_per_block_bits = + __fls((1 << v->hash_dev_block_bits) / v->digest_size); + + v->levels = 0; + if (v->data_blocks) + while (v->hash_per_block_bits * v->levels < 64 && + (unsigned long long)(v->data_blocks - 1) >> + (v->hash_per_block_bits * v->levels)) + v->levels++; + + if (v->levels > DM_VERITY_MAX_LEVELS) { + ti->error = "Too many tree levels"; + r = -E2BIG; + goto bad; + } + + hash_position = v->hash_start; + for (i = v->levels - 1; i >= 0; i--) { + sector_t s; + v->hash_level_block[i] = hash_position; + s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1) + >> ((i + 1) * v->hash_per_block_bits); + if (hash_position + s < hash_position) { + ti->error = "Hash device offset overflow"; + r = -E2BIG; + goto bad; + } + hash_position += s; + } + v->hash_blocks = hash_position; + + v->bufio = dm_bufio_client_create(v->hash_dev->bdev, + 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), + dm_bufio_alloc_callback, NULL); + if (IS_ERR(v->bufio)) { + ti->error = "Cannot initialize dm-bufio"; + r = PTR_ERR(v->bufio); + v->bufio = NULL; + goto bad; + } + + if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) { + ti->error = "Hash device is too small"; + r = -E2BIG; + goto bad; + } + + ti->per_bio_data_size = roundup(sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2, __alignof__(struct dm_verity_io)); + + v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE, + BIO_MAX_PAGES * sizeof(struct bio_vec)); + if (!v->vec_mempool) { + ti->error = "Cannot allocate vector mempool"; + r = -ENOMEM; + goto bad; + } + + /* WQ_UNBOUND greatly improves performance when running on ramdisk */ + v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus()); + if (!v->verify_wq) { + ti->error = "Cannot allocate workqueue"; + r = -ENOMEM; + goto bad; + } + + return 0; + +bad: + verity_dtr(ti); + + return r; +} + +static struct target_type verity_target = { + .name = "verity", + .version = {1, 2, 0}, + .module = THIS_MODULE, + .ctr = verity_ctr, + .dtr = verity_dtr, + .map = verity_map, + .status = verity_status, + .ioctl = verity_ioctl, + .merge = verity_merge, + .iterate_devices = verity_iterate_devices, + .io_hints = verity_io_hints, +}; + +static int __init dm_verity_init(void) +{ + int r; + + r = dm_register_target(&verity_target); + if (r < 0) + DMERR("register failed %d", r); + + return r; +} + +static void __exit dm_verity_exit(void) +{ + dm_unregister_target(&verity_target); +} + +module_init(dm_verity_init); +module_exit(dm_verity_exit); + +MODULE_AUTHOR("Mikulas Patocka "); +MODULE_AUTHOR("Mandeep Baines "); +MODULE_AUTHOR("Will Drewry "); +MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); +MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-verity.c b/drivers/md/dm-verity.c deleted file mode 100644 index e75d1dba28f9..000000000000 --- a/drivers/md/dm-verity.c +++ /dev/null @@ -1,1073 +0,0 @@ -/* - * Copyright (C) 2012 Red Hat, Inc. - * - * Author: Mikulas Patocka - * - * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors - * - * This file is released under the GPLv2. - * - * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set - * default prefetch value. Data are read in "prefetch_cluster" chunks from the - * hash device. Setting this greatly improves performance when data and hash - * are on the same disk on different partitions on devices with poor random - * access behavior. - */ - -#include "dm-bufio.h" - -#include -#include -#include -#include - -#define DM_MSG_PREFIX "verity" - -#define DM_VERITY_ENV_LENGTH 42 -#define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR" - -#define DM_VERITY_IO_VEC_INLINE 16 -#define DM_VERITY_MEMPOOL_SIZE 4 -#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 - -#define DM_VERITY_MAX_LEVELS 63 -#define DM_VERITY_MAX_CORRUPTED_ERRS 100 - -#define DM_VERITY_OPT_LOGGING "ignore_corruption" -#define DM_VERITY_OPT_RESTART "restart_on_corruption" - -#define DM_VERITY_OPTS_MAX 1 - -static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; - -module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR); - -enum verity_mode { - DM_VERITY_MODE_EIO, - DM_VERITY_MODE_LOGGING, - DM_VERITY_MODE_RESTART -}; - -enum verity_block_type { - DM_VERITY_BLOCK_TYPE_DATA, - DM_VERITY_BLOCK_TYPE_METADATA -}; - -struct dm_verity { - struct dm_dev *data_dev; - struct dm_dev *hash_dev; - struct dm_target *ti; - struct dm_bufio_client *bufio; - char *alg_name; - struct crypto_shash *tfm; - u8 *root_digest; /* digest of the root block */ - u8 *salt; /* salt: its size is salt_size */ - unsigned salt_size; - sector_t data_start; /* data offset in 512-byte sectors */ - sector_t hash_start; /* hash start in blocks */ - sector_t data_blocks; /* the number of data blocks */ - sector_t hash_blocks; /* the number of hash blocks */ - unsigned char data_dev_block_bits; /* log2(data blocksize) */ - unsigned char hash_dev_block_bits; /* log2(hash blocksize) */ - unsigned char hash_per_block_bits; /* log2(hashes in hash block) */ - unsigned char levels; /* the number of tree levels */ - unsigned char version; - unsigned digest_size; /* digest size for the current hash algorithm */ - unsigned shash_descsize;/* the size of temporary space for crypto */ - int hash_failed; /* set to 1 if hash of any block failed */ - enum verity_mode mode; /* mode for handling verification errors */ - unsigned corrupted_errs;/* Number of errors for corrupted blocks */ - - mempool_t *vec_mempool; /* mempool of bio vector */ - - struct workqueue_struct *verify_wq; - - /* starting blocks for each tree level. 0 is the lowest level. */ - sector_t hash_level_block[DM_VERITY_MAX_LEVELS]; -}; - -struct dm_verity_io { - struct dm_verity *v; - - /* original values of bio->bi_end_io and bio->bi_private */ - bio_end_io_t *orig_bi_end_io; - void *orig_bi_private; - - sector_t block; - unsigned n_blocks; - - struct bvec_iter iter; - - struct work_struct work; - - /* - * Three variably-size fields follow this struct: - * - * u8 hash_desc[v->shash_descsize]; - * u8 real_digest[v->digest_size]; - * u8 want_digest[v->digest_size]; - * - * To access them use: io_hash_desc(), io_real_digest() and io_want_digest(). - */ -}; - -struct dm_verity_prefetch_work { - struct work_struct work; - struct dm_verity *v; - sector_t block; - unsigned n_blocks; -}; - -static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io) -{ - return (struct shash_desc *)(io + 1); -} - -static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io) -{ - return (u8 *)(io + 1) + v->shash_descsize; -} - -static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io) -{ - return (u8 *)(io + 1) + v->shash_descsize + v->digest_size; -} - -/* - * Auxiliary structure appended to each dm-bufio buffer. If the value - * hash_verified is nonzero, hash of the block has been verified. - * - * The variable hash_verified is set to 0 when allocating the buffer, then - * it can be changed to 1 and it is never reset to 0 again. - * - * There is no lock around this value, a race condition can at worst cause - * that multiple processes verify the hash of the same buffer simultaneously - * and write 1 to hash_verified simultaneously. - * This condition is harmless, so we don't need locking. - */ -struct buffer_aux { - int hash_verified; -}; - -/* - * Initialize struct buffer_aux for a freshly created buffer. - */ -static void dm_bufio_alloc_callback(struct dm_buffer *buf) -{ - struct buffer_aux *aux = dm_bufio_get_aux_data(buf); - - aux->hash_verified = 0; -} - -/* - * Translate input sector number to the sector number on the target device. - */ -static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector) -{ - return v->data_start + dm_target_offset(v->ti, bi_sector); -} - -/* - * Return hash position of a specified block at a specified tree level - * (0 is the lowest level). - * The lowest "hash_per_block_bits"-bits of the result denote hash position - * inside a hash block. The remaining bits denote location of the hash block. - */ -static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, - int level) -{ - return block >> (level * v->hash_per_block_bits); -} - -/* - * Wrapper for crypto_shash_init, which handles verity salting. - */ -static int verity_hash_init(struct dm_verity *v, struct shash_desc *desc) -{ - int r; - - desc->tfm = v->tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - - r = crypto_shash_init(desc); - - if (unlikely(r < 0)) { - DMERR("crypto_shash_init failed: %d", r); - return r; - } - - if (likely(v->version >= 1)) { - r = crypto_shash_update(desc, v->salt, v->salt_size); - - if (unlikely(r < 0)) { - DMERR("crypto_shash_update failed: %d", r); - return r; - } - } - - return 0; -} - -static int verity_hash_update(struct dm_verity *v, struct shash_desc *desc, - const u8 *data, size_t len) -{ - int r = crypto_shash_update(desc, data, len); - - if (unlikely(r < 0)) - DMERR("crypto_shash_update failed: %d", r); - - return r; -} - -static int verity_hash_final(struct dm_verity *v, struct shash_desc *desc, - u8 *digest) -{ - int r; - - if (unlikely(!v->version)) { - r = crypto_shash_update(desc, v->salt, v->salt_size); - - if (r < 0) { - DMERR("crypto_shash_update failed: %d", r); - return r; - } - } - - r = crypto_shash_final(desc, digest); - - if (unlikely(r < 0)) - DMERR("crypto_shash_final failed: %d", r); - - return r; -} - -static int verity_hash(struct dm_verity *v, struct shash_desc *desc, - const u8 *data, size_t len, u8 *digest) -{ - int r; - - r = verity_hash_init(v, desc); - if (unlikely(r < 0)) - return r; - - r = verity_hash_update(v, desc, data, len); - if (unlikely(r < 0)) - return r; - - return verity_hash_final(v, desc, digest); -} - -static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, - sector_t *hash_block, unsigned *offset) -{ - sector_t position = verity_position_at_level(v, block, level); - unsigned idx; - - *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits); - - if (!offset) - return; - - idx = position & ((1 << v->hash_per_block_bits) - 1); - if (!v->version) - *offset = idx * v->digest_size; - else - *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits); -} - -/* - * Handle verification errors. - */ -static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, - unsigned long long block) -{ - char verity_env[DM_VERITY_ENV_LENGTH]; - char *envp[] = { verity_env, NULL }; - const char *type_str = ""; - struct mapped_device *md = dm_table_get_md(v->ti->table); - - /* Corruption should be visible in device status in all modes */ - v->hash_failed = 1; - - if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS) - goto out; - - v->corrupted_errs++; - - switch (type) { - case DM_VERITY_BLOCK_TYPE_DATA: - type_str = "data"; - break; - case DM_VERITY_BLOCK_TYPE_METADATA: - type_str = "metadata"; - break; - default: - BUG(); - } - - DMERR("%s: %s block %llu is corrupted", v->data_dev->name, type_str, - block); - - if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) - DMERR("%s: reached maximum errors", v->data_dev->name); - - snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu", - DM_VERITY_ENV_VAR_NAME, type, block); - - kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp); - -out: - if (v->mode == DM_VERITY_MODE_LOGGING) - return 0; - - if (v->mode == DM_VERITY_MODE_RESTART) - kernel_restart("dm-verity device corrupted"); - - return 1; -} - -/* - * Verify hash of a metadata block pertaining to the specified data block - * ("block" argument) at a specified level ("level" argument). - * - * On successful return, io_want_digest(v, io) contains the hash value for - * a lower tree level or for the data block (if we're at the lowest leve). - * - * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned. - * If "skip_unverified" is false, unverified buffer is hashed and verified - * against current value of io_want_digest(v, io). - */ -static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, - sector_t block, int level, bool skip_unverified, - u8 *want_digest) -{ - struct dm_buffer *buf; - struct buffer_aux *aux; - u8 *data; - int r; - sector_t hash_block; - unsigned offset; - - verity_hash_at_level(v, block, level, &hash_block, &offset); - - data = dm_bufio_read(v->bufio, hash_block, &buf); - if (unlikely(IS_ERR(data))) - return PTR_ERR(data); - - aux = dm_bufio_get_aux_data(buf); - - if (!aux->hash_verified) { - if (skip_unverified) { - r = 1; - goto release_ret_r; - } - - r = verity_hash(v, io_hash_desc(v, io), - data, 1 << v->hash_dev_block_bits, - io_real_digest(v, io)); - if (unlikely(r < 0)) - goto release_ret_r; - - if (likely(memcmp(io_real_digest(v, io), want_digest, - v->digest_size) == 0)) - aux->hash_verified = 1; - else if (verity_handle_err(v, - DM_VERITY_BLOCK_TYPE_METADATA, - hash_block)) { - r = -EIO; - goto release_ret_r; - } - } - - data += offset; - memcpy(want_digest, data, v->digest_size); - r = 0; - -release_ret_r: - dm_bufio_release(buf); - return r; -} - -/* - * Find a hash for a given block, write it to digest and verify the integrity - * of the hash tree if necessary. - */ -static int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, - sector_t block, u8 *digest) -{ - int i; - int r; - - if (likely(v->levels)) { - /* - * First, we try to get the requested hash for - * the current block. If the hash block itself is - * verified, zero is returned. If it isn't, this - * function returns 1 and we fall back to whole - * chain verification. - */ - r = verity_verify_level(v, io, block, 0, true, digest); - if (likely(r <= 0)) - return r; - } - - memcpy(digest, v->root_digest, v->digest_size); - - for (i = v->levels - 1; i >= 0; i--) { - r = verity_verify_level(v, io, block, i, false, digest); - if (unlikely(r)) - return r; - } - - return 0; -} - -/* - * Verify one "dm_verity_io" structure. - */ -static int verity_verify_io(struct dm_verity_io *io) -{ - struct dm_verity *v = io->v; - struct bio *bio = dm_bio_from_per_bio_data(io, - v->ti->per_bio_data_size); - unsigned b; - - for (b = 0; b < io->n_blocks; b++) { - int r; - unsigned todo; - struct shash_desc *desc = io_hash_desc(v, io); - - r = verity_hash_for_block(v, io, io->block + b, - io_want_digest(v, io)); - if (unlikely(r < 0)) - return r; - - r = verity_hash_init(v, desc); - if (unlikely(r < 0)) - return r; - - todo = 1 << v->data_dev_block_bits; - do { - u8 *page; - unsigned len; - struct bio_vec bv = bio_iter_iovec(bio, io->iter); - - page = kmap_atomic(bv.bv_page); - len = bv.bv_len; - if (likely(len >= todo)) - len = todo; - r = verity_hash_update(v, desc, page + bv.bv_offset, - len); - kunmap_atomic(page); - - if (unlikely(r < 0)) - return r; - - bio_advance_iter(bio, &io->iter, len); - todo -= len; - } while (todo); - - r = verity_hash_final(v, desc, io_real_digest(v, io)); - if (unlikely(r < 0)) - return r; - - if (likely(memcmp(io_real_digest(v, io), - io_want_digest(v, io), v->digest_size) == 0)) - continue; - else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, - io->block + b)) - return -EIO; - } - - return 0; -} - -/* - * End one "io" structure with a given error. - */ -static void verity_finish_io(struct dm_verity_io *io, int error) -{ - struct dm_verity *v = io->v; - struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size); - - bio->bi_end_io = io->orig_bi_end_io; - bio->bi_private = io->orig_bi_private; - - bio_endio_nodec(bio, error); -} - -static void verity_work(struct work_struct *w) -{ - struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); - - verity_finish_io(io, verity_verify_io(io)); -} - -static void verity_end_io(struct bio *bio, int error) -{ - struct dm_verity_io *io = bio->bi_private; - - if (error) { - verity_finish_io(io, error); - return; - } - - INIT_WORK(&io->work, verity_work); - queue_work(io->v->verify_wq, &io->work); -} - -/* - * Prefetch buffers for the specified io. - * The root buffer is not prefetched, it is assumed that it will be cached - * all the time. - */ -static void verity_prefetch_io(struct work_struct *work) -{ - struct dm_verity_prefetch_work *pw = - container_of(work, struct dm_verity_prefetch_work, work); - struct dm_verity *v = pw->v; - int i; - - for (i = v->levels - 2; i >= 0; i--) { - sector_t hash_block_start; - sector_t hash_block_end; - verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL); - verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL); - if (!i) { - unsigned cluster = ACCESS_ONCE(dm_verity_prefetch_cluster); - - cluster >>= v->data_dev_block_bits; - if (unlikely(!cluster)) - goto no_prefetch_cluster; - - if (unlikely(cluster & (cluster - 1))) - cluster = 1 << __fls(cluster); - - hash_block_start &= ~(sector_t)(cluster - 1); - hash_block_end |= cluster - 1; - if (unlikely(hash_block_end >= v->hash_blocks)) - hash_block_end = v->hash_blocks - 1; - } -no_prefetch_cluster: - dm_bufio_prefetch(v->bufio, hash_block_start, - hash_block_end - hash_block_start + 1); - } - - kfree(pw); -} - -static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) -{ - struct dm_verity_prefetch_work *pw; - - pw = kmalloc(sizeof(struct dm_verity_prefetch_work), - GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); - - if (!pw) - return; - - INIT_WORK(&pw->work, verity_prefetch_io); - pw->v = v; - pw->block = io->block; - pw->n_blocks = io->n_blocks; - queue_work(v->verify_wq, &pw->work); -} - -/* - * Bio map function. It allocates dm_verity_io structure and bio vector and - * fills them. Then it issues prefetches and the I/O. - */ -static int verity_map(struct dm_target *ti, struct bio *bio) -{ - struct dm_verity *v = ti->private; - struct dm_verity_io *io; - - bio->bi_bdev = v->data_dev->bdev; - bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector); - - if (((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) & - ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) { - DMERR_LIMIT("unaligned io"); - return -EIO; - } - - if (bio_end_sector(bio) >> - (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) { - DMERR_LIMIT("io out of range"); - return -EIO; - } - - if (bio_data_dir(bio) == WRITE) - return -EIO; - - io = dm_per_bio_data(bio, ti->per_bio_data_size); - io->v = v; - io->orig_bi_end_io = bio->bi_end_io; - io->orig_bi_private = bio->bi_private; - io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT); - io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits; - - bio->bi_end_io = verity_end_io; - bio->bi_private = io; - io->iter = bio->bi_iter; - - verity_submit_prefetch(v, io); - - generic_make_request(bio); - - return DM_MAPIO_SUBMITTED; -} - -/* - * Status: V (valid) or C (corruption found) - */ -static void verity_status(struct dm_target *ti, status_type_t type, - unsigned status_flags, char *result, unsigned maxlen) -{ - struct dm_verity *v = ti->private; - unsigned sz = 0; - unsigned x; - - switch (type) { - case STATUSTYPE_INFO: - DMEMIT("%c", v->hash_failed ? 'C' : 'V'); - break; - case STATUSTYPE_TABLE: - DMEMIT("%u %s %s %u %u %llu %llu %s ", - v->version, - v->data_dev->name, - v->hash_dev->name, - 1 << v->data_dev_block_bits, - 1 << v->hash_dev_block_bits, - (unsigned long long)v->data_blocks, - (unsigned long long)v->hash_start, - v->alg_name - ); - for (x = 0; x < v->digest_size; x++) - DMEMIT("%02x", v->root_digest[x]); - DMEMIT(" "); - if (!v->salt_size) - DMEMIT("-"); - else - for (x = 0; x < v->salt_size; x++) - DMEMIT("%02x", v->salt[x]); - if (v->mode != DM_VERITY_MODE_EIO) { - DMEMIT(" 1 "); - switch (v->mode) { - case DM_VERITY_MODE_LOGGING: - DMEMIT(DM_VERITY_OPT_LOGGING); - break; - case DM_VERITY_MODE_RESTART: - DMEMIT(DM_VERITY_OPT_RESTART); - break; - default: - BUG(); - } - } - break; - } -} - -static int verity_ioctl(struct dm_target *ti, unsigned cmd, - unsigned long arg) -{ - struct dm_verity *v = ti->private; - int r = 0; - - if (v->data_start || - ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) - r = scsi_verify_blk_ioctl(NULL, cmd); - - return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode, - cmd, arg); -} - -static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm, - struct bio_vec *biovec, int max_size) -{ - struct dm_verity *v = ti->private; - struct request_queue *q = bdev_get_queue(v->data_dev->bdev); - - if (!q->merge_bvec_fn) - return max_size; - - bvm->bi_bdev = v->data_dev->bdev; - bvm->bi_sector = verity_map_sector(v, bvm->bi_sector); - - return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); -} - -static int verity_iterate_devices(struct dm_target *ti, - iterate_devices_callout_fn fn, void *data) -{ - struct dm_verity *v = ti->private; - - return fn(ti, v->data_dev, v->data_start, ti->len, data); -} - -static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) -{ - struct dm_verity *v = ti->private; - - if (limits->logical_block_size < 1 << v->data_dev_block_bits) - limits->logical_block_size = 1 << v->data_dev_block_bits; - - if (limits->physical_block_size < 1 << v->data_dev_block_bits) - limits->physical_block_size = 1 << v->data_dev_block_bits; - - blk_limits_io_min(limits, limits->logical_block_size); -} - -static void verity_dtr(struct dm_target *ti) -{ - struct dm_verity *v = ti->private; - - if (v->verify_wq) - destroy_workqueue(v->verify_wq); - - if (v->vec_mempool) - mempool_destroy(v->vec_mempool); - - if (v->bufio) - dm_bufio_client_destroy(v->bufio); - - kfree(v->salt); - kfree(v->root_digest); - - if (v->tfm) - crypto_free_shash(v->tfm); - - kfree(v->alg_name); - - if (v->hash_dev) - dm_put_device(ti, v->hash_dev); - - if (v->data_dev) - dm_put_device(ti, v->data_dev); - - kfree(v); -} - -static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v) -{ - int r; - unsigned argc; - struct dm_target *ti = v->ti; - const char *arg_name; - - static struct dm_arg _args[] = { - {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"}, - }; - - r = dm_read_arg_group(_args, as, &argc, &ti->error); - if (r) - return -EINVAL; - - if (!argc) - return 0; - - do { - arg_name = dm_shift_arg(as); - argc--; - - if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) { - v->mode = DM_VERITY_MODE_LOGGING; - continue; - - } else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) { - v->mode = DM_VERITY_MODE_RESTART; - continue; - } - - ti->error = "Unrecognized verity feature request"; - return -EINVAL; - } while (argc && !r); - - return r; -} - -/* - * Target parameters: - * The current format is version 1. - * Vsn 0 is compatible with original Chromium OS releases. - * - * - * - * - * - * - * - * - * Hex string or "-" if no salt. - */ -static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) -{ - struct dm_verity *v; - struct dm_arg_set as; - unsigned int num; - unsigned long long num_ll; - int r; - int i; - sector_t hash_position; - char dummy; - - v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL); - if (!v) { - ti->error = "Cannot allocate verity structure"; - return -ENOMEM; - } - ti->private = v; - v->ti = ti; - - if ((dm_table_get_mode(ti->table) & ~FMODE_READ)) { - ti->error = "Device must be readonly"; - r = -EINVAL; - goto bad; - } - - if (argc < 10) { - ti->error = "Not enough arguments"; - r = -EINVAL; - goto bad; - } - - if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 || - num > 1) { - ti->error = "Invalid version"; - r = -EINVAL; - goto bad; - } - v->version = num; - - r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev); - if (r) { - ti->error = "Data device lookup failed"; - goto bad; - } - - r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev); - if (r) { - ti->error = "Data device lookup failed"; - goto bad; - } - - if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 || - !num || (num & (num - 1)) || - num < bdev_logical_block_size(v->data_dev->bdev) || - num > PAGE_SIZE) { - ti->error = "Invalid data device block size"; - r = -EINVAL; - goto bad; - } - v->data_dev_block_bits = __ffs(num); - - if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 || - !num || (num & (num - 1)) || - num < bdev_logical_block_size(v->hash_dev->bdev) || - num > INT_MAX) { - ti->error = "Invalid hash device block size"; - r = -EINVAL; - goto bad; - } - v->hash_dev_block_bits = __ffs(num); - - if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 || - (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) - >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) { - ti->error = "Invalid data blocks"; - r = -EINVAL; - goto bad; - } - v->data_blocks = num_ll; - - if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) { - ti->error = "Data device is too small"; - r = -EINVAL; - goto bad; - } - - if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 || - (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT)) - >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) { - ti->error = "Invalid hash start"; - r = -EINVAL; - goto bad; - } - v->hash_start = num_ll; - - v->alg_name = kstrdup(argv[7], GFP_KERNEL); - if (!v->alg_name) { - ti->error = "Cannot allocate algorithm name"; - r = -ENOMEM; - goto bad; - } - - v->tfm = crypto_alloc_shash(v->alg_name, 0, 0); - if (IS_ERR(v->tfm)) { - ti->error = "Cannot initialize hash function"; - r = PTR_ERR(v->tfm); - v->tfm = NULL; - goto bad; - } - v->digest_size = crypto_shash_digestsize(v->tfm); - if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) { - ti->error = "Digest size too big"; - r = -EINVAL; - goto bad; - } - v->shash_descsize = - sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm); - - v->root_digest = kmalloc(v->digest_size, GFP_KERNEL); - if (!v->root_digest) { - ti->error = "Cannot allocate root digest"; - r = -ENOMEM; - goto bad; - } - if (strlen(argv[8]) != v->digest_size * 2 || - hex2bin(v->root_digest, argv[8], v->digest_size)) { - ti->error = "Invalid root digest"; - r = -EINVAL; - goto bad; - } - - if (strcmp(argv[9], "-")) { - v->salt_size = strlen(argv[9]) / 2; - v->salt = kmalloc(v->salt_size, GFP_KERNEL); - if (!v->salt) { - ti->error = "Cannot allocate salt"; - r = -ENOMEM; - goto bad; - } - if (strlen(argv[9]) != v->salt_size * 2 || - hex2bin(v->salt, argv[9], v->salt_size)) { - ti->error = "Invalid salt"; - r = -EINVAL; - goto bad; - } - } - - argv += 10; - argc -= 10; - - /* Optional parameters */ - if (argc) { - as.argc = argc; - as.argv = argv; - - r = verity_parse_opt_args(&as, v); - if (r < 0) - goto bad; - } - - v->hash_per_block_bits = - __fls((1 << v->hash_dev_block_bits) / v->digest_size); - - v->levels = 0; - if (v->data_blocks) - while (v->hash_per_block_bits * v->levels < 64 && - (unsigned long long)(v->data_blocks - 1) >> - (v->hash_per_block_bits * v->levels)) - v->levels++; - - if (v->levels > DM_VERITY_MAX_LEVELS) { - ti->error = "Too many tree levels"; - r = -E2BIG; - goto bad; - } - - hash_position = v->hash_start; - for (i = v->levels - 1; i >= 0; i--) { - sector_t s; - v->hash_level_block[i] = hash_position; - s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1) - >> ((i + 1) * v->hash_per_block_bits); - if (hash_position + s < hash_position) { - ti->error = "Hash device offset overflow"; - r = -E2BIG; - goto bad; - } - hash_position += s; - } - v->hash_blocks = hash_position; - - v->bufio = dm_bufio_client_create(v->hash_dev->bdev, - 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), - dm_bufio_alloc_callback, NULL); - if (IS_ERR(v->bufio)) { - ti->error = "Cannot initialize dm-bufio"; - r = PTR_ERR(v->bufio); - v->bufio = NULL; - goto bad; - } - - if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) { - ti->error = "Hash device is too small"; - r = -E2BIG; - goto bad; - } - - ti->per_bio_data_size = roundup(sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2, __alignof__(struct dm_verity_io)); - - v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE, - BIO_MAX_PAGES * sizeof(struct bio_vec)); - if (!v->vec_mempool) { - ti->error = "Cannot allocate vector mempool"; - r = -ENOMEM; - goto bad; - } - - /* WQ_UNBOUND greatly improves performance when running on ramdisk */ - v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus()); - if (!v->verify_wq) { - ti->error = "Cannot allocate workqueue"; - r = -ENOMEM; - goto bad; - } - - return 0; - -bad: - verity_dtr(ti); - - return r; -} - -static struct target_type verity_target = { - .name = "verity", - .version = {1, 2, 0}, - .module = THIS_MODULE, - .ctr = verity_ctr, - .dtr = verity_dtr, - .map = verity_map, - .status = verity_status, - .ioctl = verity_ioctl, - .merge = verity_merge, - .iterate_devices = verity_iterate_devices, - .io_hints = verity_io_hints, -}; - -static int __init dm_verity_init(void) -{ - int r; - - r = dm_register_target(&verity_target); - if (r < 0) - DMERR("register failed %d", r); - - return r; -} - -static void __exit dm_verity_exit(void) -{ - dm_unregister_target(&verity_target); -} - -module_init(dm_verity_init); -module_exit(dm_verity_exit); - -MODULE_AUTHOR("Mikulas Patocka "); -MODULE_AUTHOR("Mandeep Baines "); -MODULE_AUTHOR("Will Drewry "); -MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); -MODULE_LICENSE("GPL"); -- cgit v1.2.3 From d8b345dd9b212b782039ab56b0e364e95f1bf1b5 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 3 Dec 2015 16:01:51 -0500 Subject: BACKPORT: dm verity: factor out structures and functions useful to separate object Prepare for an optional verity object to make use of existing dm-verity structures and functions. Merge conflicts in: drivers/md/dm-verity-target.c Bug: 27175947 Signed-off-by: Sami Tolvanen Signed-off-by: Mike Snitzer Signed-off-by: Badhri Jagan Sridharan Change-Id: I06c5e22a7f90a136b40f91521061f445981b5872 --- drivers/md/dm-verity-target.c | 119 ++++++------------------------------------ drivers/md/dm-verity.h | 115 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 103 deletions(-) create mode 100644 drivers/md/dm-verity.h diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index e75d1dba28f9..bf38af6ee9d2 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -14,12 +14,10 @@ * access behavior. */ -#include "dm-bufio.h" +#include "dm-verity.h" #include -#include #include -#include #define DM_MSG_PREFIX "verity" @@ -30,7 +28,6 @@ #define DM_VERITY_MEMPOOL_SIZE 4 #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 -#define DM_VERITY_MAX_LEVELS 63 #define DM_VERITY_MAX_CORRUPTED_ERRS 100 #define DM_VERITY_OPT_LOGGING "ignore_corruption" @@ -42,75 +39,6 @@ static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR); -enum verity_mode { - DM_VERITY_MODE_EIO, - DM_VERITY_MODE_LOGGING, - DM_VERITY_MODE_RESTART -}; - -enum verity_block_type { - DM_VERITY_BLOCK_TYPE_DATA, - DM_VERITY_BLOCK_TYPE_METADATA -}; - -struct dm_verity { - struct dm_dev *data_dev; - struct dm_dev *hash_dev; - struct dm_target *ti; - struct dm_bufio_client *bufio; - char *alg_name; - struct crypto_shash *tfm; - u8 *root_digest; /* digest of the root block */ - u8 *salt; /* salt: its size is salt_size */ - unsigned salt_size; - sector_t data_start; /* data offset in 512-byte sectors */ - sector_t hash_start; /* hash start in blocks */ - sector_t data_blocks; /* the number of data blocks */ - sector_t hash_blocks; /* the number of hash blocks */ - unsigned char data_dev_block_bits; /* log2(data blocksize) */ - unsigned char hash_dev_block_bits; /* log2(hash blocksize) */ - unsigned char hash_per_block_bits; /* log2(hashes in hash block) */ - unsigned char levels; /* the number of tree levels */ - unsigned char version; - unsigned digest_size; /* digest size for the current hash algorithm */ - unsigned shash_descsize;/* the size of temporary space for crypto */ - int hash_failed; /* set to 1 if hash of any block failed */ - enum verity_mode mode; /* mode for handling verification errors */ - unsigned corrupted_errs;/* Number of errors for corrupted blocks */ - - mempool_t *vec_mempool; /* mempool of bio vector */ - - struct workqueue_struct *verify_wq; - - /* starting blocks for each tree level. 0 is the lowest level. */ - sector_t hash_level_block[DM_VERITY_MAX_LEVELS]; -}; - -struct dm_verity_io { - struct dm_verity *v; - - /* original values of bio->bi_end_io and bio->bi_private */ - bio_end_io_t *orig_bi_end_io; - void *orig_bi_private; - - sector_t block; - unsigned n_blocks; - - struct bvec_iter iter; - - struct work_struct work; - - /* - * Three variably-size fields follow this struct: - * - * u8 hash_desc[v->shash_descsize]; - * u8 real_digest[v->digest_size]; - * u8 want_digest[v->digest_size]; - * - * To access them use: io_hash_desc(), io_real_digest() and io_want_digest(). - */ -}; - struct dm_verity_prefetch_work { struct work_struct work; struct dm_verity *v; @@ -118,21 +46,6 @@ struct dm_verity_prefetch_work { unsigned n_blocks; }; -static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io) -{ - return (struct shash_desc *)(io + 1); -} - -static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io) -{ - return (u8 *)(io + 1) + v->shash_descsize; -} - -static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io) -{ - return (u8 *)(io + 1) + v->shash_descsize + v->digest_size; -} - /* * Auxiliary structure appended to each dm-bufio buffer. If the value * hash_verified is nonzero, hash of the block has been verified. @@ -241,8 +154,8 @@ static int verity_hash_final(struct dm_verity *v, struct shash_desc *desc, return r; } -static int verity_hash(struct dm_verity *v, struct shash_desc *desc, - const u8 *data, size_t len, u8 *digest) +int verity_hash(struct dm_verity *v, struct shash_desc *desc, + const u8 *data, size_t len, u8 *digest) { int r; @@ -330,12 +243,12 @@ out: * Verify hash of a metadata block pertaining to the specified data block * ("block" argument) at a specified level ("level" argument). * - * On successful return, io_want_digest(v, io) contains the hash value for - * a lower tree level or for the data block (if we're at the lowest leve). + * On successful return, verity_io_want_digest(v, io) contains the hash value + * for a lower tree level or for the data block (if we're at the lowest level). * * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned. * If "skip_unverified" is false, unverified buffer is hashed and verified - * against current value of io_want_digest(v, io). + * against current value of verity_io_want_digest(v, io). */ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, sector_t block, int level, bool skip_unverified, @@ -362,13 +275,13 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, goto release_ret_r; } - r = verity_hash(v, io_hash_desc(v, io), + r = verity_hash(v, verity_io_hash_desc(v, io), data, 1 << v->hash_dev_block_bits, - io_real_digest(v, io)); + verity_io_real_digest(v, io)); if (unlikely(r < 0)) goto release_ret_r; - if (likely(memcmp(io_real_digest(v, io), want_digest, + if (likely(memcmp(verity_io_real_digest(v, io), want_digest, v->digest_size) == 0)) aux->hash_verified = 1; else if (verity_handle_err(v, @@ -392,8 +305,8 @@ release_ret_r: * Find a hash for a given block, write it to digest and verify the integrity * of the hash tree if necessary. */ -static int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, - sector_t block, u8 *digest) +int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, + sector_t block, u8 *digest) { int i; int r; @@ -435,10 +348,10 @@ static int verity_verify_io(struct dm_verity_io *io) for (b = 0; b < io->n_blocks; b++) { int r; unsigned todo; - struct shash_desc *desc = io_hash_desc(v, io); + struct shash_desc *desc = verity_io_hash_desc(v, io); r = verity_hash_for_block(v, io, io->block + b, - io_want_digest(v, io)); + verity_io_want_digest(v, io)); if (unlikely(r < 0)) return r; @@ -467,12 +380,12 @@ static int verity_verify_io(struct dm_verity_io *io) todo -= len; } while (todo); - r = verity_hash_final(v, desc, io_real_digest(v, io)); + r = verity_hash_final(v, desc, verity_io_real_digest(v, io)); if (unlikely(r < 0)) return r; - if (likely(memcmp(io_real_digest(v, io), - io_want_digest(v, io), v->digest_size) == 0)) + if (likely(memcmp(verity_io_real_digest(v, io), + verity_io_want_digest(v, io), v->digest_size) == 0)) continue; else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, io->block + b)) diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h new file mode 100644 index 000000000000..a8df0d0c832e --- /dev/null +++ b/drivers/md/dm-verity.h @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2012 Red Hat, Inc. + * Copyright (C) 2015 Google, Inc. + * + * Author: Mikulas Patocka + * + * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors + * + * This file is released under the GPLv2. + */ + +#ifndef DM_VERITY_H +#define DM_VERITY_H + +#include "dm-bufio.h" +#include +#include + +#define DM_VERITY_MAX_LEVELS 63 + +enum verity_mode { + DM_VERITY_MODE_EIO, + DM_VERITY_MODE_LOGGING, + DM_VERITY_MODE_RESTART +}; + +enum verity_block_type { + DM_VERITY_BLOCK_TYPE_DATA, + DM_VERITY_BLOCK_TYPE_METADATA +}; + +struct dm_verity { + struct dm_dev *data_dev; + struct dm_dev *hash_dev; + struct dm_target *ti; + struct dm_bufio_client *bufio; + char *alg_name; + struct crypto_shash *tfm; + u8 *root_digest; /* digest of the root block */ + u8 *salt; /* salt: its size is salt_size */ + unsigned salt_size; + sector_t data_start; /* data offset in 512-byte sectors */ + sector_t hash_start; /* hash start in blocks */ + sector_t data_blocks; /* the number of data blocks */ + sector_t hash_blocks; /* the number of hash blocks */ + unsigned char data_dev_block_bits; /* log2(data blocksize) */ + unsigned char hash_dev_block_bits; /* log2(hash blocksize) */ + unsigned char hash_per_block_bits; /* log2(hashes in hash block) */ + unsigned char levels; /* the number of tree levels */ + unsigned char version; + unsigned digest_size; /* digest size for the current hash algorithm */ + unsigned shash_descsize;/* the size of temporary space for crypto */ + int hash_failed; /* set to 1 if hash of any block failed */ + enum verity_mode mode; /* mode for handling verification errors */ + unsigned corrupted_errs;/* Number of errors for corrupted blocks */ + + mempool_t *vec_mempool; /* mempool of bio vector */ + + struct workqueue_struct *verify_wq; + + /* starting blocks for each tree level. 0 is the lowest level. */ + sector_t hash_level_block[DM_VERITY_MAX_LEVELS]; +}; + +struct dm_verity_io { + struct dm_verity *v; + + /* original value of bio->bi_end_io */ + bio_end_io_t *orig_bi_end_io; + void *orig_bi_private; + + sector_t block; + unsigned n_blocks; + + struct bvec_iter iter; + + struct work_struct work; + + /* + * Three variably-size fields follow this struct: + * + * u8 hash_desc[v->shash_descsize]; + * u8 real_digest[v->digest_size]; + * u8 want_digest[v->digest_size]; + * + * To access them use: verity_io_hash_desc(), verity_io_real_digest() + * and verity_io_want_digest(). + */ +}; + +static inline struct shash_desc *verity_io_hash_desc(struct dm_verity *v, + struct dm_verity_io *io) +{ + return (struct shash_desc *)(io + 1); +} + +static inline u8 *verity_io_real_digest(struct dm_verity *v, + struct dm_verity_io *io) +{ + return (u8 *)(io + 1) + v->shash_descsize; +} + +static inline u8 *verity_io_want_digest(struct dm_verity *v, + struct dm_verity_io *io) +{ + return (u8 *)(io + 1) + v->shash_descsize + v->digest_size; +} + +extern int verity_hash(struct dm_verity *v, struct shash_desc *desc, + const u8 *data, size_t len, u8 *digest); + +extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, + sector_t block, u8 *digest); + +#endif /* DM_VERITY_H */ -- cgit v1.2.3 From 5649a3dc94f11422c944e99061cbdac1d453149f Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 3 Dec 2015 16:30:36 -0500 Subject: UPSTREAM: dm verity: factor out verity_for_bv_block() verity_for_bv_block() will be re-used by optional dm-verity object. Bug: 27175947 Signed-off-by: Sami Tolvanen Signed-off-by: Mike Snitzer Signed-off-by: Badhri Jagan Sridharan Change-Id: Ibb92babd75f71a829a65cc63f28acaf2c3839de0 --- drivers/md/dm-verity-target.c | 72 +++++++++++++++++++++++++++++-------------- drivers/md/dm-verity.h | 6 ++++ 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index bf38af6ee9d2..34f7b447c5e1 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -335,19 +335,61 @@ int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, return 0; } +/* + * Calls function process for 1 << v->data_dev_block_bits bytes in the bio_vec + * starting from iter. + */ +int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io, + struct bvec_iter *iter, + int (*process)(struct dm_verity *v, + struct dm_verity_io *io, u8 *data, + size_t len)) +{ + unsigned todo = 1 << v->data_dev_block_bits; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size); + + do { + int r; + u8 *page; + unsigned len; + struct bio_vec bv = bio_iter_iovec(bio, *iter); + + page = kmap_atomic(bv.bv_page); + len = bv.bv_len; + + if (likely(len >= todo)) + len = todo; + + r = process(v, io, page + bv.bv_offset, len); + kunmap_atomic(page); + + if (r < 0) + return r; + + bio_advance_iter(bio, iter, len); + todo -= len; + } while (todo); + + return 0; +} + +static int verity_bv_hash_update(struct dm_verity *v, struct dm_verity_io *io, + u8 *data, size_t len) +{ + return verity_hash_update(v, verity_io_hash_desc(v, io), data, len); +} + /* * Verify one "dm_verity_io" structure. */ static int verity_verify_io(struct dm_verity_io *io) { struct dm_verity *v = io->v; - struct bio *bio = dm_bio_from_per_bio_data(io, - v->ti->per_bio_data_size); + struct bvec_iter start; unsigned b; for (b = 0; b < io->n_blocks; b++) { int r; - unsigned todo; struct shash_desc *desc = verity_io_hash_desc(v, io); r = verity_hash_for_block(v, io, io->block + b, @@ -359,26 +401,10 @@ static int verity_verify_io(struct dm_verity_io *io) if (unlikely(r < 0)) return r; - todo = 1 << v->data_dev_block_bits; - do { - u8 *page; - unsigned len; - struct bio_vec bv = bio_iter_iovec(bio, io->iter); - - page = kmap_atomic(bv.bv_page); - len = bv.bv_len; - if (likely(len >= todo)) - len = todo; - r = verity_hash_update(v, desc, page + bv.bv_offset, - len); - kunmap_atomic(page); - - if (unlikely(r < 0)) - return r; - - bio_advance_iter(bio, &io->iter, len); - todo -= len; - } while (todo); + start = io->iter; + r = verity_for_bv_block(v, io, &io->iter, verity_bv_hash_update); + if (unlikely(r < 0)) + return r; r = verity_hash_final(v, desc, verity_io_real_digest(v, io)); if (unlikely(r < 0)) diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index a8df0d0c832e..575dbe1cf4ef 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -106,6 +106,12 @@ static inline u8 *verity_io_want_digest(struct dm_verity *v, return (u8 *)(io + 1) + v->shash_descsize + v->digest_size; } +extern int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io, + struct bvec_iter *iter, + int (*process)(struct dm_verity *v, + struct dm_verity_io *io, + u8 *data, size_t len)); + extern int verity_hash(struct dm_verity *v, struct shash_desc *desc, const u8 *data, size_t len, u8 *digest); -- cgit v1.2.3 From b0d6648a5c2020fbe4351b75d7b6c619d3843f42 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 3 Dec 2015 14:26:30 +0000 Subject: BACKPORT: dm verity: add support for forward error correction Add support for correcting corrupted blocks using Reed-Solomon. This code uses RS(255, N) interleaved across data and hash blocks. Each error-correcting block covers N bytes evenly distributed across the combined total data, so that each byte is a maximum distance away from the others. This makes it possible to recover from several consecutive corrupted blocks with relatively small space overhead. In addition, using verity hashes to locate erasures nearly doubles the effectiveness of error correction. Being able to detect corrupted blocks also improves performance, because only corrupted blocks need to corrected. For a 2 GiB partition, RS(255, 253) (two parity bytes for each 253-byte block) can correct up to 16 MiB of consecutive corrupted blocks if erasures can be located, and 8 MiB if they cannot, with 16 MiB space overhead. Merge conflicts in: drivers/md/dm-verity-target.c Bug: 27175947 Signed-off-by: Sami Tolvanen Signed-off-by: Mike Snitzer Signed-off-by: Badhri Jagan Sridharan Change-Id: Ib597efc1da1fc84e9bf267bd312ab84d04f0efbe --- Documentation/device-mapper/verity.txt | 35 +- drivers/md/Kconfig | 12 + drivers/md/Makefile | 4 + drivers/md/dm-verity-fec.c | 812 +++++++++++++++++++++++++++++++++ drivers/md/dm-verity-fec.h | 152 ++++++ drivers/md/dm-verity-target.c | 53 ++- drivers/md/dm-verity.h | 10 + 7 files changed, 1071 insertions(+), 7 deletions(-) create mode 100644 drivers/md/dm-verity-fec.c create mode 100644 drivers/md/dm-verity-fec.h diff --git a/Documentation/device-mapper/verity.txt b/Documentation/device-mapper/verity.txt index e15bc1a0fb98..d602c801ff59 100644 --- a/Documentation/device-mapper/verity.txt +++ b/Documentation/device-mapper/verity.txt @@ -18,11 +18,11 @@ Construction Parameters 0 is the original format used in the Chromium OS. The salt is appended when hashing, digests are stored continuously and - the rest of the block is padded with zeros. + the rest of the block is padded with zeroes. 1 is the current format that should be used for new devices. The salt is prepended when hashing and each digest is - padded with zeros to the power of two. + padded with zeroes to the power of two. This is the device containing data, the integrity of which needs to be @@ -79,6 +79,32 @@ restart_on_corruption not compatible with ignore_corruption and requires user space support to avoid restart loops. +use_fec_from_device + Use forward error correction (FEC) to recover from corruption if hash + verification fails. Use encoding data from the specified device. This + may be the same device where data and hash blocks reside, in which case + fec_start must be outside data and hash areas. + + If the encoding data covers additional metadata, it must be accessible + on the hash device after the hash blocks. + + Note: block sizes for data and hash devices must match. Also, if the + verity is encrypted the should be too. + +fec_roots + Number of generator roots. This equals to the number of parity bytes in + the encoding data. For example, in RS(M, N) encoding, the number of roots + is M-N. + +fec_blocks + The number of encoding data blocks on the FEC device. The block size for + the FEC device is . + +fec_start + This is the offset, in blocks, from the start of the + FEC device to the beginning of the encoding data. + + Theory of operation =================== @@ -98,6 +124,11 @@ per-block basis. This allows for a lightweight hash computation on first read into the page cache. Block hashes are stored linearly, aligned to the nearest block size. +If forward error correction (FEC) support is enabled any recovery of +corrupted data will be verified using the cryptographic hash of the +corresponding data. This is why combining error correction with +integrity checking is essential. + Hash Tree --------- diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig index edcf4ab66e00..41d15599fa6a 100644 --- a/drivers/md/Kconfig +++ b/drivers/md/Kconfig @@ -445,6 +445,18 @@ config DM_VERITY If unsure, say N. +config DM_VERITY_FEC + bool "Verity forward error correction support" + depends on DM_VERITY + select REED_SOLOMON + select REED_SOLOMON_DEC8 + ---help--- + Add forward error correction support to dm-verity. This option + makes it possible to use pre-generated error correction data to + recover from corrupted blocks. + + If unsure, say N. + config DM_SWITCH tristate "Switch target support (EXPERIMENTAL)" depends on BLK_DEV_DM diff --git a/drivers/md/Makefile b/drivers/md/Makefile index 452cdde2f6cb..880ce1af336a 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -62,3 +62,7 @@ obj-$(CONFIG_DM_LOG_WRITES) += dm-log-writes.o ifeq ($(CONFIG_DM_UEVENT),y) dm-mod-objs += dm-uevent.o endif + +ifeq ($(CONFIG_DM_VERITY_FEC),y) +dm-verity-objs += dm-verity-fec.o +endif diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c new file mode 100644 index 000000000000..88143d36a1d2 --- /dev/null +++ b/drivers/md/dm-verity-fec.c @@ -0,0 +1,812 @@ +/* + * Copyright (C) 2015 Google, Inc. + * + * Author: Sami Tolvanen + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include "dm-verity-fec.h" +#include + +#define DM_MSG_PREFIX "verity-fec" + +/* + * If error correction has been configured, returns true. + */ +bool verity_fec_is_enabled(struct dm_verity *v) +{ + return v->fec && v->fec->dev; +} + +/* + * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable + * length fields. + */ +static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) +{ + return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io); +} + +/* + * Return an interleaved offset for a byte in RS block. + */ +static inline u64 fec_interleave(struct dm_verity *v, u64 offset) +{ + u32 mod; + + mod = do_div(offset, v->fec->rsn); + return offset + mod * (v->fec->rounds << v->data_dev_block_bits); +} + +/* + * Decode an RS block using Reed-Solomon. + */ +static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, + u8 *data, u8 *fec, int neras) +{ + int i; + uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; + + for (i = 0; i < v->fec->roots; i++) + par[i] = fec[i]; + + return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, + fio->erasures, 0, NULL); +} + +/* + * Read error-correcting codes for the requested RS block. Returns a pointer + * to the data block. Caller is responsible for releasing buf. + */ +static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, + unsigned *offset, struct dm_buffer **buf) +{ + u64 position, block; + u8 *res; + + position = (index + rsb) * v->fec->roots; + block = position >> v->data_dev_block_bits; + *offset = (unsigned)(position - (block << v->data_dev_block_bits)); + + res = dm_bufio_read(v->fec->bufio, v->fec->start + block, buf); + if (unlikely(IS_ERR(res))) { + DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", + v->data_dev->name, (unsigned long long)rsb, + (unsigned long long)(v->fec->start + block), + PTR_ERR(res)); + *buf = NULL; + } + + return res; +} + +/* Loop over each preallocated buffer slot. */ +#define fec_for_each_prealloc_buffer(__i) \ + for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) + +/* Loop over each extra buffer slot. */ +#define fec_for_each_extra_buffer(io, __i) \ + for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++) + +/* Loop over each allocated buffer. */ +#define fec_for_each_buffer(io, __i) \ + for (__i = 0; __i < (io)->nbufs; __i++) + +/* Loop over each RS block in each allocated buffer. */ +#define fec_for_each_buffer_rs_block(io, __i, __j) \ + fec_for_each_buffer(io, __i) \ + for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++) + +/* + * Return a pointer to the current RS block when called inside + * fec_for_each_buffer_rs_block. + */ +static inline u8 *fec_buffer_rs_block(struct dm_verity *v, + struct dm_verity_fec_io *fio, + unsigned i, unsigned j) +{ + return &fio->bufs[i][j * v->fec->rsn]; +} + +/* + * Return an index to the current RS block when called inside + * fec_for_each_buffer_rs_block. + */ +static inline unsigned fec_buffer_rs_index(unsigned i, unsigned j) +{ + return (i << DM_VERITY_FEC_BUF_RS_BITS) + j; +} + +/* + * Decode all RS blocks from buffers and copy corrected bytes into fio->output + * starting from block_offset. + */ +static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, + u64 rsb, int byte_index, unsigned block_offset, + int neras) +{ + int r, corrected = 0, res; + struct dm_buffer *buf; + unsigned n, i, offset; + u8 *par, *block; + + par = fec_read_parity(v, rsb, block_offset, &offset, &buf); + if (IS_ERR(par)) + return PTR_ERR(par); + + /* + * Decode the RS blocks we have in bufs. Each RS block results in + * one corrected target byte and consumes fec->roots parity bytes. + */ + fec_for_each_buffer_rs_block(fio, n, i) { + block = fec_buffer_rs_block(v, fio, n, i); + res = fec_decode_rs8(v, fio, block, &par[offset], neras); + if (res < 0) { + dm_bufio_release(buf); + + r = res; + goto error; + } + + corrected += res; + fio->output[block_offset] = block[byte_index]; + + block_offset++; + if (block_offset >= 1 << v->data_dev_block_bits) + goto done; + + /* read the next block when we run out of parity bytes */ + offset += v->fec->roots; + if (offset >= 1 << v->data_dev_block_bits) { + dm_bufio_release(buf); + + par = fec_read_parity(v, rsb, block_offset, &offset, &buf); + if (unlikely(IS_ERR(par))) + return PTR_ERR(par); + } + } +done: + r = corrected; +error: + if (r < 0 && neras) + DMERR_LIMIT("%s: FEC %llu: failed to correct: %d", + v->data_dev->name, (unsigned long long)rsb, r); + else if (r > 0) + DMWARN_LIMIT("%s: FEC %llu: corrected %d errors", + v->data_dev->name, (unsigned long long)rsb, r); + + return r; +} + +/* + * Locate data block erasures using verity hashes. + */ +static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, + u8 *want_digest, u8 *data) +{ + if (unlikely(verity_hash(v, verity_io_hash_desc(v, io), + data, 1 << v->data_dev_block_bits, + verity_io_real_digest(v, io)))) + return 0; + + return memcmp(verity_io_real_digest(v, io), want_digest, + v->digest_size) != 0; +} + +/* + * Read data blocks that are part of the RS block and deinterleave as much as + * fits into buffers. Check for erasure locations if @neras is non-NULL. + */ +static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, + u64 rsb, u64 target, unsigned block_offset, + int *neras) +{ + int i, j, target_index = -1; + struct dm_buffer *buf; + struct dm_bufio_client *bufio; + struct dm_verity_fec_io *fio = fec_io(io); + u64 block, ileaved; + u8 *bbuf, *rs_block; + u8 want_digest[v->digest_size]; + unsigned n, k; + + if (neras) + *neras = 0; + + /* + * read each of the rsn data blocks that are part of the RS block, and + * interleave contents to available bufs + */ + for (i = 0; i < v->fec->rsn; i++) { + ileaved = fec_interleave(v, rsb * v->fec->rsn + i); + + /* + * target is the data block we want to correct, target_index is + * the index of this block within the rsn RS blocks + */ + if (ileaved == target) + target_index = i; + + block = ileaved >> v->data_dev_block_bits; + bufio = v->fec->data_bufio; + + if (block >= v->data_blocks) { + block -= v->data_blocks; + + /* + * blocks outside the area were assumed to contain + * zeros when encoding data was generated + */ + if (unlikely(block >= v->fec->hash_blocks)) + continue; + + block += v->hash_start; + bufio = v->bufio; + } + + bbuf = dm_bufio_read(bufio, block, &buf); + if (unlikely(IS_ERR(bbuf))) { + DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld", + v->data_dev->name, + (unsigned long long)rsb, + (unsigned long long)block, PTR_ERR(bbuf)); + + /* assume the block is corrupted */ + if (neras && *neras <= v->fec->roots) + fio->erasures[(*neras)++] = i; + + continue; + } + + /* locate erasures if the block is on the data device */ + if (bufio == v->fec->data_bufio && + verity_hash_for_block(v, io, block, want_digest) == 0) { + /* + * skip if we have already found the theoretical + * maximum number (i.e. fec->roots) of erasures + */ + if (neras && *neras <= v->fec->roots && + fec_is_erasure(v, io, want_digest, bbuf)) + fio->erasures[(*neras)++] = i; + } + + /* + * deinterleave and copy the bytes that fit into bufs, + * starting from block_offset + */ + fec_for_each_buffer_rs_block(fio, n, j) { + k = fec_buffer_rs_index(n, j) + block_offset; + + if (k >= 1 << v->data_dev_block_bits) + goto done; + + rs_block = fec_buffer_rs_block(v, fio, n, j); + rs_block[i] = bbuf[k]; + } +done: + dm_bufio_release(buf); + } + + return target_index; +} + +/* + * Allocate RS control structure and FEC buffers from preallocated mempools, + * and attempt to allocate as many extra buffers as available. + */ +static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) +{ + unsigned n; + + if (!fio->rs) { + fio->rs = mempool_alloc(v->fec->rs_pool, 0); + if (unlikely(!fio->rs)) { + DMERR("failed to allocate RS"); + return -ENOMEM; + } + } + + fec_for_each_prealloc_buffer(n) { + if (fio->bufs[n]) + continue; + + fio->bufs[n] = mempool_alloc(v->fec->prealloc_pool, GFP_NOIO); + if (unlikely(!fio->bufs[n])) { + DMERR("failed to allocate FEC buffer"); + return -ENOMEM; + } + } + + /* try to allocate the maximum number of buffers */ + fec_for_each_extra_buffer(fio, n) { + if (fio->bufs[n]) + continue; + + fio->bufs[n] = mempool_alloc(v->fec->extra_pool, GFP_NOIO); + /* we can manage with even one buffer if necessary */ + if (unlikely(!fio->bufs[n])) + break; + } + fio->nbufs = n; + + if (!fio->output) { + fio->output = mempool_alloc(v->fec->output_pool, GFP_NOIO); + + if (!fio->output) { + DMERR("failed to allocate FEC page"); + return -ENOMEM; + } + } + + return 0; +} + +/* + * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are + * zeroed before deinterleaving. + */ +static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) +{ + unsigned n; + + fec_for_each_buffer(fio, n) + memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); + + memset(fio->erasures, 0, sizeof(fio->erasures)); +} + +/* + * Decode all RS blocks in a single data block and return the target block + * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses + * hashes to locate erasures. + */ +static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, + struct dm_verity_fec_io *fio, u64 rsb, u64 offset, + bool use_erasures) +{ + int r, neras = 0; + unsigned pos; + + r = fec_alloc_bufs(v, fio); + if (unlikely(r < 0)) + return r; + + for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { + fec_init_bufs(v, fio); + + r = fec_read_bufs(v, io, rsb, offset, pos, + use_erasures ? &neras : NULL); + if (unlikely(r < 0)) + return r; + + r = fec_decode_bufs(v, fio, rsb, r, pos, neras); + if (r < 0) + return r; + + pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; + } + + /* Always re-validate the corrected block against the expected hash */ + r = verity_hash(v, verity_io_hash_desc(v, io), fio->output, + 1 << v->data_dev_block_bits, + verity_io_real_digest(v, io)); + if (unlikely(r < 0)) + return r; + + if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io), + v->digest_size)) { + DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)", + v->data_dev->name, (unsigned long long)rsb, neras); + return -EILSEQ; + } + + return 0; +} + +static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, + size_t len) +{ + struct dm_verity_fec_io *fio = fec_io(io); + + memcpy(data, &fio->output[fio->output_pos], len); + fio->output_pos += len; + + return 0; +} + +/* + * Correct errors in a block. Copies corrected block to dest if non-NULL, + * otherwise to a bio_vec starting from iter. + */ +int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, + enum verity_block_type type, sector_t block, u8 *dest, + struct bvec_iter *iter) +{ + int r; + struct dm_verity_fec_io *fio = fec_io(io); + u64 offset, res, rsb; + + if (!verity_fec_is_enabled(v)) + return -EOPNOTSUPP; + + if (type == DM_VERITY_BLOCK_TYPE_METADATA) + block += v->data_blocks; + + /* + * For RS(M, N), the continuous FEC data is divided into blocks of N + * bytes. Since block size may not be divisible by N, the last block + * is zero padded when decoding. + * + * Each byte of the block is covered by a different RS(M, N) code, + * and each code is interleaved over N blocks to make it less likely + * that bursty corruption will leave us in unrecoverable state. + */ + + offset = block << v->data_dev_block_bits; + + res = offset; + div64_u64(res, v->fec->rounds << v->data_dev_block_bits); + + /* + * The base RS block we can feed to the interleaver to find out all + * blocks required for decoding. + */ + rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); + + /* + * Locating erasures is slow, so attempt to recover the block without + * them first. Do a second attempt with erasures if the corruption is + * bad enough. + */ + r = fec_decode_rsb(v, io, fio, rsb, offset, false); + if (r < 0) { + r = fec_decode_rsb(v, io, fio, rsb, offset, true); + if (r < 0) + return r; + } + + if (dest) + memcpy(dest, fio->output, 1 << v->data_dev_block_bits); + else if (iter) { + fio->output_pos = 0; + r = verity_for_bv_block(v, io, iter, fec_bv_copy); + } + + return r; +} + +/* + * Clean up per-bio data. + */ +void verity_fec_finish_io(struct dm_verity_io *io) +{ + unsigned n; + struct dm_verity_fec *f = io->v->fec; + struct dm_verity_fec_io *fio = fec_io(io); + + if (!verity_fec_is_enabled(io->v)) + return; + + mempool_free(fio->rs, f->rs_pool); + + fec_for_each_prealloc_buffer(n) + mempool_free(fio->bufs[n], f->prealloc_pool); + + fec_for_each_extra_buffer(fio, n) + mempool_free(fio->bufs[n], f->extra_pool); + + mempool_free(fio->output, f->output_pool); +} + +/* + * Initialize per-bio data. + */ +void verity_fec_init_io(struct dm_verity_io *io) +{ + struct dm_verity_fec_io *fio = fec_io(io); + + if (!verity_fec_is_enabled(io->v)) + return; + + fio->rs = NULL; + memset(fio->bufs, 0, sizeof(fio->bufs)); + fio->nbufs = 0; + fio->output = NULL; +} + +/* + * Append feature arguments and values to the status table. + */ +unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz, + char *result, unsigned maxlen) +{ + if (!verity_fec_is_enabled(v)) + return sz; + + DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s " + DM_VERITY_OPT_FEC_BLOCKS " %llu " + DM_VERITY_OPT_FEC_START " %llu " + DM_VERITY_OPT_FEC_ROOTS " %d", + v->fec->dev->name, + (unsigned long long)v->fec->blocks, + (unsigned long long)v->fec->start, + v->fec->roots); + + return sz; +} + +void verity_fec_dtr(struct dm_verity *v) +{ + struct dm_verity_fec *f = v->fec; + + if (!verity_fec_is_enabled(v)) + goto out; + + mempool_destroy(f->rs_pool); + mempool_destroy(f->prealloc_pool); + mempool_destroy(f->extra_pool); + kmem_cache_destroy(f->cache); + + if (f->data_bufio) + dm_bufio_client_destroy(f->data_bufio); + if (f->bufio) + dm_bufio_client_destroy(f->bufio); + + if (f->dev) + dm_put_device(v->ti, f->dev); +out: + kfree(f); + v->fec = NULL; +} + +static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data) +{ + struct dm_verity *v = (struct dm_verity *)pool_data; + + return init_rs(8, 0x11d, 0, 1, v->fec->roots); +} + +static void fec_rs_free(void *element, void *pool_data) +{ + struct rs_control *rs = (struct rs_control *)element; + + if (rs) + free_rs(rs); +} + +bool verity_is_fec_opt_arg(const char *arg_name) +{ + return (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV) || + !strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS) || + !strcasecmp(arg_name, DM_VERITY_OPT_FEC_START) || + !strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)); +} + +int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, + unsigned *argc, const char *arg_name) +{ + int r; + struct dm_target *ti = v->ti; + const char *arg_value; + unsigned long long num_ll; + unsigned char num_c; + char dummy; + + if (!*argc) { + ti->error = "FEC feature arguments require a value"; + return -EINVAL; + } + + arg_value = dm_shift_arg(as); + (*argc)--; + + if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) { + r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev); + if (r) { + ti->error = "FEC device lookup failed"; + return r; + } + + } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS)) { + if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || + ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) + >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { + ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; + return -EINVAL; + } + v->fec->blocks = num_ll; + + } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_START)) { + if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || + ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> + (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { + ti->error = "Invalid " DM_VERITY_OPT_FEC_START; + return -EINVAL; + } + v->fec->start = num_ll; + + } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) { + if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c || + num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || + num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { + ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; + return -EINVAL; + } + v->fec->roots = num_c; + + } else { + ti->error = "Unrecognized verity FEC feature request"; + return -EINVAL; + } + + return 0; +} + +/* + * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr. + */ +int verity_fec_ctr_alloc(struct dm_verity *v) +{ + struct dm_verity_fec *f; + + f = kzalloc(sizeof(struct dm_verity_fec), GFP_KERNEL); + if (!f) { + v->ti->error = "Cannot allocate FEC structure"; + return -ENOMEM; + } + v->fec = f; + + return 0; +} + +/* + * Validate arguments and preallocate memory. Must be called after arguments + * have been parsed using verity_fec_parse_opt_args. + */ +int verity_fec_ctr(struct dm_verity *v) +{ + struct dm_verity_fec *f = v->fec; + struct dm_target *ti = v->ti; + u64 hash_blocks; + + if (!verity_fec_is_enabled(v)) { + verity_fec_dtr(v); + return 0; + } + + /* + * FEC is computed over data blocks, possible metadata, and + * hash blocks. In other words, FEC covers total of fec_blocks + * blocks consisting of the following: + * + * data blocks | hash blocks | metadata (optional) + * + * We allow metadata after hash blocks to support a use case + * where all data is stored on the same device and FEC covers + * the entire area. + * + * If metadata is included, we require it to be available on the + * hash device after the hash blocks. + */ + + hash_blocks = v->hash_blocks - v->hash_start; + + /* + * Require matching block sizes for data and hash devices for + * simplicity. + */ + if (v->data_dev_block_bits != v->hash_dev_block_bits) { + ti->error = "Block sizes must match to use FEC"; + return -EINVAL; + } + + if (!f->roots) { + ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; + return -EINVAL; + } + f->rsn = DM_VERITY_FEC_RSM - f->roots; + + if (!f->blocks) { + ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; + return -EINVAL; + } + + f->rounds = f->blocks; + if (sector_div(f->rounds, f->rsn)) + f->rounds++; + + /* + * Due to optional metadata, f->blocks can be larger than + * data_blocks and hash_blocks combined. + */ + if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { + ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; + return -EINVAL; + } + + /* + * Metadata is accessed through the hash device, so we require + * it to be large enough. + */ + f->hash_blocks = f->blocks - v->data_blocks; + if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { + ti->error = "Hash device is too small for " + DM_VERITY_OPT_FEC_BLOCKS; + return -E2BIG; + } + + f->bufio = dm_bufio_client_create(f->dev->bdev, + 1 << v->data_dev_block_bits, + 1, 0, NULL, NULL); + if (IS_ERR(f->bufio)) { + ti->error = "Cannot initialize FEC bufio client"; + return PTR_ERR(f->bufio); + } + + if (dm_bufio_get_device_size(f->bufio) < + ((f->start + f->rounds * f->roots) >> v->data_dev_block_bits)) { + ti->error = "FEC device is too small"; + return -E2BIG; + } + + f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, + 1 << v->data_dev_block_bits, + 1, 0, NULL, NULL); + if (IS_ERR(f->data_bufio)) { + ti->error = "Cannot initialize FEC data bufio client"; + return PTR_ERR(f->data_bufio); + } + + if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { + ti->error = "Data device is too small"; + return -E2BIG; + } + + /* Preallocate an rs_control structure for each worker thread */ + f->rs_pool = mempool_create(num_online_cpus(), fec_rs_alloc, + fec_rs_free, (void *) v); + if (!f->rs_pool) { + ti->error = "Cannot allocate RS pool"; + return -ENOMEM; + } + + f->cache = kmem_cache_create("dm_verity_fec_buffers", + f->rsn << DM_VERITY_FEC_BUF_RS_BITS, + 0, 0, NULL); + if (!f->cache) { + ti->error = "Cannot create FEC buffer cache"; + return -ENOMEM; + } + + /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */ + f->prealloc_pool = mempool_create_slab_pool(num_online_cpus() * + DM_VERITY_FEC_BUF_PREALLOC, + f->cache); + if (!f->prealloc_pool) { + ti->error = "Cannot allocate FEC buffer prealloc pool"; + return -ENOMEM; + } + + f->extra_pool = mempool_create_slab_pool(0, f->cache); + if (!f->extra_pool) { + ti->error = "Cannot allocate FEC buffer extra pool"; + return -ENOMEM; + } + + /* Preallocate an output buffer for each thread */ + f->output_pool = mempool_create_kmalloc_pool(num_online_cpus(), + 1 << v->data_dev_block_bits); + if (!f->output_pool) { + ti->error = "Cannot allocate FEC output pool"; + return -ENOMEM; + } + + /* Reserve space for our per-bio data */ + ti->per_bio_data_size += sizeof(struct dm_verity_fec_io); + + return 0; +} diff --git a/drivers/md/dm-verity-fec.h b/drivers/md/dm-verity-fec.h new file mode 100644 index 000000000000..7fa0298b995e --- /dev/null +++ b/drivers/md/dm-verity-fec.h @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2015 Google, Inc. + * + * Author: Sami Tolvanen + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#ifndef DM_VERITY_FEC_H +#define DM_VERITY_FEC_H + +#include "dm-verity.h" +#include + +/* Reed-Solomon(M, N) parameters */ +#define DM_VERITY_FEC_RSM 255 +#define DM_VERITY_FEC_MAX_RSN 253 +#define DM_VERITY_FEC_MIN_RSN 231 /* ~10% space overhead */ + +/* buffers for deinterleaving and decoding */ +#define DM_VERITY_FEC_BUF_PREALLOC 1 /* buffers to preallocate */ +#define DM_VERITY_FEC_BUF_RS_BITS 4 /* 1 << RS blocks per buffer */ +/* we need buffers for at most 1 << block size RS blocks */ +#define DM_VERITY_FEC_BUF_MAX \ + (1 << (PAGE_SHIFT - DM_VERITY_FEC_BUF_RS_BITS)) + +#define DM_VERITY_OPT_FEC_DEV "use_fec_from_device" +#define DM_VERITY_OPT_FEC_BLOCKS "fec_blocks" +#define DM_VERITY_OPT_FEC_START "fec_start" +#define DM_VERITY_OPT_FEC_ROOTS "fec_roots" + +/* configuration */ +struct dm_verity_fec { + struct dm_dev *dev; /* parity data device */ + struct dm_bufio_client *data_bufio; /* for data dev access */ + struct dm_bufio_client *bufio; /* for parity data access */ + sector_t start; /* parity data start in blocks */ + sector_t blocks; /* number of blocks covered */ + sector_t rounds; /* number of interleaving rounds */ + sector_t hash_blocks; /* blocks covered after v->hash_start */ + unsigned char roots; /* number of parity bytes, M-N of RS(M, N) */ + unsigned char rsn; /* N of RS(M, N) */ + mempool_t *rs_pool; /* mempool for fio->rs */ + mempool_t *prealloc_pool; /* mempool for preallocated buffers */ + mempool_t *extra_pool; /* mempool for extra buffers */ + mempool_t *output_pool; /* mempool for output */ + struct kmem_cache *cache; /* cache for buffers */ +}; + +/* per-bio data */ +struct dm_verity_fec_io { + struct rs_control *rs; /* Reed-Solomon state */ + int erasures[DM_VERITY_FEC_MAX_RSN]; /* erasures for decode_rs8 */ + u8 *bufs[DM_VERITY_FEC_BUF_MAX]; /* bufs for deinterleaving */ + unsigned nbufs; /* number of buffers allocated */ + u8 *output; /* buffer for corrected output */ + size_t output_pos; +}; + +#ifdef CONFIG_DM_VERITY_FEC + +/* each feature parameter requires a value */ +#define DM_VERITY_OPTS_FEC 8 + +extern bool verity_fec_is_enabled(struct dm_verity *v); + +extern int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, + enum verity_block_type type, sector_t block, + u8 *dest, struct bvec_iter *iter); + +extern unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz, + char *result, unsigned maxlen); + +extern void verity_fec_finish_io(struct dm_verity_io *io); +extern void verity_fec_init_io(struct dm_verity_io *io); + +extern bool verity_is_fec_opt_arg(const char *arg_name); +extern int verity_fec_parse_opt_args(struct dm_arg_set *as, + struct dm_verity *v, unsigned *argc, + const char *arg_name); + +extern void verity_fec_dtr(struct dm_verity *v); + +extern int verity_fec_ctr_alloc(struct dm_verity *v); +extern int verity_fec_ctr(struct dm_verity *v); + +#else /* !CONFIG_DM_VERITY_FEC */ + +#define DM_VERITY_OPTS_FEC 0 + +static inline bool verity_fec_is_enabled(struct dm_verity *v) +{ + return false; +} + +static inline int verity_fec_decode(struct dm_verity *v, + struct dm_verity_io *io, + enum verity_block_type type, + sector_t block, u8 *dest, + struct bvec_iter *iter) +{ + return -EOPNOTSUPP; +} + +static inline unsigned verity_fec_status_table(struct dm_verity *v, + unsigned sz, char *result, + unsigned maxlen) +{ + return sz; +} + +static inline void verity_fec_finish_io(struct dm_verity_io *io) +{ +} + +static inline void verity_fec_init_io(struct dm_verity_io *io) +{ +} + +static inline bool verity_is_fec_opt_arg(const char *arg_name) +{ + return false; +} + +static inline int verity_fec_parse_opt_args(struct dm_arg_set *as, + struct dm_verity *v, + unsigned *argc, + const char *arg_name) +{ + return -EINVAL; +} + +static inline void verity_fec_dtr(struct dm_verity *v) +{ +} + +static inline int verity_fec_ctr_alloc(struct dm_verity *v) +{ + return 0; +} + +static inline int verity_fec_ctr(struct dm_verity *v) +{ + return 0; +} + +#endif /* CONFIG_DM_VERITY_FEC */ + +#endif /* DM_VERITY_FEC_H */ diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 34f7b447c5e1..e8b8d35f1311 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -15,6 +15,7 @@ */ #include "dm-verity.h" +#include "dm-verity-fec.h" #include #include @@ -33,7 +34,7 @@ #define DM_VERITY_OPT_LOGGING "ignore_corruption" #define DM_VERITY_OPT_RESTART "restart_on_corruption" -#define DM_VERITY_OPTS_MAX 1 +#define DM_VERITY_OPTS_MAX (1 + DM_VERITY_OPTS_FEC) static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; @@ -284,6 +285,10 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, if (likely(memcmp(verity_io_real_digest(v, io), want_digest, v->digest_size) == 0)) aux->hash_verified = 1; + else if (verity_fec_decode(v, io, + DM_VERITY_BLOCK_TYPE_METADATA, + hash_block, data, NULL) == 0) + aux->hash_verified = 1; else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_METADATA, hash_block)) { @@ -413,8 +418,11 @@ static int verity_verify_io(struct dm_verity_io *io) if (likely(memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io), v->digest_size) == 0)) continue; + else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA, + io->block + b, NULL, &start) == 0) + continue; else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, - io->block + b)) + io->block + b)) return -EIO; } @@ -432,6 +440,8 @@ static void verity_finish_io(struct dm_verity_io *io, int error) bio->bi_end_io = io->orig_bi_end_io; bio->bi_private = io->orig_bi_private; + verity_fec_finish_io(io); + bio_endio_nodec(bio, error); } @@ -446,7 +456,7 @@ static void verity_end_io(struct bio *bio, int error) { struct dm_verity_io *io = bio->bi_private; - if (error) { + if (error && !verity_fec_is_enabled(io->v)) { verity_finish_io(io, error); return; } @@ -550,6 +560,8 @@ static int verity_map(struct dm_target *ti, struct bio *bio) bio->bi_private = io; io->iter = bio->bi_iter; + verity_fec_init_io(io); + verity_submit_prefetch(v, io); generic_make_request(bio); @@ -564,6 +576,7 @@ static void verity_status(struct dm_target *ti, status_type_t type, unsigned status_flags, char *result, unsigned maxlen) { struct dm_verity *v = ti->private; + unsigned args = 0; unsigned sz = 0; unsigned x; @@ -590,8 +603,15 @@ static void verity_status(struct dm_target *ti, status_type_t type, else for (x = 0; x < v->salt_size; x++) DMEMIT("%02x", v->salt[x]); + if (v->mode != DM_VERITY_MODE_EIO) + args++; + if (verity_fec_is_enabled(v)) + args += DM_VERITY_OPTS_FEC; + if (!args) + return; + DMEMIT(" %u", args); if (v->mode != DM_VERITY_MODE_EIO) { - DMEMIT(" 1 "); + DMEMIT(" "); switch (v->mode) { case DM_VERITY_MODE_LOGGING: DMEMIT(DM_VERITY_OPT_LOGGING); @@ -603,6 +623,7 @@ static void verity_status(struct dm_target *ti, status_type_t type, BUG(); } } + sz = verity_fec_status_table(v, sz, result, maxlen); break; } } @@ -684,6 +705,8 @@ static void verity_dtr(struct dm_target *ti) if (v->data_dev) dm_put_device(ti, v->data_dev); + verity_fec_dtr(v); + kfree(v); } @@ -716,6 +739,12 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v) } else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) { v->mode = DM_VERITY_MODE_RESTART; continue; + + } else if (verity_is_fec_opt_arg(arg_name)) { + r = verity_fec_parse_opt_args(as, v, &argc, arg_name); + if (r) + return r; + continue; } ti->error = "Unrecognized verity feature request"; @@ -758,6 +787,10 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) ti->private = v; v->ti = ti; + r = verity_fec_ctr_alloc(v); + if (r) + goto bad; + if ((dm_table_get_mode(ti->table) & ~FMODE_READ)) { ti->error = "Device must be readonly"; r = -EINVAL; @@ -964,6 +997,16 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) goto bad; } + ti->per_bio_data_size = sizeof(struct dm_verity_io) + + v->shash_descsize + v->digest_size * 2; + + r = verity_fec_ctr(v); + if (r) + goto bad; + + ti->per_bio_data_size = roundup(ti->per_bio_data_size, + __alignof__(struct dm_verity_io)); + return 0; bad: @@ -974,7 +1017,7 @@ bad: static struct target_type verity_target = { .name = "verity", - .version = {1, 2, 0}, + .version = {1, 3, 0}, .module = THIS_MODULE, .ctr = verity_ctr, .dtr = verity_dtr, diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index 575dbe1cf4ef..c856c7003e8f 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -29,6 +29,8 @@ enum verity_block_type { DM_VERITY_BLOCK_TYPE_METADATA }; +struct dm_verity_fec; + struct dm_verity { struct dm_dev *data_dev; struct dm_dev *hash_dev; @@ -60,6 +62,8 @@ struct dm_verity { /* starting blocks for each tree level. 0 is the lowest level. */ sector_t hash_level_block[DM_VERITY_MAX_LEVELS]; + + struct dm_verity_fec *fec; /* forward error correction */ }; struct dm_verity_io { @@ -106,6 +110,12 @@ static inline u8 *verity_io_want_digest(struct dm_verity *v, return (u8 *)(io + 1) + v->shash_descsize + v->digest_size; } +static inline u8 *verity_io_digest_end(struct dm_verity *v, + struct dm_verity_io *io) +{ + return verity_io_want_digest(v, io) + v->digest_size; +} + extern int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io, struct bvec_iter *iter, int (*process)(struct dm_verity *v, -- cgit v1.2.3 From 86dd1b8493acd33a42f87ec94385ad428a39224f Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 3 Dec 2015 14:26:31 +0000 Subject: UPSTREAM: dm verity: add ignore_zero_blocks feature If ignore_zero_blocks is enabled dm-verity will return zeroes for blocks matching a zero hash without validating the content. Bug: 27175947 Signed-off-by: Sami Tolvanen Signed-off-by: Mike Snitzer Signed-off-by: Badhri Jagan Sridharan Change-Id: Id92e20a8347d4f7846dffbf5bcf326fad3265eca --- Documentation/device-mapper/verity.txt | 5 ++ drivers/md/dm-verity-fec.c | 8 +++- drivers/md/dm-verity-target.c | 87 ++++++++++++++++++++++++++++++---- drivers/md/dm-verity.h | 3 +- 4 files changed, 93 insertions(+), 10 deletions(-) diff --git a/Documentation/device-mapper/verity.txt b/Documentation/device-mapper/verity.txt index d602c801ff59..89fd8f9a259f 100644 --- a/Documentation/device-mapper/verity.txt +++ b/Documentation/device-mapper/verity.txt @@ -79,6 +79,11 @@ restart_on_corruption not compatible with ignore_corruption and requires user space support to avoid restart loops. +ignore_zero_blocks + Do not verify blocks that are expected to contain zeroes and always return + zeroes instead. This may be useful if the partition contains unused blocks + that are not guaranteed to contain zeroes. + use_fec_from_device Use forward error correction (FEC) to recover from corruption if hash verification fails. Use encoding data from the specified device. This diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index 88143d36a1d2..1cc10c4de701 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -205,6 +205,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, u64 rsb, u64 target, unsigned block_offset, int *neras) { + bool is_zero; int i, j, target_index = -1; struct dm_buffer *buf; struct dm_bufio_client *bufio; @@ -264,7 +265,12 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, /* locate erasures if the block is on the data device */ if (bufio == v->fec->data_bufio && - verity_hash_for_block(v, io, block, want_digest) == 0) { + verity_hash_for_block(v, io, block, want_digest, + &is_zero) == 0) { + /* skip known zero blocks entirely */ + if (is_zero) + continue; + /* * skip if we have already found the theoretical * maximum number (i.e. fec->roots) of erasures diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index e8b8d35f1311..78a7fd6a48e4 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -33,8 +33,9 @@ #define DM_VERITY_OPT_LOGGING "ignore_corruption" #define DM_VERITY_OPT_RESTART "restart_on_corruption" +#define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks" -#define DM_VERITY_OPTS_MAX (1 + DM_VERITY_OPTS_FEC) +#define DM_VERITY_OPTS_MAX (2 + DM_VERITY_OPTS_FEC) static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; @@ -311,10 +312,9 @@ release_ret_r: * of the hash tree if necessary. */ int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, - sector_t block, u8 *digest) + sector_t block, u8 *digest, bool *is_zero) { - int i; - int r; + int r = 0, i; if (likely(v->levels)) { /* @@ -326,7 +326,7 @@ int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, */ r = verity_verify_level(v, io, block, 0, true, digest); if (likely(r <= 0)) - return r; + goto out; } memcpy(digest, v->root_digest, v->digest_size); @@ -334,10 +334,15 @@ int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, for (i = v->levels - 1; i >= 0; i--) { r = verity_verify_level(v, io, block, i, false, digest); if (unlikely(r)) - return r; + goto out; } +out: + if (!r && v->zero_digest) + *is_zero = !memcmp(v->zero_digest, digest, v->digest_size); + else + *is_zero = false; - return 0; + return r; } /* @@ -384,11 +389,19 @@ static int verity_bv_hash_update(struct dm_verity *v, struct dm_verity_io *io, return verity_hash_update(v, verity_io_hash_desc(v, io), data, len); } +static int verity_bv_zero(struct dm_verity *v, struct dm_verity_io *io, + u8 *data, size_t len) +{ + memset(data, 0, len); + return 0; +} + /* * Verify one "dm_verity_io" structure. */ static int verity_verify_io(struct dm_verity_io *io) { + bool is_zero; struct dm_verity *v = io->v; struct bvec_iter start; unsigned b; @@ -398,10 +411,24 @@ static int verity_verify_io(struct dm_verity_io *io) struct shash_desc *desc = verity_io_hash_desc(v, io); r = verity_hash_for_block(v, io, io->block + b, - verity_io_want_digest(v, io)); + verity_io_want_digest(v, io), + &is_zero); if (unlikely(r < 0)) return r; + if (is_zero) { + /* + * If we expect a zero block, don't validate, just + * return zeros. + */ + r = verity_for_bv_block(v, io, &io->iter, + verity_bv_zero); + if (unlikely(r < 0)) + return r; + + continue; + } + r = verity_hash_init(v, desc); if (unlikely(r < 0)) return r; @@ -607,6 +634,8 @@ static void verity_status(struct dm_target *ti, status_type_t type, args++; if (verity_fec_is_enabled(v)) args += DM_VERITY_OPTS_FEC; + if (v->zero_digest) + args++; if (!args) return; DMEMIT(" %u", args); @@ -623,6 +652,8 @@ static void verity_status(struct dm_target *ti, status_type_t type, BUG(); } } + if (v->zero_digest) + DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES); sz = verity_fec_status_table(v, sz, result, maxlen); break; } @@ -693,6 +724,7 @@ static void verity_dtr(struct dm_target *ti) kfree(v->salt); kfree(v->root_digest); + kfree(v->zero_digest); if (v->tfm) crypto_free_shash(v->tfm); @@ -710,6 +742,37 @@ static void verity_dtr(struct dm_target *ti) kfree(v); } +static int verity_alloc_zero_digest(struct dm_verity *v) +{ + int r = -ENOMEM; + struct shash_desc *desc; + u8 *zero_data; + + v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL); + + if (!v->zero_digest) + return r; + + desc = kmalloc(v->shash_descsize, GFP_KERNEL); + + if (!desc) + return r; /* verity_dtr will free zero_digest */ + + zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL); + + if (!zero_data) + goto out; + + r = verity_hash(v, desc, zero_data, 1 << v->data_dev_block_bits, + v->zero_digest); + +out: + kfree(desc); + kfree(zero_data); + + return r; +} + static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v) { int r; @@ -740,6 +803,14 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v) v->mode = DM_VERITY_MODE_RESTART; continue; + } else if (!strcasecmp(arg_name, DM_VERITY_OPT_IGN_ZEROES)) { + r = verity_alloc_zero_digest(v); + if (r) { + ti->error = "Cannot allocate zero digest"; + return r; + } + continue; + } else if (verity_is_fec_opt_arg(arg_name)) { r = verity_fec_parse_opt_args(as, v, &argc, arg_name); if (r) diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index c856c7003e8f..55805f0971ee 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -40,6 +40,7 @@ struct dm_verity { struct crypto_shash *tfm; u8 *root_digest; /* digest of the root block */ u8 *salt; /* salt: its size is salt_size */ + u8 *zero_digest; /* digest for a zero block */ unsigned salt_size; sector_t data_start; /* data offset in 512-byte sectors */ sector_t hash_start; /* hash start in blocks */ @@ -126,6 +127,6 @@ extern int verity_hash(struct dm_verity *v, struct shash_desc *desc, const u8 *data, size_t len, u8 *digest); extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, - sector_t block, u8 *digest); + sector_t block, u8 *digest, bool *is_zero); #endif /* DM_VERITY_H */ -- cgit v1.2.3 From 6328e3d5fd7a8f1bb185018a40661382d1be4aa6 Mon Sep 17 00:00:00 2001 From: Will Drewry Date: Wed, 9 Jun 2010 17:47:38 -0500 Subject: CHROMIUM: dm: boot time specification of dm= This is a wrap-up of three patches pending upstream approval. I'm bundling them because they are interdependent, and it'll be easier to drop it on rebase later. 1. dm: allow a dm-fs-style device to be shared via dm-ioctl Integrates feedback from Alisdair, Mike, and Kiyoshi. Two main changes occur here: - One function is added which allows for a programmatically created mapped device to be inserted into the dm-ioctl hash table. This binds the device to a name and, optional, uuid which is needed by udev and allows for userspace management of the mapped device. - dm_table_complete() was extended to handle all of the final functional changes required for the table to be operational once called. 2. init: boot to device-mapper targets without an initr* Add a dm= kernel parameter modeled after the md= parameter from do_mounts_md. It allows for device-mapper targets to be configured at boot time for use early in the boot process (as the root device or otherwise). It also replaces /dev/XXX calls with major:minor opportunistically. The format is dm="name uuid ro,table line 1,table line 2,...". The parser expects the comma to be safe to use as a newline substitute but, otherwise, uses the normal separator of space. Some attempt has been made to make it forgiving of additional spaces (using skip_spaces()). A mapped device created during boot will be assigned a minor of 0 and may be access via /dev/dm-0. An example dm-linear root with no uuid may look like: root=/dev/dm-0 dm="lroot none ro, 0 4096 linear /dev/ubdb 0, 4096 4096 linear /dv/ubdc 0" Once udev is started, /dev/dm-0 will become /dev/mapper/lroot. Older upstream threads: http://marc.info/?l=dm-devel&m=127429492521964&w=2 http://marc.info/?l=dm-devel&m=127429499422096&w=2 http://marc.info/?l=dm-devel&m=127429493922000&w=2 Latest upstream threads: https://patchwork.kernel.org/patch/104859/ https://patchwork.kernel.org/patch/104860/ https://patchwork.kernel.org/patch/104861/ Bug: 27175947 Signed-off-by: Will Drewry Review URL: http://codereview.chromium.org/2020011 Change-Id: I92bd53432a11241228d2e5ac89a3b20d19b05a31 --- Documentation/device-mapper/boot.txt | 42 ++++ Documentation/kernel-parameters.txt | 6 + drivers/md/dm-ioctl.c | 39 ++++ drivers/md/dm-table.c | 1 + include/linux/device-mapper.h | 6 + init/Makefile | 1 + init/do_mounts.c | 1 + init/do_mounts.h | 10 + init/do_mounts_dm.c | 410 +++++++++++++++++++++++++++++++++++ 9 files changed, 516 insertions(+) create mode 100644 Documentation/device-mapper/boot.txt create mode 100644 init/do_mounts_dm.c diff --git a/Documentation/device-mapper/boot.txt b/Documentation/device-mapper/boot.txt new file mode 100644 index 000000000000..adcaad5e5e32 --- /dev/null +++ b/Documentation/device-mapper/boot.txt @@ -0,0 +1,42 @@ +Boot time creation of mapped devices +=================================== + +It is possible to configure a device mapper device to act as the root +device for your system in two ways. + +The first is to build an initial ramdisk which boots to a minimal +userspace which configures the device, then pivot_root(8) in to it. + +For simple device mapper configurations, it is possible to boot directly +using the following kernel command line: + +dm=" ,table line 1,...,table line n" + +name = the name to associate with the device + after boot, udev, if used, will use that name to label + the device node. +uuid = may be 'none' or the UUID desired for the device. +ro = may be "ro" or "rw". If "ro", the device and device table will be + marked read-only. + +Each table line may be as normal when using the dmsetup tool except for +two variations: +1. Any use of commas will be interpreted as a newline +2. Quotation marks cannot be escaped and cannot be used without + terminating the dm= argument. + +Unless renamed by udev, the device node created will be dm-0 as the +first minor number for the device-mapper is used during early creation. + +Example +======= + +- Booting to a linear array made up of user-mode linux block devices: + + dm="lroot none 0, 0 4096 linear 98:16 0, 4096 4096 linear 98:32 0" \ + root=/dev/dm-0 + +Will boot to a rw dm-linear target of 8192 sectors split across two +block devices identified by their major:minor numbers. After boot, udev +will rename this target to /dev/mapper/lroot (depending on the rules). +No uuid was assigned. diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 6726139bd289..848163170bce 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -56,6 +56,7 @@ parameter is applicable: BLACKFIN Blackfin architecture is enabled. CLK Common clock infrastructure is enabled. CMA Contiguous Memory Area support is enabled. + DM Device mapper support is enabled. DRM Direct Rendering Management support is enabled. DYNAMIC_DEBUG Build in debug messages and enable them at runtime EDD BIOS Enhanced Disk Drive Services (EDD) is enabled @@ -898,6 +899,11 @@ bytes respectively. Such letter suffixes can also be entirely omitted. Disable PIN 1 of APIC timer Can be useful to work around chipset bugs. + dm= [DM] Allows early creation of a device-mapper device. + See Documentation/device-mapper/boot.txt. + + dmasound= [HW,OSS] Sound subsystem buffers + dma_debug=off If the kernel is compiled with DMA_API_DEBUG support, this option disables the debugging code at boot. diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index 720ceeb7fa9b..5407a53631b5 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -1925,6 +1925,45 @@ void dm_interface_exit(void) dm_hash_exit(); } + +/** + * dm_ioctl_export - Permanently export a mapped device via the ioctl interface + * @md: Pointer to mapped_device + * @name: Buffer (size DM_NAME_LEN) for name + * @uuid: Buffer (size DM_UUID_LEN) for uuid or NULL if not desired + */ +int dm_ioctl_export(struct mapped_device *md, const char *name, + const char *uuid) +{ + int r = 0; + struct hash_cell *hc; + + if (!md) { + r = -ENXIO; + goto out; + } + + /* The name and uuid can only be set once. */ + mutex_lock(&dm_hash_cells_mutex); + hc = dm_get_mdptr(md); + mutex_unlock(&dm_hash_cells_mutex); + if (hc) { + DMERR("%s: already exported", dm_device_name(md)); + r = -ENXIO; + goto out; + } + + r = dm_hash_insert(name, uuid, md); + if (r) { + DMERR("%s: could not bind to '%s'", dm_device_name(md), name); + goto out; + } + + /* Let udev know we've changed. */ + dm_kobject_uevent(md, KOBJ_CHANGE, dm_get_event_nr(md)); +out: + return r; +} /** * dm_copy_name_and_uuid - Copy mapped device name & uuid into supplied buffers * @md: Pointer to mapped_device diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 16ba55ad7089..e39e3e3f4037 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 51cc1deb7af3..65b696e8a8d9 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -384,6 +384,12 @@ void dm_put(struct mapped_device *md); void dm_set_mdptr(struct mapped_device *md, void *ptr); void *dm_get_mdptr(struct mapped_device *md); +/* + * Export the device via the ioctl interface (uses mdptr). + */ +int dm_ioctl_export(struct mapped_device *md, const char *name, + const char *uuid); + /* * A device can still be used while suspended, but I/O is deferred. */ diff --git a/init/Makefile b/init/Makefile index 692b91f1c1d4..243f61de2cba 100644 --- a/init/Makefile +++ b/init/Makefile @@ -15,6 +15,7 @@ mounts-y := do_mounts.o mounts-$(CONFIG_BLK_DEV_RAM) += do_mounts_rd.o mounts-$(CONFIG_BLK_DEV_INITRD) += do_mounts_initrd.o mounts-$(CONFIG_BLK_DEV_MD) += do_mounts_md.o +mounts-$(CONFIG_BLK_DEV_DM) += do_mounts_dm.o # dependencies on generated files need to be listed explicitly $(obj)/version.o: include/generated/compile.h diff --git a/init/do_mounts.c b/init/do_mounts.c index a95bbdb2a502..842274efcb47 100644 --- a/init/do_mounts.c +++ b/init/do_mounts.c @@ -561,6 +561,7 @@ void __init prepare_namespace(void) wait_for_device_probe(); md_run_setup(); + dm_run_setup(); if (saved_root_name[0]) { root_device_name = saved_root_name; diff --git a/init/do_mounts.h b/init/do_mounts.h index f5b978a9bb92..09d22862e8c3 100644 --- a/init/do_mounts.h +++ b/init/do_mounts.h @@ -74,3 +74,13 @@ void md_run_setup(void); static inline void md_run_setup(void) {} #endif + +#ifdef CONFIG_BLK_DEV_DM + +void dm_run_setup(void); + +#else + +static inline void dm_run_setup(void) {} + +#endif diff --git a/init/do_mounts_dm.c b/init/do_mounts_dm.c new file mode 100644 index 000000000000..0fd3411533f3 --- /dev/null +++ b/init/do_mounts_dm.c @@ -0,0 +1,410 @@ +/* do_mounts_dm.c + * Copyright (C) 2010 The Chromium OS Authors + * All Rights Reserved. + * Based on do_mounts_md.c + * + * This file is released under the GPL. + */ +#include +#include +#include + +#include "do_mounts.h" + +#define DM_MAX_NAME 32 +#define DM_MAX_UUID 129 +#define DM_NO_UUID "none" + +#define DM_MSG_PREFIX "init" + +/* Separators used for parsing the dm= argument. */ +#define DM_FIELD_SEP ' ' +#define DM_LINE_SEP ',' + +/* + * When the device-mapper and any targets are compiled into the kernel + * (not a module), one target may be created and used as the root device at + * boot time with the parameters given with the boot line dm=... + * The code for that is here. + */ + +struct dm_setup_target { + sector_t begin; + sector_t length; + char *type; + char *params; + /* simple singly linked list */ + struct dm_setup_target *next; +}; + +static struct { + int minor; + int ro; + char name[DM_MAX_NAME]; + char uuid[DM_MAX_UUID]; + char *targets; + struct dm_setup_target *target; + int target_count; +} dm_setup_args __initdata; + +static __initdata int dm_early_setup; + +static size_t __init get_dm_option(char *str, char **next, char sep) +{ + size_t len = 0; + char *endp = NULL; + + if (!str) + return 0; + + endp = strchr(str, sep); + if (!endp) { /* act like strchrnul */ + len = strlen(str); + endp = str + len; + } else { + len = endp - str; + } + + if (endp == str) + return 0; + + if (!next) + return len; + + if (*endp == 0) { + /* Don't advance past the nul. */ + *next = endp; + } else { + *next = endp + 1; + } + return len; +} + +static int __init dm_setup_args_init(void) +{ + dm_setup_args.minor = 0; + dm_setup_args.ro = 0; + dm_setup_args.target = NULL; + dm_setup_args.target_count = 0; + return 0; +} + +static int __init dm_setup_cleanup(void) +{ + struct dm_setup_target *target = dm_setup_args.target; + struct dm_setup_target *old_target = NULL; + while (target) { + kfree(target->type); + kfree(target->params); + old_target = target; + target = target->next; + kfree(old_target); + dm_setup_args.target_count--; + } + BUG_ON(dm_setup_args.target_count); + return 0; +} + +static char * __init dm_setup_parse_device_args(char *str) +{ + char *next = NULL; + size_t len = 0; + + /* Grab the logical name of the device to be exported to udev */ + len = get_dm_option(str, &next, DM_FIELD_SEP); + if (!len) { + DMERR("failed to parse device name"); + goto parse_fail; + } + len = min(len + 1, sizeof(dm_setup_args.name)); + strlcpy(dm_setup_args.name, str, len); /* includes nul */ + str = skip_spaces(next); + + /* Grab the UUID value or "none" */ + len = get_dm_option(str, &next, DM_FIELD_SEP); + if (!len) { + DMERR("failed to parse device uuid"); + goto parse_fail; + } + len = min(len + 1, sizeof(dm_setup_args.uuid)); + strlcpy(dm_setup_args.uuid, str, len); + str = skip_spaces(next); + + /* Determine if the table/device will be read only or read-write */ + if (!strncmp("ro,", str, 3)) { + dm_setup_args.ro = 1; + } else if (!strncmp("rw,", str, 3)) { + dm_setup_args.ro = 0; + } else { + DMERR("failed to parse table mode"); + goto parse_fail; + } + str = skip_spaces(str + 3); + + return str; + +parse_fail: + return NULL; +} + +static void __init dm_substitute_devices(char *str, size_t str_len) +{ + char *candidate = str; + char *candidate_end = str; + char old_char; + size_t len = 0; + dev_t dev; + + if (str_len < 3) + return; + + while (str && *str) { + candidate = strchr(str, '/'); + if (!candidate) + break; + + /* Avoid embedded slashes */ + if (candidate != str && *(candidate - 1) != DM_FIELD_SEP) { + str = strchr(candidate, DM_FIELD_SEP); + continue; + } + + len = get_dm_option(candidate, &candidate_end, DM_FIELD_SEP); + str = skip_spaces(candidate_end); + if (len < 3 || len > 37) /* name_to_dev_t max; maj:mix min */ + continue; + + /* Temporarily terminate with a nul */ + candidate_end--; + old_char = *candidate_end; + *candidate_end = '\0'; + + DMDEBUG("converting candidate device '%s' to dev_t", candidate); + /* Use the boot-time specific device naming */ + dev = name_to_dev_t(candidate); + *candidate_end = old_char; + + DMDEBUG(" -> %u", dev); + /* No suitable replacement found */ + if (!dev) + continue; + + /* Rewrite the /dev/path as a major:minor */ + len = snprintf(candidate, len, "%u:%u", MAJOR(dev), MINOR(dev)); + if (!len) { + DMERR("error substituting device major/minor."); + break; + } + candidate += len; + /* Pad out with spaces (fixing our nul) */ + while (candidate < candidate_end) + *(candidate++) = DM_FIELD_SEP; + } +} + +static int __init dm_setup_parse_targets(char *str) +{ + char *next = NULL; + size_t len = 0; + struct dm_setup_target **target = NULL; + + /* Targets are defined as per the table format but with a + * comma as a newline separator. */ + target = &dm_setup_args.target; + while (str && *str) { + *target = kzalloc(sizeof(struct dm_setup_target), GFP_KERNEL); + if (!*target) { + DMERR("failed to allocate memory for target %d", + dm_setup_args.target_count); + goto parse_fail; + } + dm_setup_args.target_count++; + + (*target)->begin = simple_strtoull(str, &next, 10); + if (!next || *next != DM_FIELD_SEP) { + DMERR("failed to parse starting sector for target %d", + dm_setup_args.target_count - 1); + goto parse_fail; + } + str = skip_spaces(next + 1); + + (*target)->length = simple_strtoull(str, &next, 10); + if (!next || *next != DM_FIELD_SEP) { + DMERR("failed to parse length for target %d", + dm_setup_args.target_count - 1); + goto parse_fail; + } + str = skip_spaces(next + 1); + + len = get_dm_option(str, &next, DM_FIELD_SEP); + if (!len || + !((*target)->type = kstrndup(str, len, GFP_KERNEL))) { + DMERR("failed to parse type for target %d", + dm_setup_args.target_count - 1); + goto parse_fail; + } + str = skip_spaces(next); + + len = get_dm_option(str, &next, DM_LINE_SEP); + if (!len || + !((*target)->params = kstrndup(str, len, GFP_KERNEL))) { + DMERR("failed to parse params for target %d", + dm_setup_args.target_count - 1); + goto parse_fail; + } + str = skip_spaces(next); + + /* Before moving on, walk through the copied target and + * attempt to replace all /dev/xxx with the major:minor number. + * It may not be possible to resolve them traditionally at + * boot-time. */ + dm_substitute_devices((*target)->params, len); + + target = &((*target)->next); + } + DMDEBUG("parsed %d targets", dm_setup_args.target_count); + + return 0; + +parse_fail: + return 1; +} + +/* + * Parse the command-line parameters given our kernel, but do not + * actually try to invoke the DM device now; that is handled by + * dm_setup_drive after the low-level disk drivers have initialised. + * dm format is as follows: + * dm="name uuid fmode,[table line 1],[table line 2],..." + * May be used with root=/dev/dm-0 as it always uses the first dm minor. + */ + +static int __init dm_setup(char *str) +{ + dm_setup_args_init(); + + str = dm_setup_parse_device_args(str); + if (!str) { + DMDEBUG("str is NULL"); + goto parse_fail; + } + + /* Target parsing is delayed until we have dynamic memory */ + dm_setup_args.targets = str; + + printk(KERN_INFO "dm: will configure '%s' on dm-%d\n", + dm_setup_args.name, dm_setup_args.minor); + + dm_early_setup = 1; + return 1; + +parse_fail: + printk(KERN_WARNING "dm: Invalid arguments supplied to dm=.\n"); + return 0; +} + + +static void __init dm_setup_drive(void) +{ + struct mapped_device *md = NULL; + struct dm_table *table = NULL; + struct dm_setup_target *target; + char *uuid = dm_setup_args.uuid; + fmode_t fmode = FMODE_READ; + + /* Finish parsing the targets. */ + if (dm_setup_parse_targets(dm_setup_args.targets)) + goto parse_fail; + + if (dm_create(dm_setup_args.minor, &md)) { + DMDEBUG("failed to create the device"); + goto dm_create_fail; + } + DMDEBUG("created device '%s'", dm_device_name(md)); + + /* In addition to flagging the table below, the disk must be + * set explicitly ro/rw. */ + set_disk_ro(dm_disk(md), dm_setup_args.ro); + + if (!dm_setup_args.ro) + fmode |= FMODE_WRITE; + if (dm_table_create(&table, fmode, dm_setup_args.target_count, md)) { + DMDEBUG("failed to create the table"); + goto dm_table_create_fail; + } + + target = dm_setup_args.target; + while (target) { + DMINFO("adding target '%llu %llu %s %s'", + (unsigned long long) target->begin, + (unsigned long long) target->length, target->type, + target->params); + if (dm_table_add_target(table, target->type, target->begin, + target->length, target->params)) { + DMDEBUG("failed to add the target to the table"); + goto add_target_fail; + } + target = target->next; + } + + if (dm_table_complete(table)) { + DMDEBUG("failed to complete the table"); + goto table_complete_fail; + } + + /* Suspend the device so that we can bind it to the table. */ + if (dm_suspend(md, 0)) { + DMDEBUG("failed to suspend the device pre-bind"); + goto suspend_fail; + } + + /* Bind the table to the device. This is the only way to associate + * md->map with the table and set the disk capacity directly. */ + if (dm_swap_table(md, table)) { /* should return NULL. */ + DMDEBUG("failed to bind the device to the table"); + goto table_bind_fail; + } + + /* Finally, resume and the device should be ready. */ + if (dm_resume(md)) { + DMDEBUG("failed to resume the device"); + goto resume_fail; + } + + /* Export the dm device via the ioctl interface */ + if (!strcmp(DM_NO_UUID, dm_setup_args.uuid)) + uuid = NULL; + if (dm_ioctl_export(md, dm_setup_args.name, uuid)) { + DMDEBUG("failed to export device with given name and uuid"); + goto export_fail; + } + printk(KERN_INFO "dm: dm-%d is ready\n", dm_setup_args.minor); + + dm_setup_cleanup(); + return; + +export_fail: +resume_fail: +table_bind_fail: +suspend_fail: +table_complete_fail: +add_target_fail: + dm_table_put(table); +dm_table_create_fail: + dm_put(md); +dm_create_fail: + dm_setup_cleanup(); +parse_fail: + printk(KERN_WARNING "dm: starting dm-%d (%s) failed\n", + dm_setup_args.minor, dm_setup_args.name); +} + +__setup("dm=", dm_setup); + +void __init dm_run_setup(void) +{ + if (!dm_early_setup) + return; + printk(KERN_INFO "dm: attempting early device configuration.\n"); + dm_setup_drive(); +} -- cgit v1.2.3 From a196ec9fcb69b3fa81dfbd3adddb638006f3af72 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Thu, 25 Jun 2015 15:02:22 -0700 Subject: UPSTREAM: lib/string.c: introduce strreplace() Strings are sometimes sanitized by replacing a certain character (often '/') by another (often '!'). In a few places, this is done the same way Schlemiel the Painter would do it. Others are slightly smarter but still do multiple strchr() calls. Introduce strreplace() to do this using a single function call and a single pass over the string. One would expect the return value to be one of three things: void, s, or the number of replacements made. I chose the fourth, returning a pointer to the end of the string. This is more likely to be useful (for example allowing the caller to avoid a strlen call). BUG: 27175947 Signed-off-by: Rasmus Villemoes Cc: "Theodore Ts'o" Cc: Greg Kroah-Hartman Cc: Neil Brown Cc: Steven Rostedt Cc: Joe Perches Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds Change-Id: I1ddb88534a189f2e78ae1b5b074c0662781c7665 --- include/linux/string.h | 1 + lib/string.c | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/include/linux/string.h b/include/linux/string.h index e40099e585c9..a8d90db9c4b0 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -111,6 +111,7 @@ extern int memcmp(const void *,const void *,__kernel_size_t); extern void * memchr(const void *,int,__kernel_size_t); #endif void *memchr_inv(const void *s, int c, size_t n); +char *strreplace(char *s, char old, char new); extern void kfree_const(const void *x); diff --git a/lib/string.c b/lib/string.c index bb3d4b6993c4..13d1e84ddb80 100644 --- a/lib/string.c +++ b/lib/string.c @@ -849,3 +849,20 @@ void *memchr_inv(const void *start, int c, size_t bytes) return check_bytes8(start, value, bytes % 8); } EXPORT_SYMBOL(memchr_inv); + +/** + * strreplace - Replace all occurrences of character in string. + * @s: The string to operate on. + * @old: The character being replaced. + * @new: The character @old is replaced with. + * + * Returns pointer to the nul byte at the end of @s. + */ +char *strreplace(char *s, char old, char new) +{ + for (; *s; ++s) + if (*s == old) + *s = new; + return s; +} +EXPORT_SYMBOL(strreplace); -- cgit v1.2.3 From f3e815cee7a35fe04f5aad21a9a181d31ff86e07 Mon Sep 17 00:00:00 2001 From: Badhri Jagan Sridharan Date: Mon, 14 Dec 2015 20:09:39 -0800 Subject: ANDROID: dm: Add android verity target This device-mapper target is virtually a VERITY target. This target is setup by reading the metadata contents piggybacked to the actual data blocks in the block device. The signature of the metadata contents are verified against the key included in the system keyring. Upon success, the underlying verity target is setup. BUG: 27175947 Change-Id: I7e99644a0960ac8279f02c0158ed20999510ea97 Signed-off-by: Badhri Jagan Sridharan --- drivers/md/Kconfig | 16 + drivers/md/Makefile | 4 + drivers/md/dm-android-verity.c | 771 +++++++++++++++++++++++++++++++++++++++++ drivers/md/dm-android-verity.h | 92 +++++ drivers/md/dm-verity-target.c | 16 +- drivers/md/dm-verity.h | 12 + 6 files changed, 903 insertions(+), 8 deletions(-) create mode 100644 drivers/md/dm-android-verity.c create mode 100644 drivers/md/dm-android-verity.h diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig index 41d15599fa6a..38d7ad509c1a 100644 --- a/drivers/md/Kconfig +++ b/drivers/md/Kconfig @@ -487,4 +487,20 @@ config DM_LOG_WRITES If unsure, say N. +config DM_ANDROID_VERITY + bool "Android verity target support" + depends on DM_VERITY + depends on X509_CERTIFICATE_PARSER + depends on SYSTEM_TRUSTED_KEYRING + depends on PUBLIC_KEY_ALGO_RSA + depends on KEYS + depends on ASYMMETRIC_KEY_TYPE + depends on ASYMMETRIC_PUBLIC_KEY_SUBTYPE + ---help--- + This device-mapper target is virtually a VERITY target. This + target is setup by reading the metadata contents piggybacked + to the actual data blocks in the block device. The signature + of the metadata contents are verified against the key included + in the system keyring. Upon success, the underlying verity + target is setup. endif # MD diff --git a/drivers/md/Makefile b/drivers/md/Makefile index 880ce1af336a..29a776c54cf1 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -66,3 +66,7 @@ endif ifeq ($(CONFIG_DM_VERITY_FEC),y) dm-verity-objs += dm-verity-fec.o endif + +ifeq ($(CONFIG_DM_ANDROID_VERITY),y) +dm-verity-objs += dm-android-verity.o +endif diff --git a/drivers/md/dm-android-verity.c b/drivers/md/dm-android-verity.c new file mode 100644 index 000000000000..c77c9fa7a962 --- /dev/null +++ b/drivers/md/dm-android-verity.c @@ -0,0 +1,771 @@ +/* + * Copyright (C) 2015 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "dm-verity.h" +#include "dm-android-verity.h" + +static char verifiedbootstate[VERITY_COMMANDLINE_PARAM_LENGTH]; +static char veritymode[VERITY_COMMANDLINE_PARAM_LENGTH]; + +static int __init verified_boot_state_param(char *line) +{ + strlcpy(verifiedbootstate, line, sizeof(verifiedbootstate)); + return 1; +} + +__setup("androidboot.verifiedbootstate=", verified_boot_state_param); + +static int __init verity_mode_param(char *line) +{ + strlcpy(veritymode, line, sizeof(veritymode)); + return 1; +} + +__setup("androidboot.veritymode=", verity_mode_param); + +static int table_extract_mpi_array(struct public_key_signature *pks, + const void *data, size_t len) +{ + MPI mpi = mpi_read_raw_data(data, len); + + if (!mpi) { + DMERR("Error while allocating mpi array"); + return -ENOMEM; + } + + pks->mpi[0] = mpi; + pks->nr_mpi = 1; + return 0; +} + +static struct public_key_signature *table_make_digest( + enum pkey_hash_algo hash, + const void *table, + unsigned long table_len) +{ + struct public_key_signature *pks = NULL; + struct crypto_shash *tfm; + struct shash_desc *desc; + size_t digest_size, desc_size; + int ret; + + /* Allocate the hashing algorithm we're going to need and find out how + * big the hash operational data will be. + */ + tfm = crypto_alloc_shash(pkey_hash_algo[hash], 0, 0); + if (IS_ERR(tfm)) + return ERR_CAST(tfm); + + desc_size = crypto_shash_descsize(tfm) + sizeof(*desc); + digest_size = crypto_shash_digestsize(tfm); + + /* We allocate the hash operational data storage on the end of out + * context data and the digest output buffer on the end of that. + */ + ret = -ENOMEM; + pks = kzalloc(digest_size + sizeof(*pks) + desc_size, GFP_KERNEL); + if (!pks) + goto error; + + pks->pkey_hash_algo = hash; + pks->digest = (u8 *)pks + sizeof(*pks) + desc_size; + pks->digest_size = digest_size; + + desc = (struct shash_desc *)(pks + 1); + desc->tfm = tfm; + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; + + ret = crypto_shash_init(desc); + if (ret < 0) + goto error; + + ret = crypto_shash_finup(desc, table, table_len, pks->digest); + if (ret < 0) + goto error; + + crypto_free_shash(tfm); + return pks; + +error: + kfree(pks); + crypto_free_shash(tfm); + return ERR_PTR(ret); +} + +static int read_block_dev(struct bio_read *payload, struct block_device *bdev, + sector_t offset, int length) +{ + struct bio *bio; + int err = 0, i; + + payload->number_of_pages = DIV_ROUND_UP(length, PAGE_SIZE); + + bio = bio_alloc(GFP_KERNEL, payload->number_of_pages); + if (!bio) { + DMERR("Error while allocating bio"); + return -ENOMEM; + } + + bio->bi_bdev = bdev; + bio->bi_sector = offset; + + payload->page_io = kzalloc(sizeof(struct page *) * + payload->number_of_pages, GFP_KERNEL); + if (!payload->page_io) { + DMERR("page_io array alloc failed"); + err = -ENOMEM; + goto free_bio; + } + + for (i = 0; i < payload->number_of_pages; i++) { + payload->page_io[i] = alloc_page(GFP_KERNEL); + if (!payload->page_io[i]) { + DMERR("alloc_page failed"); + err = -ENOMEM; + goto free_pages; + } + if (!bio_add_page(bio, payload->page_io[i], PAGE_SIZE, 0)) { + DMERR("bio_add_page error"); + err = -EIO; + goto free_pages; + } + } + + if (!submit_bio_wait(READ, bio)) + /* success */ + goto free_bio; + DMERR("bio read failed"); + err = -EIO; + +free_pages: + for (i = 0; i < payload->number_of_pages; i++) + if (payload->page_io[i]) + __free_page(payload->page_io[i]); + kfree(payload->page_io); +free_bio: + bio_put(bio); + return err; +} + +static inline u64 fec_div_round_up(u64 x, u64 y) +{ + u64 remainder; + + return div64_u64_rem(x, y, &remainder) + + (remainder > 0 ? 1 : 0); +} + +static inline void populate_fec_metadata(struct fec_header *header, + struct fec_ecc_metadata *ecc) +{ + ecc->blocks = fec_div_round_up(le64_to_cpu(header->inp_size), + FEC_BLOCK_SIZE); + ecc->roots = le32_to_cpu(header->roots); + ecc->start = le64_to_cpu(header->inp_size); +} + +static inline int validate_fec_header(struct fec_header *header, u64 offset) +{ + /* move offset to make the sanity check work for backup header + * as well. */ + offset -= offset % FEC_BLOCK_SIZE; + if (le32_to_cpu(header->magic) != FEC_MAGIC || + le32_to_cpu(header->version) != FEC_VERSION || + le32_to_cpu(header->size) != sizeof(struct fec_header) || + le32_to_cpu(header->roots) == 0 || + le32_to_cpu(header->roots) >= FEC_RSM || + offset < le32_to_cpu(header->fec_size) || + offset - le32_to_cpu(header->fec_size) != + le64_to_cpu(header->inp_size)) + return -EINVAL; + + return 0; +} + +static int extract_fec_header(dev_t dev, struct fec_header *fec, + struct fec_ecc_metadata *ecc) +{ + u64 device_size; + struct bio_read payload; + int i, err = 0; + struct block_device *bdev; + + bdev = blkdev_get_by_dev(dev, FMODE_READ, NULL); + + if (IS_ERR(bdev)) { + DMERR("bdev get error"); + return PTR_ERR(bdev); + } + + device_size = i_size_read(bdev->bd_inode); + + /* fec metadata size is a power of 2 and PAGE_SIZE + * is a power of 2 as well. + */ + BUG_ON(FEC_BLOCK_SIZE > PAGE_SIZE); + /* 512 byte sector alignment */ + BUG_ON(((device_size - FEC_BLOCK_SIZE) % (1 << SECTOR_SHIFT)) != 0); + + err = read_block_dev(&payload, bdev, (device_size - + FEC_BLOCK_SIZE) / (1 << SECTOR_SHIFT), FEC_BLOCK_SIZE); + if (err) { + DMERR("Error while reading verity metadata"); + goto error; + } + + BUG_ON(sizeof(struct fec_header) > PAGE_SIZE); + memcpy(fec, page_address(payload.page_io[0]), + sizeof(*fec)); + + ecc->valid = true; + if (validate_fec_header(fec, device_size - FEC_BLOCK_SIZE)) { + /* Try the backup header */ + memcpy(fec, page_address(payload.page_io[0]) + FEC_BLOCK_SIZE + - sizeof(*fec) , + sizeof(*fec)); + if (validate_fec_header(fec, device_size - + sizeof(struct fec_header))) + ecc->valid = false; + } + + if (ecc->valid) + populate_fec_metadata(fec, ecc); + + for (i = 0; i < payload.number_of_pages; i++) + __free_page(payload.page_io[i]); + kfree(payload.page_io); + +error: + blkdev_put(bdev, FMODE_READ); + return err; +} +static void find_metadata_offset(struct fec_header *fec, + struct block_device *bdev, u64 *metadata_offset) +{ + u64 device_size; + + device_size = i_size_read(bdev->bd_inode); + + if (le32_to_cpu(fec->magic) == FEC_MAGIC) + *metadata_offset = le64_to_cpu(fec->inp_size) - + VERITY_METADATA_SIZE; + else + *metadata_offset = device_size - VERITY_METADATA_SIZE; +} + +static struct android_metadata *extract_metadata(dev_t dev, + struct fec_header *fec) +{ + struct block_device *bdev; + struct android_metadata_header *header; + struct android_metadata *uninitialized_var(metadata); + int i; + u32 table_length, copy_length, offset; + u64 metadata_offset; + struct bio_read payload; + int err = 0; + + bdev = blkdev_get_by_dev(dev, FMODE_READ, NULL); + + if (IS_ERR(bdev)) { + DMERR("blkdev_get_by_dev failed"); + return ERR_CAST(bdev); + } + + find_metadata_offset(fec, bdev, &metadata_offset); + + /* Verity metadata size is a power of 2 and PAGE_SIZE + * is a power of 2 as well. + * PAGE_SIZE is also a multiple of 512 bytes. + */ + if (VERITY_METADATA_SIZE > PAGE_SIZE) + BUG_ON(VERITY_METADATA_SIZE % PAGE_SIZE != 0); + /* 512 byte sector alignment */ + BUG_ON(metadata_offset % (1 << SECTOR_SHIFT) != 0); + + err = read_block_dev(&payload, bdev, metadata_offset / + (1 << SECTOR_SHIFT), VERITY_METADATA_SIZE); + if (err) { + DMERR("Error while reading verity metadata"); + metadata = ERR_PTR(err); + goto blkdev_release; + } + + header = kzalloc(sizeof(*header), GFP_KERNEL); + if (!header) { + DMERR("kzalloc failed for header"); + err = -ENOMEM; + goto free_payload; + } + + memcpy(header, page_address(payload.page_io[0]), + sizeof(*header)); + + DMINFO("bio magic_number:%u protocol_version:%d table_length:%u", + le32_to_cpu(header->magic_number), + le32_to_cpu(header->protocol_version), + le32_to_cpu(header->table_length)); + + metadata = kzalloc(sizeof(*metadata), GFP_KERNEL); + if (!metadata) { + DMERR("kzalloc for metadata failed"); + err = -ENOMEM; + goto free_header; + } + + metadata->header = header; + table_length = le32_to_cpu(header->table_length); + + if (table_length == 0 || + table_length > (VERITY_METADATA_SIZE - + sizeof(struct android_metadata_header))) + goto free_metadata; + + metadata->verity_table = kzalloc(table_length + 1, GFP_KERNEL); + + if (!metadata->verity_table) { + DMERR("kzalloc verity_table failed"); + err = -ENOMEM; + goto free_metadata; + } + + if (sizeof(struct android_metadata_header) + + table_length <= PAGE_SIZE) { + memcpy(metadata->verity_table, page_address(payload.page_io[0]) + + sizeof(struct android_metadata_header), + table_length); + } else { + copy_length = PAGE_SIZE - + sizeof(struct android_metadata_header); + memcpy(metadata->verity_table, page_address(payload.page_io[0]) + + sizeof(struct android_metadata_header), + copy_length); + table_length -= copy_length; + offset = copy_length; + i = 1; + while (table_length != 0) { + if (table_length > PAGE_SIZE) { + memcpy(metadata->verity_table + offset, + page_address(payload.page_io[i]), + PAGE_SIZE); + offset += PAGE_SIZE; + table_length -= PAGE_SIZE; + } else { + memcpy(metadata->verity_table + offset, + page_address(payload.page_io[i]), + table_length); + table_length = 0; + } + i++; + } + } + metadata->verity_table[table_length] = '\0'; + + goto free_payload; + +free_metadata: + kfree(metadata); +free_header: + kfree(header); + metadata = ERR_PTR(err); +free_payload: + for (i = 0; i < payload.number_of_pages; i++) + if (payload.page_io[i]) + __free_page(payload.page_io[i]); + kfree(payload.page_io); + + DMINFO("verity_table: %s", metadata->verity_table); +blkdev_release: + blkdev_put(bdev, FMODE_READ); + return metadata; +} + +/* helper functions to extract properties from dts */ +const char *find_dt_value(const char *name) +{ + struct device_node *firmware; + const char *value; + + firmware = of_find_node_by_path("/firmware/android"); + if (!firmware) + return NULL; + value = of_get_property(firmware, name, NULL); + of_node_put(firmware); + + return value; +} + +static bool is_unlocked(void) +{ + static const char unlocked[] = "orange"; + static const char verified_boot_prop[] = "verifiedbootstate"; + const char *value; + + value = find_dt_value(verified_boot_prop); + if (!value) + value = verifiedbootstate; + + return !strncmp(value, unlocked, sizeof(unlocked) - 1); +} + +static int verity_mode(void) +{ + static const char enforcing[] = "enforcing"; + static const char verified_mode_prop[] = "veritymode"; + const char *value; + + value = find_dt_value(verified_mode_prop); + if (!value) + value = veritymode; + if (!strncmp(value, enforcing, sizeof(enforcing) - 1)) + return DM_VERITY_MODE_RESTART; + + return DM_VERITY_MODE_EIO; +} + +static int verify_header(struct android_metadata_header *header) +{ + int retval = -EINVAL; + + if (is_unlocked() && le32_to_cpu(header->magic_number) == + VERITY_METADATA_MAGIC_DISABLE) { + retval = VERITY_STATE_DISABLE; + return retval; + } + + if (!(le32_to_cpu(header->magic_number) == + VERITY_METADATA_MAGIC_NUMBER) || + (le32_to_cpu(header->magic_number) == + VERITY_METADATA_MAGIC_DISABLE)) { + DMERR("Incorrect magic number"); + return retval; + } + + if (le32_to_cpu(header->protocol_version) != + VERITY_METADATA_VERSION) { + DMERR("Unsupported version %u", + le32_to_cpu(header->protocol_version)); + return retval; + } + + return 0; +} + +static int verify_verity_signature(char *key_id, + struct android_metadata *metadata) +{ + key_ref_t key_ref; + struct key *key; + struct public_key_signature *pks = NULL; + int retval = -EINVAL; + + key_ref = keyring_search(make_key_ref(system_trusted_keyring, 1), + &key_type_asymmetric, key_id); + + if (IS_ERR(key_ref)) { + DMERR("keyring: key not found"); + return -ENOKEY; + } + + key = key_ref_to_ptr(key_ref); + + pks = table_make_digest(PKEY_HASH_SHA256, + (const void *)metadata->verity_table, + le32_to_cpu(metadata->header->table_length)); + + if (IS_ERR(pks)) { + DMERR("hashing failed"); + goto error; + } + + retval = table_extract_mpi_array(pks, &metadata->header->signature[0], + RSANUMBYTES); + if (retval < 0) { + DMERR("Error extracting mpi %d", retval); + goto error; + } + + retval = verify_signature(key, pks); + mpi_free(pks->rsa.s); +error: + kfree(pks); + key_put(key); + + return retval; +} + +static void handle_error(void) +{ + int mode = verity_mode(); + if (mode == DM_VERITY_MODE_RESTART) { + DMERR("triggering restart"); + kernel_restart("dm-verity device corrupted"); + } else { + DMERR("Mounting verity root failed"); + } +} + +static inline bool test_mult_overflow(sector_t a, u32 b) +{ + sector_t r = (sector_t)~0ULL; + + sector_div(r, b); + return a > r; +} + +/* + * Target parameters: + * Key id of the public key in the system keyring. + * Verity metadata's signature would be verified against + * this. If the key id contains spaces, replace them + * with '#'. + * The block device for which dm-verity is being setup. + */ +static int android_verity_ctr(struct dm_target *ti, unsigned argc, char **argv) +{ + dev_t uninitialized_var(dev); + struct android_metadata *uninitialized_var(metadata); + int err = 0, i, mode; + char *key_id, *table_ptr, dummy, + *verity_table_args[VERITY_TABLE_ARGS + 2 + VERITY_TABLE_OPT_FEC_ARGS]; + /* One for specifying number of opt args and one for mode */ + sector_t data_sectors; + u32 data_block_size; + unsigned int major, minor, + no_of_args = VERITY_TABLE_ARGS + 2 + VERITY_TABLE_OPT_FEC_ARGS; + struct fec_header fec; + struct fec_ecc_metadata uninitialized_var(ecc); + char buf[FEC_ARG_LENGTH], *buf_ptr; + unsigned long long tmpll; + + if (argc != 2) { + DMERR("Incorrect number of arguments"); + handle_error(); + return -EINVAL; + } + + /* should come as one of the arguments for the verity target */ + key_id = argv[0]; + strreplace(argv[0], '#', ' '); + + if (sscanf(argv[1], "%u:%u%c", &major, &minor, &dummy) == 2) { + dev = MKDEV(major, minor); + if (MAJOR(dev) != major || MINOR(dev) != minor) { + DMERR("Incorrect bdev major minor number"); + handle_error(); + return -EOVERFLOW; + } + } + + DMINFO("key:%s dev:%s", argv[0], argv[1]); + + if (extract_fec_header(dev, &fec, &ecc)) { + DMERR("Error while extracting fec header"); + handle_error(); + return -EINVAL; + } + + metadata = extract_metadata(dev, &fec); + + if (IS_ERR(metadata)) { + DMERR("Error while extracting metadata"); + handle_error(); + return -EINVAL; + } + + err = verify_header(metadata->header); + + if (err == VERITY_STATE_DISABLE) { + DMERR("Mounting root with verity disabled"); + return -EINVAL; + } else if (err) { + DMERR("Verity header handle error"); + handle_error(); + goto free_metadata; + } + + err = verify_verity_signature(key_id, metadata); + + if (err) { + DMERR("Signature verification failed"); + handle_error(); + goto free_metadata; + } else + DMINFO("Signature verification success"); + + table_ptr = metadata->verity_table; + + for (i = 0; i < VERITY_TABLE_ARGS; i++) { + verity_table_args[i] = strsep(&table_ptr, " "); + if (verity_table_args[i] == NULL) + break; + } + + if (i != VERITY_TABLE_ARGS) { + DMERR("Verity table not in the expected format"); + err = -EINVAL; + handle_error(); + goto free_metadata; + } + + if (sscanf(verity_table_args[5], "%llu%c", &tmpll, &dummy) + != 1) { + DMERR("Verity table not in the expected format"); + handle_error(); + err = -EINVAL; + goto free_metadata; + } + + if (tmpll > ULONG_MAX) { + DMERR(" too large. Forgot to turn on CONFIG_LBDAF?"); + handle_error(); + err = -EINVAL; + goto free_metadata; + } + + data_sectors = tmpll; + + if (sscanf(verity_table_args[3], "%u%c", &data_block_size, &dummy) + != 1) { + DMERR("Verity table not in the expected format"); + handle_error(); + err = -EINVAL; + goto free_metadata; + } + + if (test_mult_overflow(data_sectors, data_block_size >> + SECTOR_SHIFT)) { + DMERR("data_sectors too large"); + handle_error(); + err = -EOVERFLOW; + goto free_metadata; + } + + data_sectors *= data_block_size >> SECTOR_SHIFT; + DMINFO("Data sectors %llu", (unsigned long long)data_sectors); + + /* update target length */ + ti->len = data_sectors; + + /*substitute data_dev and hash_dev*/ + verity_table_args[1] = argv[1]; + verity_table_args[2] = argv[1]; + + mode = verity_mode(); + + if (ecc.valid && IS_BUILTIN(CONFIG_DM_VERITY_FEC)) { + if (mode) { + err = snprintf(buf, FEC_ARG_LENGTH, + "%u %s " VERITY_TABLE_OPT_FEC_FORMAT, + 1 + VERITY_TABLE_OPT_FEC_ARGS, + mode == DM_VERITY_MODE_RESTART ? + VERITY_TABLE_OPT_RESTART : VERITY_TABLE_OPT_LOGGING, + argv[1], ecc.start / FEC_BLOCK_SIZE, ecc.blocks, + ecc.roots); + } else { + err = snprintf(buf, FEC_ARG_LENGTH, + "%u " VERITY_TABLE_OPT_FEC_FORMAT, + VERITY_TABLE_OPT_FEC_ARGS, argv[1], + ecc.start / FEC_BLOCK_SIZE, ecc.blocks, ecc.roots); + } + } else if (mode) { + err = snprintf(buf, FEC_ARG_LENGTH, + "2 " VERITY_TABLE_OPT_IGNZERO " %s", + mode == DM_VERITY_MODE_RESTART ? + VERITY_TABLE_OPT_RESTART : VERITY_TABLE_OPT_LOGGING); + } else { + err = snprintf(buf, FEC_ARG_LENGTH, "1 %s", + "ignore_zero_blocks"); + } + + if (err < 0 || err >= FEC_ARG_LENGTH) + goto free_metadata; + + buf_ptr = buf; + + for (i = VERITY_TABLE_ARGS; i < (VERITY_TABLE_ARGS + + VERITY_TABLE_OPT_FEC_ARGS + 2); i++) { + verity_table_args[i] = strsep(&buf_ptr, " "); + if (verity_table_args[i] == NULL) { + no_of_args = i; + break; + } + } + + err = verity_ctr(ti, no_of_args, verity_table_args); + +free_metadata: + kfree(metadata->header); + kfree(metadata->verity_table); + kfree(metadata); + return err; +} + +static struct target_type android_verity_target = { + .name = "android-verity", + .version = {1, 0, 0}, + .module = THIS_MODULE, + .ctr = android_verity_ctr, + .dtr = verity_dtr, + .map = verity_map, + .status = verity_status, + .ioctl = verity_ioctl, + .merge = verity_merge, + .iterate_devices = verity_iterate_devices, + .io_hints = verity_io_hints, +}; + +static int __init dm_android_verity_init(void) +{ + int r; + + r = dm_register_target(&android_verity_target); + if (r < 0) + DMERR("register failed %d", r); + + return r; +} + +static void __exit dm_android_verity_exit(void) +{ + dm_unregister_target(&android_verity_target); +} + +module_init(dm_android_verity_init); +module_exit(dm_android_verity_exit); diff --git a/drivers/md/dm-android-verity.h b/drivers/md/dm-android-verity.h new file mode 100644 index 000000000000..11477ffd2243 --- /dev/null +++ b/drivers/md/dm-android-verity.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2015 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef DM_ANDROID_VERITY_H +#define DM_ANDROID_VERITY_H + +#include + +#define RSANUMBYTES 256 +#define VERITY_METADATA_MAGIC_NUMBER 0xb001b001 +#define VERITY_METADATA_MAGIC_DISABLE 0x46464f56 +#define VERITY_METADATA_VERSION 0 +#define VERITY_STATE_DISABLE 1 +#define DATA_BLOCK_SIZE (4 * 1024) +#define VERITY_METADATA_SIZE (8 * DATA_BLOCK_SIZE) +#define VERITY_TABLE_ARGS 10 +#define VERITY_COMMANDLINE_PARAM_LENGTH 20 + +#define FEC_MAGIC 0xFECFECFE +#define FEC_BLOCK_SIZE (4 * 1024) +#define FEC_VERSION 0 +#define FEC_RSM 255 +#define FEC_ARG_LENGTH 300 + +#define VERITY_TABLE_OPT_RESTART "restart_on_corruption" +#define VERITY_TABLE_OPT_LOGGING "ignore_corruption" +#define VERITY_TABLE_OPT_IGNZERO "ignore_zero_blocks" + +#define VERITY_TABLE_OPT_FEC_FORMAT \ + "use_fec_from_device %s fec_start %llu fec_blocks %llu fec_roots %u ignore_zero_blocks" +#define VERITY_TABLE_OPT_FEC_ARGS 9 + +#define VERITY_DEBUG 0 + +#define DM_MSG_PREFIX "android-verity" +/* + * There can be two formats. + * if fec is present + * + * if fec is not present + * + */ +/* TODO: rearrange structure to reduce memory holes + * depends on userspace change. + */ +struct fec_header { + __le32 magic; + __le32 version; + __le32 size; + __le32 roots; + __le32 fec_size; + __le64 inp_size; + u8 hash[SHA256_DIGEST_SIZE]; +}; + +struct android_metadata_header { + __le32 magic_number; + __le32 protocol_version; + char signature[RSANUMBYTES]; + __le32 table_length; +}; + +struct android_metadata { + struct android_metadata_header *header; + char *verity_table; +}; + +struct fec_ecc_metadata { + bool valid; + u32 roots; + u64 blocks; + u64 rounds; + u64 start; +}; + +struct bio_read { + struct page **page_io; + int number_of_pages; +}; + +#endif /* DM_ANDROID_VERITY_H */ diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 78a7fd6a48e4..713bdb2726f9 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -553,7 +553,7 @@ static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) * Bio map function. It allocates dm_verity_io structure and bio vector and * fills them. Then it issues prefetches and the I/O. */ -static int verity_map(struct dm_target *ti, struct bio *bio) +int verity_map(struct dm_target *ti, struct bio *bio) { struct dm_verity *v = ti->private; struct dm_verity_io *io; @@ -599,7 +599,7 @@ static int verity_map(struct dm_target *ti, struct bio *bio) /* * Status: V (valid) or C (corruption found) */ -static void verity_status(struct dm_target *ti, status_type_t type, +void verity_status(struct dm_target *ti, status_type_t type, unsigned status_flags, char *result, unsigned maxlen) { struct dm_verity *v = ti->private; @@ -659,7 +659,7 @@ static void verity_status(struct dm_target *ti, status_type_t type, } } -static int verity_ioctl(struct dm_target *ti, unsigned cmd, +int verity_ioctl(struct dm_target *ti, unsigned cmd, unsigned long arg) { struct dm_verity *v = ti->private; @@ -673,7 +673,7 @@ static int verity_ioctl(struct dm_target *ti, unsigned cmd, cmd, arg); } -static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm, +int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm, struct bio_vec *biovec, int max_size) { struct dm_verity *v = ti->private; @@ -688,7 +688,7 @@ static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm, return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); } -static int verity_iterate_devices(struct dm_target *ti, +int verity_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data) { struct dm_verity *v = ti->private; @@ -696,7 +696,7 @@ static int verity_iterate_devices(struct dm_target *ti, return fn(ti, v->data_dev, v->data_start, ti->len, data); } -static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) +void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) { struct dm_verity *v = ti->private; @@ -709,7 +709,7 @@ static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) blk_limits_io_min(limits, limits->logical_block_size); } -static void verity_dtr(struct dm_target *ti) +void verity_dtr(struct dm_target *ti) { struct dm_verity *v = ti->private; @@ -839,7 +839,7 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v) * * Hex string or "-" if no salt. */ -static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) +int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) { struct dm_verity *v; struct dm_arg_set as; diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index 55805f0971ee..cc2af69ffa72 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -129,4 +129,16 @@ extern int verity_hash(struct dm_verity *v, struct shash_desc *desc, extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, sector_t block, u8 *digest, bool *is_zero); +extern void verity_status(struct dm_target *ti, status_type_t type, + unsigned status_flags, char *result, unsigned maxlen); +extern int verity_ioctl(struct dm_target *ti, unsigned cmd, + unsigned long arg); +extern int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm, + struct bio_vec *biovec, int max_size); +extern int verity_iterate_devices(struct dm_target *ti, + iterate_devices_callout_fn fn, void *data); +extern void verity_io_hints(struct dm_target *ti, struct queue_limits *limits); +extern void verity_dtr(struct dm_target *ti); +extern int verity_ctr(struct dm_target *ti, unsigned argc, char **argv); +extern int verity_map(struct dm_target *ti, struct bio *bio); #endif /* DM_VERITY_H */ -- cgit v1.2.3 From 7901ce3f41363fc5642bdd199955c2d49230dd47 Mon Sep 17 00:00:00 2001 From: Badhri Jagan Sridharan Date: Mon, 8 Feb 2016 16:28:43 -0800 Subject: ANDROID: dm-android-verity: Rebase on top of 4.1 Following CLs in upstream causes minor changes to dm-android-verity target. 1. keys: change asymmetric keys to use common hash definitions 2. block: Abstract out bvec iterator Rebase dm-android-verity on top of these changes. Bug: 27175947 Signed-off-by: Badhri Jagan Sridharan Change-Id: Icfdc3e7b3ead5de335a059cade1aca70414db415 --- drivers/md/dm-android-verity.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/md/dm-android-verity.c b/drivers/md/dm-android-verity.c index c77c9fa7a962..aeb5045830d9 100644 --- a/drivers/md/dm-android-verity.c +++ b/drivers/md/dm-android-verity.c @@ -75,7 +75,7 @@ static int table_extract_mpi_array(struct public_key_signature *pks, } static struct public_key_signature *table_make_digest( - enum pkey_hash_algo hash, + enum hash_algo hash, const void *table, unsigned long table_len) { @@ -88,7 +88,7 @@ static struct public_key_signature *table_make_digest( /* Allocate the hashing algorithm we're going to need and find out how * big the hash operational data will be. */ - tfm = crypto_alloc_shash(pkey_hash_algo[hash], 0, 0); + tfm = crypto_alloc_shash(hash_algo_name[hash], 0, 0); if (IS_ERR(tfm)) return ERR_CAST(tfm); @@ -143,7 +143,7 @@ static int read_block_dev(struct bio_read *payload, struct block_device *bdev, } bio->bi_bdev = bdev; - bio->bi_sector = offset; + bio->bi_iter.bi_sector = offset; payload->page_io = kzalloc(sizeof(struct page *) * payload->number_of_pages, GFP_KERNEL); @@ -505,7 +505,7 @@ static int verify_verity_signature(char *key_id, key = key_ref_to_ptr(key_ref); - pks = table_make_digest(PKEY_HASH_SHA256, + pks = table_make_digest(HASH_ALGO_SHA256, (const void *)metadata->verity_table, le32_to_cpu(metadata->header->table_length)); @@ -569,7 +569,7 @@ static int android_verity_ctr(struct dm_target *ti, unsigned argc, char **argv) u32 data_block_size; unsigned int major, minor, no_of_args = VERITY_TABLE_ARGS + 2 + VERITY_TABLE_OPT_FEC_ARGS; - struct fec_header fec; + struct fec_header uninitialized_var(fec); struct fec_ecc_metadata uninitialized_var(ecc); char buf[FEC_ARG_LENGTH], *buf_ptr; unsigned long long tmpll; -- cgit v1.2.3 From 52ed47fbcfdec2e5c440da4b68ed0ea202075ab9 Mon Sep 17 00:00:00 2001 From: Badhri Jagan Sridharan Date: Mon, 8 Feb 2016 16:47:41 -0800 Subject: ANDROID: dm: Rebase on top of 4.1 1. "dm: optimize use SRCU and RCU" removes the use of dm_table_put. 2. "dm: remove request-based logic from make_request_fn wrapper" necessitates calling dm_setup_md_queue or else the request_queue's make_request_fn pointer ends being unset. [ 7.711600] Internal error: Oops - bad mode: 0 [#1] PREEMPT SMP [ 7.717519] CPU: 1 PID: 1 Comm: swapper/0 Tainted: G W 4.1.15-02273-gb057d16-dirty #33 [ 7.726559] Hardware name: HiKey Development Board (DT) [ 7.731779] task: ffffffc005f8acc0 ti: ffffffc005f8c000 task.ti: ffffffc005f8c000 [ 7.739257] PC is at 0x0 [ 7.741787] LR is at generic_make_request+0x8c/0x108 .... [ 9.082931] Call trace: [ 9.085372] [< (null)>] (null) [ 9.090074] [] submit_bio+0x98/0x1e0 [ 9.095212] [] _submit_bh+0x120/0x1f0 [ 9.096165] cfg80211: Calling CRDA to update world regulatory domain [ 9.106781] [] __bread_gfp+0x94/0x114 [ 9.112004] [] ext4_fill_super+0x18c/0x2d64 [ 9.117750] [] mount_bdev+0x194/0x1c0 [ 9.122973] [] ext4_mount+0x14/0x1c [ 9.128021] [] mount_fs+0x3c/0x194 [ 9.132985] [] vfs_kern_mount+0x4c/0x134 [ 9.138467] [] do_mount+0x204/0xbbc [ 9.143514] [] SyS_mount+0x94/0xe8 [ 9.148479] [] mount_block_root+0x120/0x24c [ 9.154222] [] mount_root+0x110/0x12c [ 9.159443] [] prepare_namespace+0x170/0x1b8 [ 9.165273] [] kernel_init_freeable+0x23c/0x260 [ 9.171365] [] kernel_init+0x10/0x118 [ 9.176589] Code: bad PC value [ 9.179807] ---[ end trace 75e1bc52ba364d13 ]--- Bug: 27175947 Signed-off-by: Badhri Jagan Sridharan Change-Id: I952d86fd1475f0825f9be1386e3497b36127abd0 --- init/do_mounts_dm.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/init/do_mounts_dm.c b/init/do_mounts_dm.c index 0fd3411533f3..f521bc5ae248 100644 --- a/init/do_mounts_dm.c +++ b/init/do_mounts_dm.c @@ -10,6 +10,7 @@ #include #include "do_mounts.h" +#include "../drivers/md/dm.h" #define DM_MAX_NAME 32 #define DM_MAX_UUID 129 @@ -333,6 +334,7 @@ static void __init dm_setup_drive(void) goto dm_table_create_fail; } + dm_lock_md_type(md); target = dm_setup_args.target; while (target) { DMINFO("adding target '%llu %llu %s %s'", @@ -352,6 +354,17 @@ static void __init dm_setup_drive(void) goto table_complete_fail; } + if (dm_get_md_type(md) == DM_TYPE_NONE) { + dm_set_md_type(md, dm_table_get_type(table)); + if (dm_setup_md_queue(md)) { + DMWARN("unable to set up device queue for new table."); + goto setup_md_queue_fail; + } + } else if (dm_get_md_type(md) != dm_table_get_type(table)) { + DMWARN("can't change device type after initial table load."); + goto setup_md_queue_fail; + } + /* Suspend the device so that we can bind it to the table. */ if (dm_suspend(md, 0)) { DMDEBUG("failed to suspend the device pre-bind"); @@ -380,6 +393,7 @@ static void __init dm_setup_drive(void) } printk(KERN_INFO "dm: dm-%d is ready\n", dm_setup_args.minor); + dm_unlock_md_type(md); dm_setup_cleanup(); return; @@ -387,9 +401,10 @@ export_fail: resume_fail: table_bind_fail: suspend_fail: +setup_md_queue_fail: table_complete_fail: add_target_fail: - dm_table_put(table); + dm_unlock_md_type(md); dm_table_create_fail: dm_put(md); dm_create_fail: -- cgit v1.2.3