Pull request

-----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE+ber27ys35W+dsvQfe+BBqr8OQ4FAlyIFSwACgkQfe+BBqr8
 OQ7wghAAm16eCEr57oTO7QXR3y8uVFsKqXBn9cNH6nbrFp2PUQSglwMDKBls1Z5m
 olF23X/JaqSlSmkL9BBuzDZ6Up+kkHKuxPq4/5RKXfiDI0pr3R0eqts0COAlaN9q
 Bew3ipj99m8gzMi2093AW4+Ob0N3658fuDTGLe1M1Uoy7CEg1QJ7rVOBBEui7vIl
 RbZ8l/Zmb4ldNpB3lnE4Nh9ue8fy0RAj3Nai161nCnNeXNF/VzD3Ye8bojSBbnux
 PIMX6/RWmykX4feIf9QP8apDpxX4HkyuPq5EdwT9PD8PwdyXPAXZtsYUNCuNtQuk
 n5VKFVgFYgqUclBeVHmrMYPU4K4iCFQp4/Fua7wzPEC0iG05NiiDv91oVkEJCp3L
 ManHeuGfNLCcXaIntKZhuJl1cK8yMM3yDww6/pPTehrPjcyvKa0NOqhQBExektcD
 R6q7maJRzFaxSxdcs+Zzuog9zESvH1mlJxQCKzeYhAP0kkxInyTELE/Vbx37xuqR
 RFfZYyVQ6x87Q/sxHx4EMiV97WUM8elZOQdSEC/okt5WUUNpgIu0WF9nSQ1VKZ8C
 CZmv5xh9ogfwvB/kOm6IVwNkLvVagJQcLwddORI5LLXLbSIUcuwVSuyMp/7iDtQ/
 hnHkGs2mIJ2JUYbSSNsSJNs6oTurn8eTFCeGoYKJgd9l4QxaThU=
 =ekU+
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/jnsnow/tags/bitmaps-pull-request' into staging

Pull request

# gpg: Signature made Tue 12 Mar 2019 20:23:08 GMT
# gpg:                using RSA key F9B7ABDBBCACDF95BE76CBD07DEF8106AAFC390E
# gpg: Good signature from "John Snow (John Huston) <jsnow@redhat.com>" [full]
# Primary key fingerprint: FAEB 9711 A12C F475 812F  18F2 88A9 064D 1835 61EB
#      Subkey fingerprint: F9B7 ABDB BCAC DF95 BE76  CBD0 7DEF 8106 AAFC 390E

* remotes/jnsnow/tags/bitmaps-pull-request: (22 commits)
  tests/qemu-iotests: add bitmap resize test 246
  block/qcow2-bitmap: Allow resizes with persistent bitmaps
  block/qcow2-bitmap: Don't check size for IN_USE bitmap
  docs/interop/qcow2: Improve bitmap flag in_use specification
  bitmaps: Fix typo in function name
  block/dirty-bitmaps: implement inconsistent bit
  block/dirty-bitmaps: disallow busy bitmaps as merge source
  block/dirty-bitmaps: prohibit removing readonly bitmaps
  block/dirty-bitmaps: prohibit readonly bitmaps for backups
  block/dirty-bitmaps: add block_dirty_bitmap_check function
  block/dirty-bitmap: add inconsistent status
  block/dirty-bitmaps: add inconsistent bit
  iotests: add busy/recording bit test to 124
  blockdev: remove unused paio parameter documentation
  block/dirty-bitmaps: move comment block
  block/dirty-bitmaps: unify qmp_locked and user_locked calls
  block/dirty-bitmap: explicitly lock bitmaps with successors
  nbd: change error checking order for bitmaps
  block/dirty-bitmap: change semantics of enabled predicate
  block/dirty-bitmap: remove set/reset assertions against enabled bit
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

# Conflicts:
#	tests/qemu-iotests/group
This commit is contained in:
Peter Maydell 2019-03-13 17:30:34 +00:00
commit 523a2a42c3
17 changed files with 872 additions and 196 deletions

View file

@ -634,6 +634,119 @@ class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase):
self.vm.shutdown()
self.check_backups()
def test_incremental_pause(self):
"""
Test an incremental backup that errors into a pause and is resumed.
"""
drive0 = self.drives[0]
# NB: The blkdebug script here looks for a "flush, read, read" pattern.
# The flush occurs in hmp_io_writes, the first read in device_add, and
# the last read during the block job.
result = self.vm.qmp('blockdev-add',
node_name=drive0['id'],
driver=drive0['fmt'],
file={
'driver': 'blkdebug',
'image': {
'driver': 'file',
'filename': drive0['file']
},
'set-state': [{
'event': 'flush_to_disk',
'state': 1,
'new_state': 2
},{
'event': 'read_aio',
'state': 2,
'new_state': 3
}],
'inject-error': [{
'event': 'read_aio',
'errno': 5,
'state': 3,
'immediately': False,
'once': True
}],
})
self.assert_qmp(result, 'return', {})
self.create_anchor_backup(drive0)
bitmap = self.add_bitmap('bitmap0', drive0)
# Emulate guest activity
self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
('0xfe', '16M', '256k'),
('0x64', '32736k', '64k')))
# For the purposes of query-block visibility of bitmaps, add a drive
# frontend after we've written data; otherwise we can't use hmp-io
result = self.vm.qmp("device_add",
id="device0",
drive=drive0['id'],
driver="virtio-blk")
self.assert_qmp(result, 'return', {})
# Bitmap Status Check
query = self.vm.qmp('query-block')
ret = [bmap for bmap in query['return'][0]['dirty-bitmaps']
if bmap.get('name') == bitmap.name][0]
self.assert_qmp(ret, 'count', 458752)
self.assert_qmp(ret, 'granularity', 65536)
self.assert_qmp(ret, 'status', 'active')
self.assert_qmp(ret, 'busy', False)
self.assert_qmp(ret, 'recording', True)
# Start backup
parent, _ = bitmap.last_target()
target = self.prepare_backup(bitmap, parent)
res = self.vm.qmp('drive-backup',
job_id=bitmap.drive['id'],
device=bitmap.drive['id'],
sync='incremental',
bitmap=bitmap.name,
format=bitmap.drive['fmt'],
target=target,
mode='existing',
on_source_error='stop')
self.assert_qmp(res, 'return', {})
# Wait for the error
event = self.vm.event_wait(name="BLOCK_JOB_ERROR",
match={"data":{"device":bitmap.drive['id']}})
self.assert_qmp(event, 'data', {'device': bitmap.drive['id'],
'action': 'stop',
'operation': 'read'})
# Bitmap Status Check
query = self.vm.qmp('query-block')
ret = [bmap for bmap in query['return'][0]['dirty-bitmaps']
if bmap.get('name') == bitmap.name][0]
self.assert_qmp(ret, 'count', 458752)
self.assert_qmp(ret, 'granularity', 65536)
self.assert_qmp(ret, 'status', 'frozen')
self.assert_qmp(ret, 'busy', True)
self.assert_qmp(ret, 'recording', True)
# Resume and check incremental backup for consistency
res = self.vm.qmp('block-job-resume', device=bitmap.drive['id'])
self.assert_qmp(res, 'return', {})
self.wait_qmp_backup(bitmap.drive['id'])
# Bitmap Status Check
query = self.vm.qmp('query-block')
ret = [bmap for bmap in query['return'][0]['dirty-bitmaps']
if bmap.get('name') == bitmap.name][0]
self.assert_qmp(ret, 'count', 0)
self.assert_qmp(ret, 'granularity', 65536)
self.assert_qmp(ret, 'status', 'active')
self.assert_qmp(ret, 'busy', False)
self.assert_qmp(ret, 'recording', True)
# Finalize / Cleanup
self.make_reference_backup(bitmap)
self.vm.shutdown()
self.check_backups()
if __name__ == '__main__':
iotests.main(supported_fmts=['qcow2'])

View file

@ -1,5 +1,5 @@
...........
............
----------------------------------------------------------------------
Ran 11 tests
Ran 12 tests
OK

View file

@ -22,17 +22,21 @@ write -P0xcd 0x3ff0000 64k
"bitmaps": {
"drive0": [
{
"busy": false,
"count": 262144,
"granularity": 65536,
"name": "bitmapB",
"persistent": false,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 262144,
"granularity": 65536,
"name": "bitmapA",
"persistent": false,
"recording": true,
"status": "active"
}
]
@ -84,17 +88,21 @@ write -P0xcd 0x3ff0000 64k
"bitmaps": {
"drive0": [
{
"busy": false,
"count": 262144,
"granularity": 65536,
"name": "bitmapB",
"persistent": false,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 262144,
"granularity": 65536,
"name": "bitmapA",
"persistent": false,
"recording": true,
"status": "active"
}
]
@ -184,24 +192,30 @@ write -P0xea 0x3fe0000 64k
"bitmaps": {
"drive0": [
{
"busy": false,
"count": 393216,
"granularity": 65536,
"name": "bitmapC",
"persistent": false,
"recording": false,
"status": "disabled"
},
{
"busy": false,
"count": 262144,
"granularity": 65536,
"name": "bitmapB",
"persistent": false,
"recording": false,
"status": "disabled"
},
{
"busy": false,
"count": 458752,
"granularity": 65536,
"name": "bitmapA",
"persistent": false,
"recording": false,
"status": "disabled"
}
]
@ -251,24 +265,30 @@ write -P0xea 0x3fe0000 64k
"bitmaps": {
"drive0": [
{
"busy": false,
"count": 393216,
"granularity": 65536,
"name": "bitmapC",
"persistent": false,
"recording": false,
"status": "disabled"
},
{
"busy": false,
"count": 262144,
"granularity": 65536,
"name": "bitmapB",
"persistent": false,
"recording": false,
"status": "disabled"
},
{
"busy": false,
"count": 458752,
"granularity": 65536,
"name": "bitmapA",
"persistent": false,
"recording": false,
"status": "disabled"
}
]
@ -311,31 +331,39 @@ write -P0xea 0x3fe0000 64k
"bitmaps": {
"drive0": [
{
"busy": false,
"count": 458752,
"granularity": 65536,
"name": "bitmapD",
"persistent": false,
"recording": false,
"status": "disabled"
},
{
"busy": false,
"count": 393216,
"granularity": 65536,
"name": "bitmapC",
"persistent": false,
"recording": false,
"status": "disabled"
},
{
"busy": false,
"count": 262144,
"granularity": 65536,
"name": "bitmapB",
"persistent": false,
"recording": false,
"status": "disabled"
},
{
"busy": false,
"count": 458752,
"granularity": 65536,
"name": "bitmapA",
"persistent": false,
"recording": false,
"status": "disabled"
}
]

114
tests/qemu-iotests/246 Executable file
View file

@ -0,0 +1,114 @@
#!/usr/bin/env python
#
# Test persistent bitmap resizing.
#
# Copyright (c) 2019 John Snow for Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# owner=jsnow@redhat.com
import iotests
from iotests import log
iotests.verify_image_format(supported_fmts=['qcow2'])
size = 64 * 1024 * 1024 * 1024
gran_small = 32 * 1024
gran_large = 128 * 1024
def query_bitmaps(vm):
res = vm.qmp("query-block")
return { "bitmaps": { device['device']: device.get('dirty-bitmaps', []) for
device in res['return'] } }
with iotests.FilePath('img') as img_path, \
iotests.VM() as vm:
log('--- Preparing image & VM ---\n')
iotests.qemu_img_create('-f', iotests.imgfmt, img_path, str(size))
vm.add_drive(img_path)
log('--- 1st Boot (Establish Baseline Image) ---\n')
vm.launch()
log('\n--- Adding bitmaps Small, Medium, Large, and Transient ---\n')
vm.qmp_log("block-dirty-bitmap-add", node="drive0",
name="Small", granularity=gran_small, persistent=True)
vm.qmp_log("block-dirty-bitmap-add", node="drive0",
name="Medium", persistent=True)
vm.qmp_log("block-dirty-bitmap-add", node="drive0",
name="Large", granularity=gran_large, persistent=True)
vm.qmp_log("block-dirty-bitmap-add", node="drive0",
name="Transient", persistent=False)
log('--- Forcing flush of bitmaps to disk ---\n')
log(query_bitmaps(vm), indent=2)
vm.shutdown()
log('--- 2nd Boot (Grow Image) ---\n')
vm.launch()
log(query_bitmaps(vm), indent=2)
log('--- Adding new bitmap, growing image, and adding 2nd new bitmap ---')
vm.qmp_log("block-dirty-bitmap-add", node="drive0",
name="New", persistent=True)
vm.qmp_log("human-monitor-command",
command_line="block_resize drive0 70G")
vm.qmp_log("block-dirty-bitmap-add", node="drive0",
name="Newtwo", persistent=True)
log(query_bitmaps(vm), indent=2)
log('--- Forcing flush of bitmaps to disk ---\n')
vm.shutdown()
log('--- 3rd Boot (Shrink Image) ---\n')
vm.launch()
log(query_bitmaps(vm), indent=2)
log('--- Adding "NewB" bitmap, removing "New" bitmap ---')
vm.qmp_log("block-dirty-bitmap-add", node="drive0",
name="NewB", persistent=True)
vm.qmp_log("block-dirty-bitmap-remove", node="drive0",
name="New")
log('--- Truncating image ---\n')
vm.qmp_log("human-monitor-command",
command_line="block_resize drive0 50G")
log('--- Adding "NewC" bitmap, removing "NewTwo" bitmap ---')
vm.qmp_log("block-dirty-bitmap-add", node="drive0",
name="NewC", persistent=True)
vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="Newtwo")
log('--- Forcing flush of bitmaps to disk ---\n')
vm.shutdown()
log('--- 4th Boot (Verification and Cleanup) ---\n')
vm.launch()
log(query_bitmaps(vm), indent=2)
log('--- Removing all Bitmaps ---\n')
vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="Small")
vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="Medium")
vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="Large")
vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="NewB")
vm.qmp_log("block-dirty-bitmap-remove", node="drive0", name="NewC")
log(query_bitmaps(vm), indent=2)
log('\n--- Done ---')
vm.shutdown()

295
tests/qemu-iotests/246.out Normal file
View file

@ -0,0 +1,295 @@
--- Preparing image & VM ---
--- 1st Boot (Establish Baseline Image) ---
--- Adding bitmaps Small, Medium, Large, and Transient ---
{"execute": "block-dirty-bitmap-add", "arguments": {"granularity": 32768, "name": "Small", "node": "drive0", "persistent": true}}
{"return": {}}
{"execute": "block-dirty-bitmap-add", "arguments": {"name": "Medium", "node": "drive0", "persistent": true}}
{"return": {}}
{"execute": "block-dirty-bitmap-add", "arguments": {"granularity": 131072, "name": "Large", "node": "drive0", "persistent": true}}
{"return": {}}
{"execute": "block-dirty-bitmap-add", "arguments": {"name": "Transient", "node": "drive0", "persistent": false}}
{"return": {}}
--- Forcing flush of bitmaps to disk ---
{
"bitmaps": {
"drive0": [
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "Transient",
"persistent": false,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 131072,
"name": "Large",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "Medium",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 32768,
"name": "Small",
"persistent": true,
"recording": true,
"status": "active"
}
]
}
}
--- 2nd Boot (Grow Image) ---
{
"bitmaps": {
"drive0": [
{
"busy": false,
"count": 0,
"granularity": 32768,
"name": "Small",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "Medium",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 131072,
"name": "Large",
"persistent": true,
"recording": true,
"status": "active"
}
]
}
}
--- Adding new bitmap, growing image, and adding 2nd new bitmap ---
{"execute": "block-dirty-bitmap-add", "arguments": {"name": "New", "node": "drive0", "persistent": true}}
{"return": {}}
{"execute": "human-monitor-command", "arguments": {"command-line": "block_resize drive0 70G"}}
{"return": ""}
{"execute": "block-dirty-bitmap-add", "arguments": {"name": "Newtwo", "node": "drive0", "persistent": true}}
{"return": {}}
{
"bitmaps": {
"drive0": [
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "Newtwo",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "New",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 32768,
"name": "Small",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "Medium",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 131072,
"name": "Large",
"persistent": true,
"recording": true,
"status": "active"
}
]
}
}
--- Forcing flush of bitmaps to disk ---
--- 3rd Boot (Shrink Image) ---
{
"bitmaps": {
"drive0": [
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "New",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "Newtwo",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 32768,
"name": "Small",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "Medium",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 131072,
"name": "Large",
"persistent": true,
"recording": true,
"status": "active"
}
]
}
}
--- Adding "NewB" bitmap, removing "New" bitmap ---
{"execute": "block-dirty-bitmap-add", "arguments": {"name": "NewB", "node": "drive0", "persistent": true}}
{"return": {}}
{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "New", "node": "drive0"}}
{"return": {}}
--- Truncating image ---
{"execute": "human-monitor-command", "arguments": {"command-line": "block_resize drive0 50G"}}
{"return": ""}
--- Adding "NewC" bitmap, removing "NewTwo" bitmap ---
{"execute": "block-dirty-bitmap-add", "arguments": {"name": "NewC", "node": "drive0", "persistent": true}}
{"return": {}}
{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "Newtwo", "node": "drive0"}}
{"return": {}}
--- Forcing flush of bitmaps to disk ---
--- 4th Boot (Verification and Cleanup) ---
{
"bitmaps": {
"drive0": [
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "NewB",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "NewC",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 32768,
"name": "Small",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 65536,
"name": "Medium",
"persistent": true,
"recording": true,
"status": "active"
},
{
"busy": false,
"count": 0,
"granularity": 131072,
"name": "Large",
"persistent": true,
"recording": true,
"status": "active"
}
]
}
}
--- Removing all Bitmaps ---
{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "Small", "node": "drive0"}}
{"return": {}}
{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "Medium", "node": "drive0"}}
{"return": {}}
{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "Large", "node": "drive0"}}
{"return": {}}
{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "NewB", "node": "drive0"}}
{"return": {}}
{"execute": "block-dirty-bitmap-remove", "arguments": {"name": "NewC", "node": "drive0"}}
{"return": {}}
{
"bitmaps": {
"drive0": []
}
}
--- Done ---

View file

@ -244,3 +244,4 @@
243 rw auto quick
244 rw auto quick
245 rw auto
246 rw auto quick