iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge back new changes related to system sleep for v6.6.

Browse Source
This commit is contained in:
Rafael J. Wysocki 2023-07-31 20:05:02 +02:00
commit 3fc68004d4
1 changed files with 149 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
kernel/power/snapshot.c
View File

@ -404,6 +404,7 @@ struct bm_position {
struct mem_zone_bm_rtree *zone; struct mem_zone_bm_rtree *zone;
struct rtree_node *node; struct rtree_node *node;
unsigned long node_pfn; unsigned long node_pfn;
unsigned long cur_pfn;
int node_bit; int node_bit;
}; };
@ -589,6 +590,7 @@ static void memory_bm_position_reset(struct memory_bitmap *bm)
bm->cur.node = list_entry(bm->cur.zone->leaves.next, bm->cur.node = list_entry(bm->cur.zone->leaves.next,
struct rtree_node, list); struct rtree_node, list);
bm->cur.node_pfn = 0; bm->cur.node_pfn = 0;
bm->cur.cur_pfn = BM_END_OF_MAP;
bm->cur.node_bit = 0; bm->cur.node_bit = 0;
} }
@ -799,6 +801,7 @@ node_found:
bm->cur.zone = zone; bm->cur.zone = zone;
bm->cur.node = node; bm->cur.node = node;
bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK; bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;
bm->cur.cur_pfn = pfn;
/* Set return values */ /* Set return values */
*addr = node->data; *addr = node->data;
@ -850,6 +853,11 @@ static void memory_bm_clear_current(struct memory_bitmap *bm)
clear_bit(bit, bm->cur.node->data); clear_bit(bit, bm->cur.node->data);
} }
static unsigned long memory_bm_get_current(struct memory_bitmap *bm)
{
return bm->cur.cur_pfn;
}
static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn) static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
{ {
void *addr; void *addr;
@ -929,10 +937,12 @@ static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
if (bit < bits) { if (bit < bits) {
pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit; pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;
bm->cur.node_bit = bit + 1; bm->cur.node_bit = bit + 1;
bm->cur.cur_pfn = pfn;
return pfn; return pfn;
} }
} while (rtree_next_node(bm)); } while (rtree_next_node(bm));
bm->cur.cur_pfn = BM_END_OF_MAP;
return BM_END_OF_MAP; return BM_END_OF_MAP;
} }
@ -1423,14 +1433,19 @@ static unsigned int count_data_pages(void)
/* /*
* This is needed, because copy_page and memcpy are not usable for copying * This is needed, because copy_page and memcpy are not usable for copying
* task structs. * task structs. Returns true if the page was filled with only zeros,
* otherwise false.
*/ */
static inline void do_copy_page(long *dst, long *src) static inline bool do_copy_page(long *dst, long *src)
{ {
long z = 0;
int n; int n;
for (n = PAGE_SIZE / sizeof(long); n; n--) for (n = PAGE_SIZE / sizeof(long); n; n--) {
z |= *src;
*dst++ = *src++; *dst++ = *src++;
}
return !z;
} }
/** /**
@ -1439,17 +1454,21 @@ static inline void do_copy_page(long *dst, long *src)
* Check if the page we are going to copy is marked as present in the kernel * Check if the page we are going to copy is marked as present in the kernel
* page tables. This always is the case if CONFIG_DEBUG_PAGEALLOC or * page tables. This always is the case if CONFIG_DEBUG_PAGEALLOC or
* CONFIG_ARCH_HAS_SET_DIRECT_MAP is not set. In that case kernel_page_present() * CONFIG_ARCH_HAS_SET_DIRECT_MAP is not set. In that case kernel_page_present()
* always returns 'true'. * always returns 'true'. Returns true if the page was entirely composed of
* zeros, otherwise it will return false.
*/ */
static void safe_copy_page(void *dst, struct page *s_page) static bool safe_copy_page(void *dst, struct page *s_page)
{ {
bool zeros_only;
if (kernel_page_present(s_page)) { if (kernel_page_present(s_page)) {
do_copy_page(dst, page_address(s_page)); zeros_only = do_copy_page(dst, page_address(s_page));
} else { } else {
hibernate_map_page(s_page); hibernate_map_page(s_page);
do_copy_page(dst, page_address(s_page)); zeros_only = do_copy_page(dst, page_address(s_page));
hibernate_unmap_page(s_page); hibernate_unmap_page(s_page);
} }
return zeros_only;
} }
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM
@ -1459,17 +1478,18 @@ static inline struct page *page_is_saveable(struct zone *zone, unsigned long pfn
saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn); saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);
} }
static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) static bool copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
{ {
struct page *s_page, *d_page; struct page *s_page, *d_page;
void *src, *dst; void *src, *dst;
bool zeros_only;
s_page = pfn_to_page(src_pfn); s_page = pfn_to_page(src_pfn);
d_page = pfn_to_page(dst_pfn); d_page = pfn_to_page(dst_pfn);
if (PageHighMem(s_page)) { if (PageHighMem(s_page)) {
src = kmap_atomic(s_page); src = kmap_atomic(s_page);
dst = kmap_atomic(d_page); dst = kmap_atomic(d_page);
do_copy_page(dst, src); zeros_only = do_copy_page(dst, src);
kunmap_atomic(dst); kunmap_atomic(dst);
kunmap_atomic(src); kunmap_atomic(src);
} else { } else {
@ -1478,30 +1498,39 @@ static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
* The page pointed to by src may contain some kernel * The page pointed to by src may contain some kernel
* data modified by kmap_atomic() * data modified by kmap_atomic()
*/ */
safe_copy_page(buffer, s_page); zeros_only = safe_copy_page(buffer, s_page);
dst = kmap_atomic(d_page); dst = kmap_atomic(d_page);
copy_page(dst, buffer); copy_page(dst, buffer);
kunmap_atomic(dst); kunmap_atomic(dst);
} else { } else {
safe_copy_page(page_address(d_page), s_page); zeros_only = safe_copy_page(page_address(d_page), s_page);
} }
} }
return zeros_only;
} }
#else #else
#define page_is_saveable(zone, pfn) saveable_page(zone, pfn) #define page_is_saveable(zone, pfn) saveable_page(zone, pfn)
static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) static inline int copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
{ {
safe_copy_page(page_address(pfn_to_page(dst_pfn)), return safe_copy_page(page_address(pfn_to_page(dst_pfn)),
pfn_to_page(src_pfn)); pfn_to_page(src_pfn));
} }
#endif /* CONFIG_HIGHMEM */ #endif /* CONFIG_HIGHMEM */
static void copy_data_pages(struct memory_bitmap *copy_bm, /*
struct memory_bitmap *orig_bm) * Copy data pages will copy all pages into pages pulled from the copy_bm.
* If a page was entirely filled with zeros it will be marked in the zero_bm.
*
* Returns the number of pages copied.
*/
static unsigned long copy_data_pages(struct memory_bitmap *copy_bm,
struct memory_bitmap *orig_bm,
struct memory_bitmap *zero_bm)
{ {
unsigned long copied_pages = 0;
struct zone *zone; struct zone *zone;
unsigned long pfn; unsigned long pfn, copy_pfn;
for_each_populated_zone(zone) { for_each_populated_zone(zone) {
unsigned long max_zone_pfn; unsigned long max_zone_pfn;
@ -1514,18 +1543,29 @@ static void copy_data_pages(struct memory_bitmap *copy_bm,
} }
memory_bm_position_reset(orig_bm); memory_bm_position_reset(orig_bm);
memory_bm_position_reset(copy_bm); memory_bm_position_reset(copy_bm);
copy_pfn = memory_bm_next_pfn(copy_bm);
for(;;) { for(;;) {
pfn = memory_bm_next_pfn(orig_bm); pfn = memory_bm_next_pfn(orig_bm);
if (unlikely(pfn == BM_END_OF_MAP)) if (unlikely(pfn == BM_END_OF_MAP))
break; break;
copy_data_page(memory_bm_next_pfn(copy_bm), pfn); if (copy_data_page(copy_pfn, pfn)) {
memory_bm_set_bit(zero_bm, pfn);
/* Use this copy_pfn for a page that is not full of zeros */
continue;
} }
copied_pages++;
copy_pfn = memory_bm_next_pfn(copy_bm);
}
return copied_pages;
} }
/* Total number of image pages */ /* Total number of image pages */
static unsigned int nr_copy_pages; static unsigned int nr_copy_pages;
/* Number of pages needed for saving the original pfns of the image pages */ /* Number of pages needed for saving the original pfns of the image pages */
static unsigned int nr_meta_pages; static unsigned int nr_meta_pages;
/* Number of zero pages */
static unsigned int nr_zero_pages;
/* /*
* Numbers of normal and highmem page frames allocated for hibernation image * Numbers of normal and highmem page frames allocated for hibernation image
* before suspending devices. * before suspending devices.
@ -1546,6 +1586,9 @@ static struct memory_bitmap orig_bm;
*/ */
static struct memory_bitmap copy_bm; static struct memory_bitmap copy_bm;
/* Memory bitmap which tracks which saveable pages were zero filled. */
static struct memory_bitmap zero_bm;
/** /**
* swsusp_free - Free pages allocated for hibernation image. * swsusp_free - Free pages allocated for hibernation image.
* *
@ -1590,6 +1633,7 @@ loop:
out: out:
nr_copy_pages = 0; nr_copy_pages = 0;
nr_meta_pages = 0; nr_meta_pages = 0;
nr_zero_pages = 0;
restore_pblist = NULL; restore_pblist = NULL;
buffer = NULL; buffer = NULL;
alloc_normal = 0; alloc_normal = 0;
@ -1808,8 +1852,15 @@ int hibernate_preallocate_memory(void)
goto err_out; goto err_out;
} }
error = memory_bm_create(&zero_bm, GFP_IMAGE, PG_ANY);
if (error) {
pr_err("Cannot allocate zero bitmap\n");
goto err_out;
}
alloc_normal = 0; alloc_normal = 0;
alloc_highmem = 0; alloc_highmem = 0;
nr_zero_pages = 0;
/* Count the number of saveable data pages. */ /* Count the number of saveable data pages. */
save_highmem = count_highmem_pages(); save_highmem = count_highmem_pages();
@ -2089,19 +2140,19 @@ asmlinkage __visible int swsusp_save(void)
* Kill them. * Kill them.
*/ */
drain_local_pages(NULL); drain_local_pages(NULL);
copy_data_pages(&copy_bm, &orig_bm); nr_copy_pages = copy_data_pages(&copy_bm, &orig_bm, &zero_bm);
/* /*
* End of critical section. From now on, we can write to memory, * End of critical section. From now on, we can write to memory,
* but we should not touch disk. This specially means we must _not_ * but we should not touch disk. This specially means we must _not_
* touch swap space! Except we must write out our image of course. * touch swap space! Except we must write out our image of course.
*/ */
nr_pages += nr_highmem; nr_pages += nr_highmem;
nr_copy_pages = nr_pages; /* We don't actually copy the zero pages */
nr_zero_pages = nr_pages - nr_copy_pages;
nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE); nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
pr_info("Image created (%d pages copied)\n", nr_pages); pr_info("Image created (%d pages copied, %d zero pages)\n", nr_copy_pages, nr_zero_pages);
return 0; return 0;
} }
@ -2146,15 +2197,22 @@ static int init_header(struct swsusp_info *info)
return init_header_complete(info); return init_header_complete(info);
} }
#define ENCODED_PFN_ZERO_FLAG ((unsigned long)1 << (BITS_PER_LONG - 1))
#define ENCODED_PFN_MASK (~ENCODED_PFN_ZERO_FLAG)
/** /**
* pack_pfns - Prepare PFNs for saving. * pack_pfns - Prepare PFNs for saving.
* @bm: Memory bitmap. * @bm: Memory bitmap.
* @buf: Memory buffer to store the PFNs in. * @buf: Memory buffer to store the PFNs in.
* @zero_bm: Memory bitmap containing PFNs of zero pages.
* *
* PFNs corresponding to set bits in @bm are stored in the area of memory * PFNs corresponding to set bits in @bm are stored in the area of memory
* pointed to by @buf (1 page at a time). * pointed to by @buf (1 page at a time). Pages which were filled with only
* zeros will have the highest bit set in the packed format to distinguish
* them from PFNs which will be contained in the image file.
*/ */
static inline void pack_pfns(unsigned long *buf, struct memory_bitmap *bm) static inline void pack_pfns(unsigned long *buf, struct memory_bitmap *bm,
struct memory_bitmap *zero_bm)
{ {
int j; int j;
@ -2162,6 +2220,8 @@ static inline void pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
buf[j] = memory_bm_next_pfn(bm); buf[j] = memory_bm_next_pfn(bm);
if (unlikely(buf[j] == BM_END_OF_MAP)) if (unlikely(buf[j] == BM_END_OF_MAP))
break; break;
if (memory_bm_test_bit(zero_bm, buf[j]))
buf[j] |= ENCODED_PFN_ZERO_FLAG;
} }
} }
@ -2203,7 +2263,7 @@ int snapshot_read_next(struct snapshot_handle *handle)
memory_bm_position_reset(&copy_bm); memory_bm_position_reset(&copy_bm);
} else if (handle->cur <= nr_meta_pages) { } else if (handle->cur <= nr_meta_pages) {
clear_page(buffer); clear_page(buffer);
pack_pfns(buffer, &orig_bm); pack_pfns(buffer, &orig_bm, &zero_bm);
} else { } else {
struct page *page; struct page *page;
@ -2299,24 +2359,35 @@ static int load_header(struct swsusp_info *info)
* unpack_orig_pfns - Set bits corresponding to given PFNs in a memory bitmap. * unpack_orig_pfns - Set bits corresponding to given PFNs in a memory bitmap.
* @bm: Memory bitmap. * @bm: Memory bitmap.
* @buf: Area of memory containing the PFNs. * @buf: Area of memory containing the PFNs.
* @zero_bm: Memory bitmap with the zero PFNs marked.
* *
* For each element of the array pointed to by @buf (1 page at a time), set the * For each element of the array pointed to by @buf (1 page at a time), set the
* corresponding bit in @bm. * corresponding bit in @bm. If the page was originally populated with only
* zeros then a corresponding bit will also be set in @zero_bm.
*/ */
static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm) static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm,
struct memory_bitmap *zero_bm)
{ {
unsigned long decoded_pfn;
bool zero;
int j; int j;
for (j = 0; j < PAGE_SIZE / sizeof(long); j++) { for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
if (unlikely(buf[j] == BM_END_OF_MAP)) if (unlikely(buf[j] == BM_END_OF_MAP))
break; break;
if (pfn_valid(buf[j]) && memory_bm_pfn_present(bm, buf[j])) { zero = !!(buf[j] & ENCODED_PFN_ZERO_FLAG);
memory_bm_set_bit(bm, buf[j]); decoded_pfn = buf[j] & ENCODED_PFN_MASK;
if (pfn_valid(decoded_pfn) && memory_bm_pfn_present(bm, decoded_pfn)) {
memory_bm_set_bit(bm, decoded_pfn);
if (zero) {
memory_bm_set_bit(zero_bm, decoded_pfn);
nr_zero_pages++;
}
} else { } else {
if (!pfn_valid(buf[j])) if (!pfn_valid(decoded_pfn))
pr_err(FW_BUG "Memory map mismatch at 0x%llx after hibernation\n", pr_err(FW_BUG "Memory map mismatch at 0x%llx after hibernation\n",
(unsigned long long)PFN_PHYS(buf[j])); (unsigned long long)PFN_PHYS(decoded_pfn));
return -EFAULT; return -EFAULT;
} }
} }
@ -2538,6 +2609,7 @@ static inline void free_highmem_data(void) {}
* prepare_image - Make room for loading hibernation image. * prepare_image - Make room for loading hibernation image.
* @new_bm: Uninitialized memory bitmap structure. * @new_bm: Uninitialized memory bitmap structure.
* @bm: Memory bitmap with unsafe pages marked. * @bm: Memory bitmap with unsafe pages marked.
* @zero_bm: Memory bitmap containing the zero pages.
* *
* Use @bm to mark the pages that will be overwritten in the process of * Use @bm to mark the pages that will be overwritten in the process of
* restoring the system memory state from the suspend image ("unsafe" pages) * restoring the system memory state from the suspend image ("unsafe" pages)
@ -2548,10 +2620,15 @@ static inline void free_highmem_data(void) {}
* pages will be used for just yet. Instead, we mark them all as allocated and * pages will be used for just yet. Instead, we mark them all as allocated and
* create a lists of "safe" pages to be used later. On systems with high * create a lists of "safe" pages to be used later. On systems with high
* memory a list of "safe" highmem pages is created too. * memory a list of "safe" highmem pages is created too.
*
* Because it was not known which pages were unsafe when @zero_bm was created,
* make a copy of it and recreate it within safe pages.
*/ */
static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm) static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm,
struct memory_bitmap *zero_bm)
{ {
unsigned int nr_pages, nr_highmem; unsigned int nr_pages, nr_highmem;
struct memory_bitmap tmp;
struct linked_page *lp; struct linked_page *lp;
int error; int error;
@ -2568,6 +2645,24 @@ static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
duplicate_memory_bitmap(new_bm, bm); duplicate_memory_bitmap(new_bm, bm);
memory_bm_free(bm, PG_UNSAFE_KEEP); memory_bm_free(bm, PG_UNSAFE_KEEP);
/* Make a copy of zero_bm so it can be created in safe pages */
error = memory_bm_create(&tmp, GFP_ATOMIC, PG_ANY);
if (error)
goto Free;
duplicate_memory_bitmap(&tmp, zero_bm);
memory_bm_free(zero_bm, PG_UNSAFE_KEEP);
/* Recreate zero_bm in safe pages */
error = memory_bm_create(zero_bm, GFP_ATOMIC, PG_SAFE);
if (error)
goto Free;
duplicate_memory_bitmap(zero_bm, &tmp);
memory_bm_free(&tmp, PG_UNSAFE_KEEP);
/* At this point zero_bm is in safe pages and it can be used for restoring. */
if (nr_highmem > 0) { if (nr_highmem > 0) {
error = prepare_highmem_image(bm, &nr_highmem); error = prepare_highmem_image(bm, &nr_highmem);
if (error) if (error)
@ -2582,7 +2677,7 @@ static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
* *
* nr_copy_pages cannot be less than allocated_unsafe_pages too. * nr_copy_pages cannot be less than allocated_unsafe_pages too.
*/ */
nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages; nr_pages = (nr_zero_pages + nr_copy_pages) - nr_highmem - allocated_unsafe_pages;
nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE); nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
while (nr_pages > 0) { while (nr_pages > 0) {
lp = get_image_page(GFP_ATOMIC, PG_SAFE); lp = get_image_page(GFP_ATOMIC, PG_SAFE);
@ -2595,7 +2690,7 @@ static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
nr_pages--; nr_pages--;
} }
/* Preallocate memory for the image */ /* Preallocate memory for the image */
nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages; nr_pages = (nr_zero_pages + nr_copy_pages) - nr_highmem - allocated_unsafe_pages;
while (nr_pages > 0) { while (nr_pages > 0) {
lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC); lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
if (!lp) { if (!lp) {
@ -2683,8 +2778,9 @@ int snapshot_write_next(struct snapshot_handle *handle)
static struct chain_allocator ca; static struct chain_allocator ca;
int error = 0; int error = 0;
next:
/* Check if we have already loaded the entire image */ /* Check if we have already loaded the entire image */
if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages)
return 0; return 0;
handle->sync_read = 1; handle->sync_read = 1;
@ -2709,19 +2805,26 @@ int snapshot_write_next(struct snapshot_handle *handle)
if (error) if (error)
return error; return error;
error = memory_bm_create(&zero_bm, GFP_ATOMIC, PG_ANY);
if (error)
return error;
nr_zero_pages = 0;
hibernate_restore_protection_begin(); hibernate_restore_protection_begin();
} else if (handle->cur <= nr_meta_pages + 1) { } else if (handle->cur <= nr_meta_pages + 1) {
error = unpack_orig_pfns(buffer, &copy_bm); error = unpack_orig_pfns(buffer, &copy_bm, &zero_bm);
if (error) if (error)
return error; return error;
if (handle->cur == nr_meta_pages + 1) { if (handle->cur == nr_meta_pages + 1) {
error = prepare_image(&orig_bm, &copy_bm); error = prepare_image(&orig_bm, &copy_bm, &zero_bm);
if (error) if (error)
return error; return error;
chain_init(&ca, GFP_ATOMIC, PG_SAFE); chain_init(&ca, GFP_ATOMIC, PG_SAFE);
memory_bm_position_reset(&orig_bm); memory_bm_position_reset(&orig_bm);
memory_bm_position_reset(&zero_bm);
restore_pblist = NULL; restore_pblist = NULL;
handle->buffer = get_buffer(&orig_bm, &ca); handle->buffer = get_buffer(&orig_bm, &ca);
handle->sync_read = 0; handle->sync_read = 0;
@ -2738,6 +2841,14 @@ int snapshot_write_next(struct snapshot_handle *handle)
handle->sync_read = 0; handle->sync_read = 0;
} }
handle->cur++; handle->cur++;
/* Zero pages were not included in the image, memset it and move on. */
if (handle->cur > nr_meta_pages + 1 &&
memory_bm_test_bit(&zero_bm, memory_bm_get_current(&orig_bm))) {
memset(handle->buffer, 0, PAGE_SIZE);
goto next;
}
return PAGE_SIZE; return PAGE_SIZE;
} }
@ -2754,7 +2865,7 @@ void snapshot_write_finalize(struct snapshot_handle *handle)
copy_last_highmem_page(); copy_last_highmem_page();
hibernate_restore_protect_page(handle->buffer); hibernate_restore_protect_page(handle->buffer);
/* Do that only if we have loaded the image entirely */ /* Do that only if we have loaded the image entirely */
if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) { if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages) {
memory_bm_recycle(&orig_bm); memory_bm_recycle(&orig_bm);
free_highmem_data(); free_highmem_data();
} }
@ -2763,7 +2874,7 @@ void snapshot_write_finalize(struct snapshot_handle *handle)
int snapshot_image_loaded(struct snapshot_handle *handle) int snapshot_image_loaded(struct snapshot_handle *handle)
{ {
return !(!nr_copy_pages || !last_highmem_page_copied() || return !(!nr_copy_pages || !last_highmem_page_copied() ||
handle->cur <= nr_meta_pages + nr_copy_pages); handle->cur <= nr_meta_pages + nr_copy_pages + nr_zero_pages);
} }
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM