test_migration() {
  # setup a second LXD
  local LXD2_DIR LXD2_ADDR lxd_backend
  # shellcheck disable=2153
  lxd_backend=$(storage_backend "$LXD_DIR")

  if [ "${lxd_backend}" = "dir" ] && uname -r | grep -- -kvm$; then
    echo "==> SKIP: the -kvm kernel flavor is does not work for this test on ${lxd_backend}"
    return
  fi

  LXD2_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
  spawn_lxd "${LXD2_DIR}" true
  LXD2_ADDR=$(< "${LXD2_DIR}/lxd.addr")

  token="$(lxc config trust add --name foo -q)"
  # shellcheck disable=2153
  lxc_remote remote add l1 "${LXD_ADDR}" --token "${token}"

  token="$(LXD_DIR=${LXD2_DIR} lxc config trust add --name foo -q)"
  lxc_remote remote add l2 "${LXD2_ADDR}" --token "${token}"

  migration "$LXD2_DIR"

  # This should only run on lvm and when the backend is not random. Otherwise
  # we might perform existence checks for files or dirs that won't be available
  # since the logical volume is not mounted when the container is not running.
  # shellcheck disable=2153
  if [ "${LXD_BACKEND}" = "lvm" ]; then
    # Test that non-thinpool lvm backends work fine with migration.

    local storage_pool1 storage_pool2
    # shellcheck disable=2153
    storage_pool1="lxdtest-$(basename "${LXD_DIR}")-non-thinpool-lvm-migration"
    storage_pool2="lxdtest-$(basename "${LXD2_DIR}")-non-thinpool-lvm-migration"
    lxc_remote storage create l1:"$storage_pool1" lvm lvm.use_thinpool=false size=1GiB volume.size=25MiB
    lxc_remote profile device set l1:default root pool "$storage_pool1"

    lxc_remote storage create l2:"$storage_pool2" lvm lvm.use_thinpool=false size=1GiB volume.size=25MiB
    lxc_remote profile device set l2:default root pool "$storage_pool2"

    migration "$LXD2_DIR"

    lxc_remote profile device set l1:default root pool "lxdtest-$(basename "${LXD_DIR}")"
    lxc_remote profile device set l2:default root pool "lxdtest-$(basename "${LXD2_DIR}")"

    lxc_remote storage delete l1:"$storage_pool1"
    lxc_remote storage delete l2:"$storage_pool2"
  fi

  if [ "${LXD_BACKEND}" = "zfs" ]; then
    # Test that block mode zfs backends work fine with migration.
    for fs in "ext4" "btrfs" "xfs"; do
      local storage_pool1 storage_pool2
      # shellcheck disable=2153
      storage_pool1="lxdtest-$(basename "${LXD_DIR}")-block-mode-${fs}"
      storage_pool2="lxdtest-$(basename "${LXD2_DIR}")-block-mode-${fs}"
      lxc_remote storage create l1:"$storage_pool1" zfs size=1GiB volume.zfs.block_mode=true volume.block.filesystem="${fs}"
      lxc_remote profile device set l1:default root pool "$storage_pool1"

      lxc_remote storage create l2:"$storage_pool2" zfs size=1GiB volume.zfs.block_mode=true volume.block.filesystem="${fs}"
      lxc_remote profile device set l2:default root pool "$storage_pool2"

      migration "$LXD2_DIR"

      lxc_remote profile device set l1:default root pool "lxdtest-$(basename "${LXD_DIR}")"
      lxc_remote profile device set l2:default root pool "lxdtest-$(basename "${LXD2_DIR}")"

      lxc_remote storage delete l1:"$storage_pool1"
      lxc_remote storage delete l2:"$storage_pool2"
    done
  fi

  # Test config overrides for migration of instance with snapshots
  lxc_remote network create l1:foonet ipv4.address=10.100.10.1/24
  lxc_remote network create l2:foonet2 ipv4.address=10.100.100.1/24
  lxc_remote init --empty l1:u1
  lxc_remote config device add l1:u1 eth1 nic name=eth1 network=foonet ipv4.address=10.100.10.10
  lxc_remote snapshot l1:u1 snap

  lxc_remote copy l1:u1 l2: -d eth1,ipv4.address=10.100.100.10 -d eth1,network=foonet2

  lxc_remote delete l1:u1
  lxc_remote delete l2:u1
  lxc_remote network delete l1:foonet
  lxc_remote network delete l2:foonet2

  lxc_remote remote remove l1
  lxc_remote remote remove l2
  kill_lxd "$LXD2_DIR"
}

migration() {
  local lxd2_dir lxd_backend lxd2_backend
  lxd2_dir="$1"
  lxd_backend=$(storage_backend "$LXD_DIR")
  lxd2_backend=$(storage_backend "$lxd2_dir")
  ensure_import_testimage

  lxc_remote init testimage nonlive
  # test moving snapshots
  lxc_remote config set l1:nonlive user.tester foo
  lxc_remote snapshot l1:nonlive
  lxc_remote config unset l1:nonlive user.tester
  lxc_remote move l1:nonlive l2:
  lxc_remote config show l2:nonlive/snap0 | grep user.tester | grep foo

  # This line exists so that the container's storage volume is mounted when we
  # perform existence check for various files.
  lxc_remote start l2:nonlive
  # FIXME: make this backend agnostic
  if [ "$lxd2_backend" != "lvm" ] && [ "$lxd2_backend" != "zfs" ] && [ "$lxd2_backend" != "ceph" ]; then
    [ -d "${lxd2_dir}/containers/nonlive/rootfs" ]
  fi
  lxc_remote stop l2:nonlive --force

  [ ! -d "${LXD_DIR}/containers/nonlive" ]
  # FIXME: make this backend agnostic
  if [ "$lxd2_backend" = "dir" ]; then
    [ -d "${lxd2_dir}/snapshots/nonlive/snap0/rootfs/bin" ]
  fi

  lxc_remote copy l2:nonlive l1:nonlive2 --mode=push
  # This line exists so that the container's storage volume is mounted when we
  # perform existence check for various files.
  lxc_remote start l2:nonlive
  [ -d "${LXD_DIR}/containers/nonlive2" ]
  # FIXME: make this backend agnostic
  if [ "$lxd2_backend" != "lvm" ] && [ "$lxd2_backend" != "zfs" ] && [ "$lxd2_backend" != "ceph" ]; then
    [ -d "${lxd2_dir}/containers/nonlive/rootfs/bin" ]
  fi

  # FIXME: make this backend agnostic
  if [ "$lxd_backend" = "dir" ]; then
    [ -d "${LXD_DIR}/snapshots/nonlive2/snap0/rootfs/bin" ]
  fi

  lxc_remote copy l1:nonlive2/snap0 l2:nonlive3 --mode=relay
  # FIXME: make this backend agnostic
  if [ "$lxd2_backend" != "lvm" ] && [ "$lxd2_backend" != "zfs" ] && [ "$lxd2_backend" != "ceph" ]; then
    [ -d "${lxd2_dir}/containers/nonlive3/rootfs/bin" ]
  fi
  lxc_remote delete l2:nonlive3 --force

  lxc_remote stop l2:nonlive --force
  lxc_remote copy l2:nonlive l2:nonlive2 --mode=push
  # should have the same base image tag
  [ "$(lxc_remote config get l2:nonlive volatile.base_image)" = "$(lxc_remote config get l2:nonlive2 volatile.base_image)" ]
  # check that nonlive2 has a new addr in volatile
  [ "$(lxc_remote config get l2:nonlive volatile.eth0.hwaddr)" != "$(lxc_remote config get l2:nonlive2 volatile.eth0.hwaddr)" ]

  lxc_remote config unset l2:nonlive volatile.base_image
  lxc_remote copy l2:nonlive l1:nobase
  lxc_remote delete l1:nobase

  lxc_remote start l1:nonlive2
  lxc_remote list l1: | grep RUNNING | grep nonlive2
  lxc_remote delete l1:nonlive2 l2:nonlive2 --force

  lxc_remote launch testimage cccp
  lxc_remote copy l1:cccp l2:udssr --stateless
  lxc_remote delete l2:udssr --force
  lxc_remote copy l1:cccp l2:udssr --stateless --mode=push
  lxc_remote delete l2:udssr --force
  lxc_remote copy l1:cccp l2:udssr --stateless --mode=relay
  lxc_remote delete l2:udssr --force

  lxc_remote move l1:cccp l2:udssr --stateless
  lxc_remote delete l2:udssr --force
  lxc_remote launch testimage cccp
  lxc_remote move l1:cccp l2:udssr --stateless --mode=push
  lxc_remote delete l2:udssr --force
  lxc_remote launch testimage cccp
  lxc_remote move l1:cccp l2:udssr --stateless --mode=relay
  lxc_remote delete l2:udssr --force

  lxc_remote start l2:nonlive
  lxc_remote list l2: | grep RUNNING | grep nonlive
  lxc_remote delete l2:nonlive --force

  # Get container's pool.
  pool=$(lxc config profile device get default root pool)
  remote_pool=$(lxc_remote config profile device get l2:default root pool)

  # Test container only copies
  lxc init testimage cccp

  lxc storage volume set "${pool}" container/cccp user.foo=snap0
  echo "before" | lxc file push - cccp/blah
  lxc snapshot cccp
  lxc storage volume set "${pool}" container/cccp user.foo=snap1
  lxc snapshot cccp
  echo "after" | lxc file push - cccp/blah
  lxc storage volume set "${pool}" container/cccp user.foo=postsnap1

  # Check storage volume creation times are set.
  [ "$(lxc query /1.0/storage-pools/"${pool}"/volumes/container/cccp | jq -r .created_at)" != "0001-01-01T00:00:00Z" ]
  [ "$(lxc query /1.0/storage-pools/"${pool}"/volumes/container/cccp/snapshots/snap0 | jq -r .created_at)" != "0001-01-01T00:00:00Z" ]

  # Local container only copy.
  lxc copy cccp udssr --instance-only
  [ "$(lxc list -f csv -c S udssr)" = "0" ]
  [ "$(lxc file pull udssr/blah -)" = "after" ]
  lxc delete udssr

  # Local container with snapshots copy.
  lxc copy cccp udssr
  [ "$(lxc list -f csv -c S udssr)" = "2" ]
  [ "$(lxc file pull udssr/blah -)" = "after" ]
  lxc storage volume show "${pool}" container/udssr
  [ "$(lxc storage volume get "${pool}" container/udssr user.foo)" = "postsnap1" ]
  [ "$(lxc storage volume get "${pool}" container/udssr/snap0 user.foo)" = "snap0" ]
  [ "$(lxc storage volume get "${pool}" container/udssr/snap1 user.foo)" = "snap1" ]
  lxc delete udssr

  # Remote container only copy.
  lxc_remote copy l1:cccp l2:udssr --instance-only
  [ "$(lxc_remote list -f csv -c S l2:udssr)" = "0" ]
  [ "$(lxc_remote file pull l2:udssr/blah -)" = "after" ]
  lxc_remote delete l2:udssr

  # Remote container with snapshots copy.
  lxc_remote copy l1:cccp l2:udssr
  [ "$(lxc_remote list -f csv -c S l2:udssr)" = "2" ]
  [ "$(lxc_remote file pull l2:udssr/blah -)" = "after" ]
  lxc_remote storage volume show l2:"${remote_pool}" container/udssr
  [ "$(lxc_remote storage volume get l2:"${remote_pool}" container/udssr user.foo)" = "postsnap1" ]
  [ "$(lxc_remote storage volume get l2:"${remote_pool}" container/udssr/snap0 user.foo)" = "snap0" ]
  [ "$(lxc_remote storage volume get l2:"${remote_pool}" container/udssr/snap1 user.foo)" = "snap1" ]
  lxc_remote delete l2:udssr

  # Remote container only move.
  lxc_remote move l1:cccp l2:udssr --instance-only --mode=relay
  ! lxc_remote info l1:cccp || false
  [ "$(lxc_remote list -f csv -c S l2:udssr)" = "0" ]
  lxc_remote delete l2:udssr

  lxc_remote init testimage l1:cccp
  lxc_remote snapshot l1:cccp
  lxc_remote snapshot l1:cccp

  # Remote container with snapshots move.
  lxc_remote move l1:cccp l2:udssr --mode=push
  ! lxc_remote info l1:cccp || false
  [ "$(lxc_remote list -f csv -c S l2:udssr)" = "2" ]
  lxc_remote delete l2:udssr

  # Test container only copies
  lxc init testimage cccp
  lxc snapshot cccp
  lxc snapshot cccp

  # Local container with snapshots move.
  lxc move cccp udssr --mode=pull
  ! lxc info cccp || false
  [ "$(lxc list -f csv -c S udssr)" = "2" ]
  lxc delete udssr

  if [ "$lxd_backend" = "zfs" ]; then
    # Test container only copies when zfs.clone_copy is set to false.
    lxc storage set "lxdtest-$(basename "${LXD_DIR}")" zfs.clone_copy false
    lxc init testimage cccp
    lxc snapshot cccp
    lxc snapshot cccp

    # Test container only copies when zfs.clone_copy is set to false.
    lxc copy cccp udssr --instance-only
    [ "$(lxc list -f csv -c S udssr)" = "0" ]
    lxc delete udssr

    # Test container with snapshots copy when zfs.clone_copy is set to false.
    lxc copy cccp udssr
    [ "$(lxc list -f csv -c S udssr)" = "2" ]
    lxc delete cccp
    lxc delete udssr

    lxc storage unset "lxdtest-$(basename "${LXD_DIR}")" zfs.clone_copy
  fi

  lxc_remote init testimage l1:c1
  lxc_remote copy l1:c1 l2:c2
  lxc_remote copy l1:c1 l2:c2 --refresh

  lxc_remote start l1:c1 l2:c2

  # Make sure the testfile doesn't exist
  ! lxc file pull l1:c1 -- /root/testfile1 || false
  ! lxc file pull l2:c2 -- /root/testfile1 || false

  #lxc_remote start l1:c1 l2:c2

  # Containers may not be running when refreshing
  ! lxc_remote copy l1:c1 l2:c2 --refresh || false

  # Create test file in c1
  echo test | lxc_remote file push - l1:c1/root/testfile1

  lxc_remote stop -f l1:c1 l2:c2

  # Refresh the container and validate the contents
  lxc_remote copy l1:c1 l2:c2 --refresh
  lxc_remote start l2:c2
  [ "$(lxc_remote file pull l2:c2/root/testfile1 -)" = "test" ]
  lxc_remote stop -f l2:c2

  # Change the files modification time by adding one nanosecond.
  # Perform the change on the test runner since the busybox instances `touch` doesn't support setting nanoseconds.
  lxc_remote start l1:c1
  c1_pid="$(lxc_remote list -f csv -c p l1:c1)"
  mtime_old="$(stat -c %y "/proc/${c1_pid}/root/root/testfile1")"
  mtime_old_ns="$(date -d "$mtime_old" +%N | sed 's/^0*//')"

  # Ensure the final nanoseconds are padded with zeros to create a valid format.
  mtime_new_ns="$(printf "%09d\n" "$((mtime_old_ns+1))")"
  mtime_new="$(date -d "$mtime_old" "+%Y-%m-%d %H:%M:%S.${mtime_new_ns} %z")"
  lxc_remote stop -f l1:c1

  # Before setting the new mtime create a local copy too.
  lxc_remote copy l1:c1 l1:c2

  # Change the modification time.
  lxc_remote start l1:c1
  c1_pid="$(lxc_remote list -f csv -c p l1:c1)"
  touch -m -d "$mtime_new" "/proc/${c1_pid}/root/root/testfile1"
  lxc_remote stop -f l1:c1

  # Starting from rsync 3.1.3 it should discover the change of +1 nanosecond.
  # Check if the file got refreshed to a different remote.
  lxc_remote copy l1:c1 l2:c2 --refresh
  lxc_remote start l1:c1 l2:c2
  c1_pid="$(lxc_remote list -f csv -c p l1:c1)"
  c2_pid="$(lxc_remote list -f csv -c p l2:c2)"
  [ "$(stat "/proc/${c1_pid}/root/root/testfile1" -c %y)" = "$(stat "/proc/${c2_pid}/root/root/testfile1" -c %y)" ]
  lxc_remote stop -f l1:c1 l2:c2

  # Check if the file got refreshed locally.
  lxc_remote copy l1:c1 l1:c2 --refresh
  lxc_remote start l1:c1 l1:c2
  c1_pid="$(lxc_remote list -f csv -c p l1:c1)"
  c2_pid="$(lxc_remote list -f csv -c p l1:c2)"
  [ "$(stat "/proc/${c1_pid}/root/root/testfile1" -c %y)" = "$(stat "/proc/${c2_pid}/root/root/testfile1" -c %y)" ]
  lxc_remote rm -f l1:c2
  lxc_remote stop -f l1:c1

  # This will create snapshot c1/snap0 with test device and expiry date.
  lxc_remote config device add l1:c1 testsnapdev none
  lxc_remote config set l1:c1 snapshots.expiry '1d'
  lxc_remote snapshot l1:c1
  lxc_remote config device remove l1:c1 testsnapdev
  lxc_remote config device add l1:c1 testdev none

  # Remove the testfile from c1 and refresh again
  lxc_remote file delete l1:c1/root/testfile1
  lxc_remote copy l1:c1 l2:c2 --refresh --instance-only
  lxc_remote start l2:c2
  ! lxc_remote file pull l2:c2/root/testfile1 . || false
  lxc_remote stop -f l2:c2

  # Check whether snapshot c2/snap0 has been created with its config intact.
  ! lxc_remote config show l2:c2/snap0 || false
  lxc_remote copy l1:c1 l2:c2 --refresh
  lxc_remote ls l2:
  lxc_remote config show l2:c2/snap0
  ! lxc_remote config show l2:c2/snap0 | grep -F 'expires_at: 0001-01-01T00:00:00Z' || false
  [ "$(lxc_remote config device get l2:c2 testdev type)" = "none" ]
  lxc_remote restore l2:c2 snap0
  [ "$(lxc_remote config device get l2:c2 testsnapdev type)" = "none" ]

  # This will create snapshot c2/snap1
  lxc_remote snapshot l2:c2
  lxc_remote config show l2:c2/snap1

  # This should remove c2/snap1
  lxc_remote copy l1:c1 l2:c2 --refresh
  ! lxc_remote config show l2:c2/snap1 || false

  lxc_remote rm -f l1:c1 l2:c2

  remote_pool1="lxdtest-$(basename "${LXD_DIR}")"
  remote_pool2="lxdtest-$(basename "${lxd2_dir}")"

  lxc_remote storage volume create l1:"$remote_pool1" vol1 size=4MiB
  lxc_remote storage volume set l1:"$remote_pool1" vol1 user.foo=snap0vol1
  lxc_remote storage volume snapshot l1:"$remote_pool1" vol1
  lxc_remote storage volume set l1:"$remote_pool1" vol1 user.foo=snap1vol1
  lxc_remote storage volume snapshot l1:"$remote_pool1" vol1
  lxc_remote storage volume set l1:"$remote_pool1" vol1 user.foo=postsnap1vol1

  # remote storage volume and snapshots migration in "pull" mode
  lxc_remote storage volume copy l1:"$remote_pool1/vol1" l2:"$remote_pool2/vol2"
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2 user.foo)" = "postsnap1vol1" ]
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2/snap0 user.foo)" = "snap0vol1" ]
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2/snap1 user.foo)" = "snap1vol1" ]

  # check copied volume and snapshots have different UUIDs
  [ "$(lxc_remote storage volume get l1:"$remote_pool1" vol1 volatile.uuid)" != "$(lxc_remote storage volume get l2:"$remote_pool2" vol2 volatile.uuid)" ]
  [ "$(lxc_remote storage volume get l1:"$remote_pool1" vol1/snap0 volatile.uuid)" != "$(lxc_remote storage volume get l2:"$remote_pool2" vol2/snap0 volatile.uuid)" ]
  [ "$(lxc_remote storage volume get l1:"$remote_pool1" vol1/snap1 volatile.uuid)" != "$(lxc_remote storage volume get l2:"$remote_pool2" vol2/snap1 volatile.uuid)" ]

  lxc_remote storage volume delete l2:"$remote_pool2" vol2

  # check moving volume and snapshots.
  lxc_remote storage volume copy l1:"$remote_pool1/vol1" l1:"$remote_pool1/vol2"
  lxc_remote storage volume move l1:"$remote_pool1/vol2" l2:"$remote_pool2/vol3"
  ! lxc_remote storage volume show l1:"$remote_pool1" vol2 || false
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol3 user.foo)" = "postsnap1vol1" ]
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol3/snap0 user.foo)" = "snap0vol1" ]
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol3/snap1 user.foo)" = "snap1vol1" ]
  lxc_remote storage volume delete l2:"$remote_pool2" vol3

  lxc_remote storage volume copy l1:"$remote_pool1/vol1" l2:"$remote_pool2/vol2" --volume-only
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2 user.foo)" = "postsnap1vol1" ]
  ! lxc_remote storage volume show l2:"$remote_pool2" vol2/snap0 || false
  ! lxc_remote storage volume show l2:"$remote_pool2" vol2/snap1 || false
  lxc_remote storage volume delete l2:"$remote_pool2" vol2

  # remote storage volume and snapshots migration refresh in "pull" mode
  lxc_remote storage volume set l1:"$remote_pool1" vol1 user.foo=snapremovevol1
  lxc_remote storage volume snapshot l1:"$remote_pool1" vol1 snapremove
  lxc_remote storage volume set l1:"$remote_pool1" vol1 user.foo=postsnap1vol1
  lxc_remote storage volume copy l1:"$remote_pool1/vol1" l2:"$remote_pool2/vol2" --refresh
  lxc_remote storage volume delete l1:"$remote_pool1" vol1

  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2 user.foo)" = "postsnap1vol1" ]
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2/snap0 user.foo)" = "snap0vol1" ]
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2/snap1 user.foo)" = "snap1vol1" ]
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2/snapremove user.foo)" = "snapremovevol1" ]

  # check remote storage volume refresh from a different volume
  lxc_remote storage volume create l1:"$remote_pool1" vol3 size=4MiB
  lxc_remote storage volume set l1:"$remote_pool1" vol3 user.foo=snap0vol3
  lxc_remote storage volume snapshot l1:"$remote_pool1" vol3
  lxc_remote storage volume set l1:"$remote_pool1" vol3 user.foo=snap1vol3
  lxc_remote storage volume snapshot l1:"$remote_pool1" vol3
  lxc_remote storage volume set l1:"$remote_pool1" vol3 user.foo=snap2vol3
  lxc_remote storage volume snapshot l1:"$remote_pool1" vol3
  lxc_remote storage volume set l1:"$remote_pool1" vol3 user.foo=postsnap1vol3

  # check snapshot volumes and snapshots are refreshed
  lxc_remote storage volume copy l1:"$remote_pool1/vol3" l2:"$remote_pool2/vol2" --refresh
  lxc_remote storage volume ls l2:"$remote_pool2"
  lxc_remote storage volume delete l1:"$remote_pool1" vol3
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2 user.foo)" = "postsnap1vol1" ] # FIXME Should be postsnap1vol3
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2/snap0 user.foo)" = "snap0vol3" ]
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2/snap1 user.foo)" = "snap1vol3" ]
  [ "$(lxc_remote storage volume get l2:"$remote_pool2" vol2/snap2 user.foo)" = "snap2vol3" ]
  ! lxc_remote storage volume show l2:"$remote_pool2" vol2/snapremove || false
  lxc_remote storage volume delete l2:"$remote_pool2" vol2

  # check that a refresh doesn't change the volume's and snapshot's UUID.
  lxc_remote storage volume create l1:"$remote_pool1" vol1 size=4MiB
  lxc_remote storage volume snapshot l1:"$remote_pool1" vol1
  lxc_remote storage volume copy l1:"$remote_pool1"/vol1 l2:"$remote_pool2"/vol2
  old_uuid="$(lxc storage volume get l2:"$remote_pool2" vol2 volatile.uuid)"
  old_snap0_uuid="$(lxc storage volume get l2:"$remote_pool2" vol2/snap0 volatile.uuid)"
  lxc_remote storage volume copy l1:"$remote_pool1/vol1" l2:"$remote_pool2/vol2" --refresh
  [ "$(lxc storage volume get l2:"$remote_pool2" vol2 volatile.uuid)" = "${old_uuid}" ]
  [ "$(lxc storage volume get l2:"$remote_pool2" vol2/snap0 volatile.uuid)" = "${old_snap0_uuid}" ]
  lxc_remote storage volume delete l2:"$remote_pool2" vol2
  lxc_remote storage volume delete l1:"$remote_pool1" vol1

  # remote storage volume migration in "push" mode
  lxc_remote storage volume create l1:"$remote_pool1" vol1 size=4MiB
  lxc_remote storage volume create l1:"$remote_pool1" vol2 size=4MiB
  lxc_remote storage volume snapshot l1:"$remote_pool1" vol2

  lxc_remote storage volume copy l1:"$remote_pool1/vol1" l2:"$remote_pool2/vol2" --mode=push
  lxc_remote storage volume move l1:"$remote_pool1/vol1" l2:"$remote_pool2/vol3" --mode=push
  ! lxc_remote storage volume list l1:"$remote_pool1/vol1" || false
  lxc_remote storage volume copy l1:"$remote_pool1/vol2" l2:"$remote_pool2/vol4" --volume-only --mode=push
  lxc_remote storage volume copy l1:"$remote_pool1/vol2" l2:"$remote_pool2/vol5" --mode=push
  lxc_remote storage volume move l1:"$remote_pool1/vol2" l2:"$remote_pool2/vol6" --mode=push

  lxc_remote storage volume delete l2:"$remote_pool2" vol2
  lxc_remote storage volume delete l2:"$remote_pool2" vol3
  lxc_remote storage volume delete l2:"$remote_pool2" vol4
  lxc_remote storage volume delete l2:"$remote_pool2" vol5
  lxc_remote storage volume delete l2:"$remote_pool2" vol6

  # remote storage volume migration in "relay" mode
  lxc_remote storage volume create l1:"$remote_pool1" vol1 size=4MiB
  lxc_remote storage volume create l1:"$remote_pool1" vol2 size=4MiB
  lxc_remote storage volume snapshot l1:"$remote_pool1" vol2

  lxc_remote storage volume copy l1:"$remote_pool1/vol1" l2:"$remote_pool2/vol2" --mode=relay
  lxc_remote storage volume move l1:"$remote_pool1/vol1" l2:"$remote_pool2/vol3" --mode=relay
  ! lxc_remote storage volume list l1:"$remote_pool1/vol1" || false
  lxc_remote storage volume copy l1:"$remote_pool1/vol2" l2:"$remote_pool2/vol4" --volume-only --mode=relay
  lxc_remote storage volume copy l1:"$remote_pool1/vol2" l2:"$remote_pool2/vol5" --mode=relay
  lxc_remote storage volume move l1:"$remote_pool1/vol2" l2:"$remote_pool2/vol6" --mode=relay

  lxc_remote storage volume delete l2:"$remote_pool2" vol2
  lxc_remote storage volume delete l2:"$remote_pool2" vol3
  lxc_remote storage volume delete l2:"$remote_pool2" vol4
  lxc_remote storage volume delete l2:"$remote_pool2" vol5
  lxc_remote storage volume delete l2:"$remote_pool2" vol6

  # Test migration when rsync compression is disabled
  lxc_remote storage set l1:"$remote_pool1" rsync.compression false
  lxc_remote storage volume create l1:"$remote_pool1" foo size=4MiB
  lxc_remote storage volume copy l1:"$remote_pool1"/foo l2:"$remote_pool2"/bar
  lxc_remote storage volume delete l1:"$remote_pool1" foo
  lxc_remote storage volume delete l2:"$remote_pool2" bar
  lxc_remote storage unset l1:"$remote_pool1" rsync.compression

  echo "==> Test container migration with attached local volumes."

  echo "==> Create a local storage with the same name."
  lxc_remote storage create l1:dir dir
  lxc_remote storage create l2:dir dir

  echo "==> Create a volume to attach to container."
  lxc_remote storage volume create l1:dir vol1 size=4MiB

  echo "==> Create a container to test migration with attached local volume."
  lxc_remote init --empty l1:c1
  lxc_remote storage volume attach l1:dir vol1 c1 /files

  echo "==> Check that copying a container with attached local volume fails, if the destination does not have a volume with the same name."
  ! lxc_remote copy l1:c1 l2: || false

  echo "==> Check that moving a container with attached local volume fails, if the destination does not have a volume with the same name."
  ! lxc_remote move l1:c1 l2: || false

  echo "==> Copy the volume."
  lxc_remote storage volume copy l1:dir/vol1 l2:dir/vol1

  echo "==> Check that copying a container with attached local volume succeeds, if the destination has a volume with the same name."
  lxc_remote copy l1:c1 l2:

  echo "==> Check that moving a container with attached local volume succeeds, if the destination has a volume with the same name."
  lxc_remote move l2:c1 l1:c2

  echo "==> Clean up the containers and local volumes."
  lxc_remote delete -f l1:c1
  lxc_remote delete -f l1:c2
  lxc_remote storage volume delete l1:dir vol1
  lxc_remote storage volume delete l2:dir vol1

  # Test VM Migration.
  if [ "${LXD_VM_TESTS:-0}" = "0" ]; then
    echo "==> SKIP: VM tests are disabled"
  elif [ "${LXD_TMPFS:-0}" = "1" ] && ! runsMinimumKernel 6.6; then
    echo "==> SKIP: QEMU requires direct-io support which requires a kernel >= 6.6 for tmpfs support (LXD_TMPFS=${LXD_TMPFS})"
  else
    echo "==> Test VM migration with attached local volumes."

    echo "==> Create a volume to attach to VM."
    lxc_remote storage volume create l1:dir vol1 size=4MiB

    echo "==> Create a VM to test migration with attached local volume."
    lxc_remote init --vm --empty l1:v1 -c limits.memory=128MiB -d "${SMALL_ROOT_DISK}"
    lxc_remote storage volume attach l1:dir vol1 v1 /files

    echo "==> Check that copying a VM with attached local volume fails, if the destination does not have a volume with the same name."
    ! lxc_remote copy l1:v1 l2: || false

    echo "==> Check that moving a VM with attached local volume fails, if the destination does not have a volume with the same name."
    ! lxc_remote move l1:v1 l2: || false

    echo "==> Copy the volume."
    lxc_remote storage volume copy l1:dir/vol1 l2:dir/vol1

    echo "==> Check that copying a VM with attached local volume succeeds, if the destination has a volume with the same name."
    lxc_remote copy l1:v1 l2:

    echo "==> Check that moving a VM with attached local volume succeeds, if the destination has a volume with the same name."
    lxc_remote move l2:v1 l1:v2

    echo "==> Clean up the VMs and local volumes."
    lxc_remote delete -f l1:v1
    lxc_remote delete -f l1:v2
    lxc_remote storage volume delete l1:dir vol1
    lxc_remote storage volume delete l2:dir vol1
  fi

  echo "==> Clean up the storage pool."
  lxc_remote storage delete l1:dir
  lxc_remote storage delete l2:dir

  # Test some migration between projects
  lxc_remote project create l1:proj -c features.images=false -c features.profiles=false
  lxc_remote project switch l1:proj

  lxc_remote init testimage l1:c1
  lxc_remote copy l1:c1 l2:
  lxc_remote start l2:c1
  lxc_remote delete l2:c1 -f

  lxc_remote snapshot l1:c1
  lxc_remote snapshot l1:c1
  lxc_remote snapshot l1:c1
  lxc_remote copy l1:c1 l2:
  lxc_remote start l2:c1
  lxc_remote stop l2:c1 -f
  lxc_remote delete l1:c1

  lxc_remote copy l2:c1 l1:
  lxc_remote start l1:c1
  lxc_remote delete l1:c1 -f

  lxc_remote delete l2:c1/snap0
  lxc_remote delete l2:c1/snap1
  lxc_remote delete l2:c1/snap2
  lxc_remote copy l2:c1 l1:
  lxc_remote start l1:c1
  lxc_remote delete l1:c1 -f
  lxc_remote delete l2:c1 -f

  lxc_remote project switch l1:default
  lxc_remote project delete l1:proj

  # Check snapshot creation dates after migration.
  lxc_remote init testimage l1:c1
  lxc_remote snapshot l1:c1
  ! lxc_remote storage volume show "l1:${remote_pool1}" container/c1 | grep '^created_at: 0001-01-01T00:00:00Z' || false
  ! lxc_remote storage volume show "l1:${remote_pool1}" container/c1/snap0 | grep '^created_at: 0001-01-01T00:00:00Z' || false
  lxc_remote copy l1:c1 l2:c1
  ! lxc_remote storage volume show "l2:${remote_pool2}" container/c1 | grep '^created_at: 0001-01-01T00:00:00Z' || false
  [ "$(lxc_remote storage volume get --property "l1:${remote_pool1}" container/c1/snap0 created_at)" = "$(lxc_remote storage volume get --property "l2:${remote_pool2}" container/c1/snap0 created_at)" ]
  lxc_remote delete l1:c1 -f
  lxc_remote delete l2:c1 -f

  # Check migration with invalid snapshot config (disks attached with missing source pool and source path).
  lxc_remote init testimage l1:c1
  lxc_remote storage create l1:dir dir
  lxc_remote storage volume create l1:dir vol1 size=4MiB
  lxc_remote storage volume attach l1:dir vol1 c1 /mnt
  mkdir "$LXD_DIR/testvol2"
  lxc_remote config device add l1:c1 vol2 disk source="$LXD_DIR/testvol2" path=/vol2
  lxc_remote snapshot l1:c1 # Take snapshot with disk devices still attached.
  lxc_remote config device remove c1 vol1
  lxc_remote config device remove c1 vol2
  rmdir "$LXD_DIR/testvol2"
  lxc_remote copy l1:c1 l2:
  lxc_remote info l2:c1 | grep snap0
  lxc_remote delete l1:c1 -f
  lxc_remote delete l2:c1 -f
  lxc_remote storage volume delete l1:dir vol1
  lxc_remote storage delete l1:dir

  # Test optimized refresh
  lxc_remote init testimage l1:c1
  echo test | lxc_remote file push - l1:c1/tmp/foo
  lxc_remote copy l1:c1 l2:c1
  lxc_remote file pull l2:c1/tmp/foo .
  lxc_remote snapshot l1:c1
  echo test | lxc_remote file push - l1:c1/tmp/bar
  lxc_remote copy l1:c1 l2:c1 --refresh
  lxc_remote start l2:c1
  lxc_remote file pull l2:c1/tmp/foo .
  lxc_remote file pull l2:c1/tmp/bar .
  lxc_remote stop l2:c1 -f

  lxc_remote restore l2:c1 snap0
  lxc_remote start l2:c1
  lxc_remote file pull l2:c1/tmp/foo .
  ! lxc_remote file pull l2:c1/tmp/bar . || false
  lxc_remote stop l2:c1 -f

  rm foo bar

  lxc_remote rm l1:c1
  lxc_remote rm l2:c1

  lxc_remote init testimage l1:c1
  # This creates snap0
  lxc_remote snapshot l1:c1
  # This creates snap1
  lxc_remote snapshot l1:c1
  lxc_remote copy l1:c1 l2:c1
  # This creates snap2
  lxc_remote snapshot l1:c1

  # Delete first snapshot from target
  lxc_remote rm l2:c1/snap0

  # Refresh
  lxc_remote copy l1:c1 l2:c1 --refresh

  lxc_remote rm -f l1:c1
  lxc_remote rm -f l2:c1

  # In this scenario the source LXD server used to crash due to a missing slice check.
  # Let's test this to make sure it doesn't happen again.
  lxc_remote init testimage l1:c1
  lxc_remote copy l1:c1 l2:c1
  lxc_remote snapshot l1:c1
  lxc_remote snapshot l1:c1

  lxc_remote copy l1:c1 l2:c1 --refresh
  lxc_remote copy l1:c1 l2:c1 --refresh

  lxc_remote rm -f l1:c1
  lxc_remote rm -f l2:c1

  # On btrfs, this used to cause a failure because btrfs couldn't find the parent subvolume.
  lxc_remote init testimage l1:c1
  lxc_remote copy l1:c1 l2:c1
  lxc_remote snapshot l1:c1
  lxc_remote copy l1:c1 l2:c1 --refresh
  lxc_remote snapshot l1:c1
  lxc_remote copy l1:c1 l2:c1 --refresh

  lxc_remote rm -f l1:c1
  lxc_remote rm -f l2:c1

  # On zfs, this used to crash due to a websocket read issue.
  lxc launch testimage c1
  lxc snapshot c1
  lxc copy c1 l2:c1 --stateless
  lxc copy c1 l2:c1 --stateless --refresh

  lxc_remote rm -f l1:c1
  lxc_remote rm -f l2:c1

  # migrate ISO custom volumes
  truncate -s 8MiB foo.iso
  lxc storage volume import l1:"${pool}" ./foo.iso iso1
  lxc storage volume copy l1:"${pool}"/iso1 l2:"${remote_pool}"/iso1

  lxc storage volume show l2:"${remote_pool}" iso1 | grep -xF 'content_type: iso'
  lxc storage volume move l1:"${pool}"/iso1 l2:"${remote_pool}"/iso2
  lxc storage volume show l2:"${remote_pool}" iso2 | grep -xF 'content_type: iso'
  ! lxc storage volume show l1:"${pool}" iso1 || false

  lxc storage volume delete l2:"${remote_pool}" iso1
  lxc storage volume delete l2:"${remote_pool}" iso2
  rm -f foo.iso

  echo "==> Test container live migration (not supported)"
  lxc_remote launch testimage l1:migratee -c raw.lxc=lxc.console.path=none

  # Stateful stop is not supported for containers.
  ! lxc_remote stop --stateful l1:migratee || false

  # Stateful snapshots are not supported for containers.
  ! lxc_remote snapshot --stateful l1:migratee || false

  # Take stateless snapshot.
  lxc_remote snapshot l1:migratee

  # Check container isn't frozen.
  LXC_LOCAL='' lxc_remote exec l1:migratee -- ls

  # Live migration is not supported for containers.
  ! lxc_remote move l1:migratee l2:migratee || false

  # Test stateless move of running container with snapshot.
  lxc_remote move --stateless l1:migratee l2:migratee

  # Test stateless copies
  lxc_remote copy --stateless l2:migratee/snap0 l1:migratee-new-name

  # Cleanup
  lxc_remote delete --force l2:migratee
  lxc_remote delete --force l1:migratee-new-name
}
