diff --git a/src/object/srv_obj_migrate.c b/src/object/srv_obj_migrate.c index 63015b261af..be1494c6f73 100644 --- a/src/object/srv_obj_migrate.c +++ b/src/object/srv_obj_migrate.c @@ -1434,15 +1434,21 @@ migrate_fetch_update_single(struct migrate_one *mrone, daos_handle_t oh, if (mrone->mo_iods[i].iod_size == 0) { static __thread int log_nr; - /* zero size iod will cause assertion failure - * in VOS, so let's check here. - * So the object is being destroyed between - * object enumeration and object fetch on - * the remote target, which is usually caused - * by container destroy or snapshot deletion. - * Since this is rare, let's simply return - * failure for this rebuild, then reschedule - * the rebuild and retry. + /* Do not pass a zero-sized IOD to VOS: it would trigger an assertion. + * + * Enumeration found a record, but the fetch returned iod_size == 0. + * Possible causes include: + * 1. A placement mismatch routes the fetch to a target without the record. + * 2. The fetch epoch precedes the epoch at which the record is visible. + * 3. An in-flight update was not resolved by DTX resync. For example, + * enumeration may see a committable DTX on a parity shard while the + * corresponding DTX on a data shard is still prepared and its record + * is not visible to the migration fetch. + * + * No safe recovery is implemented here. Return -DER_DATA_LOSS to stop + * this migration attempt; migrate_one_ult() does not propagate this + * error to the target's migration status, so it does not fail the + * overall rebuild. */ rc = -DER_DATA_LOSS; DL_INFO(rc, @@ -1640,15 +1646,21 @@ __migrate_fetch_update_bulk(struct migrate_one *mrone, daos_handle_t oh, if (iods[i].iod_size == 0) { static __thread int log_nr; - /* zero size iod will cause assertion failure - * in VOS, so let's check here. - * So the object is being destroyed between - * object enumeration and object fetch on - * the remote target, which is usually caused - * by container destroy or snapshot deletion. - * Since this is rare, let's simply return - * failure for this rebuild, then reschedule - * the rebuild and retry. + /* Do not pass a zero-sized IOD to VOS: it would trigger an assertion. + * + * Enumeration found a record, but the fetch returned iod_size == 0. + * Possible causes include: + * 1. A placement mismatch routes the fetch to a target without the record. + * 2. The fetch epoch precedes the epoch at which the record is visible. + * 3. An in-flight update was not resolved by DTX resync. For example, + * enumeration may see a committable DTX on a parity shard while the + * corresponding DTX on a data shard is still prepared and its record + * is not visible to the migration fetch. + * + * No safe recovery is implemented here. Return -DER_DATA_LOSS to stop + * this migration attempt; migrate_one_ult() does not propagate this + * error to the target's migration status, so it does not fail the + * overall rebuild. */ rc = -DER_DATA_LOSS; DL_INFO(rc, @@ -1658,7 +1670,7 @@ __migrate_fetch_update_bulk(struct migrate_one *mrone, daos_handle_t oh, DP_UOID(mrone->mo_oid), DP_KEY(&mrone->mo_dkey), DP_KEY(&iods[i].iod_name), iod_num, i, mrone->mo_epoch, fetch_eph); if (log_nr <= 128) { - mrone_dump_info(mrone, oh, &mrone->mo_iods[i]); + mrone_dump_info(mrone, oh, &iods[i]); log_nr++; } D_GOTO(end, rc); diff --git a/src/placement/jump_map.c b/src/placement/jump_map.c index f6fabd748c1..eba5c5c6d46 100644 --- a/src/placement/jump_map.c +++ b/src/placement/jump_map.c @@ -395,7 +395,8 @@ static int obj_remap_shards(struct pl_jump_map *jmap, uint32_t layout_ver, struct daos_obj_md *md, struct pl_obj_layout *layout, struct jm_obj_placement *jmop, d_list_t *remap_list, uint32_t allow_version, enum layout_gen_mode gen_mode, uint8_t *tgts_used, - uint8_t *dom_used, uint8_t *dom_full, uint32_t failed_in_layout, uint32_t fdom_lvl) + uint8_t *dom_used, uint8_t *dom_full, uint32_t failed_in_layout, uint32_t fdom_lvl, + bool *mode_dependent) { struct failed_shard *f_shard; struct pool_target *spare_tgt = NULL; @@ -452,7 +453,7 @@ obj_remap_shards(struct pl_jump_map *jmap, uint32_t layout_ver, struct daos_obj_ get_target(root, curr_pd, layout_ver, &spare_tgt, &spare_dom, rebuild_key, dom_used, dom_full, dgu->dgu_used, dgu->dgu_real, tgts_used, shard_id, allow_version, gen_mode, fdom_lvl, jmop->jmop_grp_size, - &spares_left, &spare_avail); + &spares_left, &spare_avail, mode_dependent); if (layout_ver > 0) { /* * After 2.4 (layout_ver > 0), it will always assign each shard @@ -468,7 +469,7 @@ obj_remap_shards(struct pl_jump_map *jmap, uint32_t layout_ver, struct daos_obj_ } rc = determine_valid_spares(spare_tgt, md, spare_avail, remap_list, allow_version, - gen_mode, f_shard, layout); + gen_mode, f_shard, layout, mode_dependent); if (rc == 1) { d_list_del(&f_shard->fs_list); D_FREE(f_shard); @@ -572,7 +573,7 @@ remap_gpu_alloc_one(d_list_t *remap_list, uint8_t *dom_cur_grp_used, static int get_object_layout(struct pl_jump_map *jmap, uint32_t layout_ver, struct pl_obj_layout *layout, struct jm_obj_placement *jmop, uint32_t allow_version, - enum layout_gen_mode gen_mode, struct daos_obj_md *md) + enum layout_gen_mode gen_mode, struct daos_obj_md *md, bool *mode_dependent) { struct pool_target *target; struct pool_domain *domain; @@ -690,7 +691,7 @@ get_object_layout(struct pl_jump_map *jmap, uint32_t layout_ver, struct pl_obj_l get_target(root, curr_pd, layout_ver, &target, &domain, key, dom_used, dom_full, dom_cur_grp_used, dom_cur_grp_real, tgts_used, k, allow_version, gen_mode, fdom_lvl, - jmop->jmop_grp_size, NULL, NULL); + jmop->jmop_grp_size, NULL, NULL, mode_dependent); } if (target == NULL) { @@ -708,8 +709,8 @@ get_object_layout(struct pl_jump_map *jmap, uint32_t layout_ver, struct pl_obj_l layout->ol_shards[k].po_index = target->ta_comp.co_index; /** If target is failed queue it for remap*/ - if (comp_need_remap(&target->ta_comp, allow_version, gen_mode, - &remap_flags)) { + if (comp_need_remap(&target->ta_comp, allow_version, gen_mode, &remap_flags, + mode_dependent)) { struct failed_shard *shard; fail_tgt_cnt++; @@ -755,7 +756,7 @@ get_object_layout(struct pl_jump_map *jmap, uint32_t layout_ver, struct pl_obj_l if (fail_tgt_cnt > 0) rc = obj_remap_shards(jmap, layout_ver, md, layout, jmop, &remap_list, allow_version, gen_mode, tgts_used, dom_used, dom_full, - fail_tgt_cnt, fdom_lvl); + fail_tgt_cnt, fdom_lvl, mode_dependent); out: if (rc) D_ERROR("jump_map_obj_layout_fill failed, rc "DF_RC"\n", DP_RC(rc)); @@ -803,7 +804,7 @@ static int obj_layout_alloc_and_get(struct pl_jump_map *jmap, uint32_t layout_ver, struct jm_obj_placement *jmop, struct daos_obj_md *md, uint32_t allow_version, enum layout_gen_mode gen_mode, - struct pl_obj_layout **layout_p) + struct pl_obj_layout **layout_p, bool *mode_dependent) { int rc; @@ -822,7 +823,8 @@ obj_layout_alloc_and_get(struct pl_jump_map *jmap, uint32_t layout_ver, if (gen_mode == CURRENT || !(md->omd_flags & PL_FL_GRP_SPEC)) md->omd_grp_spec = PL_GRP_MAX; - rc = get_object_layout(jmap, layout_ver, *layout_p, jmop, allow_version, gen_mode, md); + rc = get_object_layout(jmap, layout_ver, *layout_p, jmop, allow_version, gen_mode, md, + mode_dependent); if (rc) { D_ERROR("get object layout failed, rc "DF_RC"\n", DP_RC(rc)); @@ -837,6 +839,211 @@ obj_layout_alloc_and_get(struct pl_jump_map *jmap, uint32_t layout_ver, return rc; } +/** + * Generate a second layout of @md in @gen_mode, for comparison against @layout, the one being + * built. + * + * Only the groups @layout has are worth generating. A caller which asked for a single group + * (PL_FL_GRP_SPEC, e.g. the rebuild scanner and the EC aggregation, which place every object + * they walk over) gets a layout truncated after that group, and comparing it against the + * full object would cost as many groups as the object has instead of as many as the caller + * asked for. The truncation is exact for the group asked for: get_object_layout() falls back + * to the full layout when a shard of that group needs remapping, and the initial pass is the + * same in both cases otherwise. The groups in front of it may have been remapped differently, + * but no caller looks at them, and @layout carries the same inexactness. + * + * obj_layout_alloc_and_get() cannot be used for this, since it forces md->omd_grp_spec to + * PL_GRP_MAX for the CURRENT mode and leaves it that way. That is what the caller of a CURRENT + * layout wants, everything it feeds the layout to afterwards expects the full width, but a + * comparison layout must neither be widened behind the caller's back nor leave md changed. + */ +static int +layout_gen_for_compare(struct pl_jump_map *jmap, uint32_t layout_ver, struct jm_obj_placement *jmop, + struct daos_obj_md *md, enum layout_gen_mode gen_mode, + struct pl_obj_layout *layout, struct pl_obj_layout **layout_p) +{ + uint32_t grp_spec = md->omd_grp_spec; + int rc; + + rc = pl_obj_layout_alloc(jmop->jmop_grp_size, jmop->jmop_grp_nr, layout_p); + if (rc != 0) { + D_ERROR(DF_OID ": pl_obj_layout_alloc failed, " DF_RC "\n", DP_OID(md->omd_id), + DP_RC(rc)); + return rc; + } + + md->omd_grp_spec = + layout->ol_grp_nr < jmop->jmop_grp_nr ? layout->ol_grp_nr - 1 : PL_GRP_MAX; + rc = get_object_layout(jmap, layout_ver, *layout_p, jmop, md->omd_ver, gen_mode, md, NULL); + md->omd_grp_spec = grp_spec; + if (rc != 0) { + D_ERROR(DF_OID ": failed to get the %s layout, " DF_RC "\n", DP_OID(md->omd_id), + gen_mode == PRE_REBUILD ? "pre-rebuild" + : gen_mode == CURRENT ? "current" + : "post-rebuild", + DP_RC(rc)); + pl_obj_layout_free(*layout_p); + *layout_p = NULL; + } + return rc; +} + +/** + * Keep the shards which the ongoing rebuild relocated on their old target, and make the new + * target a write-only peer, the way a drain or a reintegration is handled. + * + * A shard which sits on a spare of an earlier rebuild can be handed a different spare by a new + * failure which has nothing to do with it: the spare targets are allocated sequentially over + * the whole remap list, sharing the used-domain/used-target bookkeeping, and a shard whose + * spare candidate is unavailable is requeued with the fseq of that candidate (see + * determine_valid_spares()). Left alone, the CURRENT layout then points at the new target, + * which has never been written to, while the data is on the old one, which is out of the + * write path from the new pool map version on. Everything written after that is missing from + * the old target, so the migration, which fetches through the PRE_REBUILD layout, and a + * read-only client both read stale or empty data from it, and a read-write client is offered + * the new target, which has nothing yet. + * + * Which shards sit on a spare is told by the PRE_REBUILD layout: it remaps a shard only when + * the target of the shard is out (DOWNOUT) or is being reintegrated after having been out + * (regular UP), which are the only two states a target reaches after a rebuild has moved the + * data of the shard elsewhere. Neither the fseq nor the rebuilding flag of a shard tells it + * apart from a shard the ongoing rebuild is handling: a requeued shard inherits the fseq and + * the flags of the unavailable spare candidate it passed by, so a displaced shard is usually + * flagged rebuilding as well, and so is a shard whose spare did not even change. So the + * comparison with PRE_REBUILD identifies the old data location. + * + * A historical spare is not required: while newly added domains are excluded from initial + * placement, a group wider than the old domain set can reuse a domain. Mode-dependent domain + * reuse can then relocate an initial target too, so the caller compares every mode-dependent + * CURRENT layout. + * + * The old target is still healthy, so there is no reason to stop using it. Put it back in + * place as the readable primary of the shard and append the new target as a rebuilding peer: + * writes go to both, reads keep going to the old one, the migration copies the old one to the + * new one and finds it complete, and the PRE_REBUILD layout is correct as it stands. Nothing + * has to be hidden from anybody and the read-only path needs no second layout. + * + * The peer is what jump_map_obj_extend_layout() would add for the shard anyway when it diffs + * the CURRENT layout against the POST_REBUILD one, since the latter still has the shard on the + * new target. So when that function is going to run, because a drain, a reintegration or an + * extension is in flight as well, only the primary is put back here and the diff supplies the + * peer, which is then the target the migration really goes to. Otherwise the CURRENT and the + * POST_REBUILD layouts agree on the new target and it is appended here directly, without + * generating a third layout. + * + * A shard which passed by a newly failed spare candidate on its way to a spare inherits the + * rebuilding flag of that candidate (see determine_valid_spares()), which hides the shard from + * the readers although the spare still has all its data, even when it is the same spare as + * before. The PRE_REBUILD layout tells that as well, and the flag is dropped along the way. + * + * The new target can already be the primary of another shard of the same group, since that + * is how the shard got displaced in the first place. The extended layout then carries one + * target twice in a group, once readable and once rebuilding, which is a shape a drain or a + * reintegration produces today as well. + */ +static int +layout_keep_relocated_source(struct pl_jump_map *jmap, uint32_t layout_ver, + struct jm_obj_placement *jmop, struct daos_obj_md *md, + struct pl_obj_layout *layout, bool will_extend) +{ + struct pl_obj_layout *ref_layout = NULL; + d_list_t peer_list; + int nr; + int i; + int rc; + + rc = layout_gen_for_compare(jmap, layout_ver, jmop, md, PRE_REBUILD, layout, &ref_layout); + if (rc != 0) + return rc; + + D_INIT_LIST_HEAD(&peer_list); + + /* Both layouts cover the same groups, and none of them has been through + * pl_map_extend() yet, so the group sizes and the shard indexes match. + */ + D_ASSERT(ref_layout->ol_grp_size == layout->ol_grp_size); + D_ASSERTF(ref_layout->ol_nr == layout->ol_nr, "%u != %u\n", ref_layout->ol_nr, + layout->ol_nr); + nr = layout->ol_nr; + for (i = 0; i < nr; i++) { + struct pl_obj_shard *cur = &layout->ol_shards[i]; + struct pl_obj_shard *ref = &ref_layout->ol_shards[i]; + struct pool_target *ref_pot; + struct pool_target *cur_pot; + + if (cur->po_target == (uint32_t)-1 || ref->po_target == (uint32_t)-1) + continue; + if (cur->po_target == ref->po_target && cur->po_rebuilding == ref->po_rebuilding) + continue; + + /* + * The shard is only relocated, rather than rebuilt, when the target it was on is + * still serving I/O. Anything else is a failure the ongoing rebuild is handling, + * and the PRE_REBUILD layout has flagged the shard on it as rebuilding already + * (a reintegrating target is flagged as well but not available). + */ + rc = pool_map_find_target(jmap->jmp_map.pl_poolmap, ref->po_target, &ref_pot); + D_ASSERT(rc == 1); + if (ref->po_rebuilding || + !pool_target_avail(ref_pot, PO_COMP_ST_UPIN | PO_COMP_ST_DRAIN)) { + /* + * The data is nowhere readable, and the new target is only being written + * to, whether or not the remap flagged it. The remap does not when the + * shard was handed a different spare without passing by its failed one, + * as the spare it sat on is not known to the remap. + */ + if (cur->po_target != ref->po_target && !cur->po_rebuilding) { + D_DEBUG(DB_PL, + DF_OID " shard %d ver %u rebuilt on target %u, " + "the data on %u is unavailable\n", + DP_OID(md->omd_id), i, md->omd_ver, cur->po_target, + ref->po_target); + cur->po_rebuilding = 1; + } + continue; + } + + if (cur->po_target == ref->po_target) { + /* Same target, the flag alone is stale: see the comment above. */ + *cur = *ref; + continue; + } + + if (!will_extend) { + struct failed_shard *peer; + + rc = pool_map_find_target(jmap->jmp_map.pl_poolmap, cur->po_target, + &cur_pot); + D_ASSERT(rc == 1); + peer = remap_alloc_one(i, cur_pot, cur_pot->ta_comp.co_id, 0, NULL); + if (peer == NULL) + D_GOTO(out, rc = -DER_NOMEM); + d_list_add_tail(&peer->fs_list, &peer_list); + } + + D_DEBUG(DB_PL, + DF_OID + " shard %d ver %u relocated from target %u to %u, keep the old one\n", + DP_OID(md->omd_id), i, md->omd_ver, ref->po_target, cur->po_target); + *cur = *ref; + } + + if (!d_list_empty(&peer_list)) { + rc = pl_map_extend(layout, &peer_list); + if (rc != 0) { + D_ERROR(DF_OID ": extend layout with relocation peers failed, " DF_RC "\n", + DP_OID(md->omd_id), DP_RC(rc)); + D_GOTO(out, rc); + } + } + rc = 0; + +out: + remap_list_free_all(&peer_list); + pl_obj_layout_free(ref_layout); + return rc; +} + /** * Frees the placement map * @@ -965,7 +1172,7 @@ jump_map_obj_extend_layout(struct pl_jump_map *jmap, struct jm_obj_placement *jm D_INIT_LIST_HEAD(&extend_list); rc = obj_layout_alloc_and_get(jmap, layout_version, jmop, md, md->omd_ver, POST_REBUILD, - &new_layout); + &new_layout, NULL); if (rc != 0) { D_ERROR(DF_OID" get_layout_alloc failed, rc "DF_RC"\n", DP_OID(md->omd_id), DP_RC(rc)); @@ -1022,6 +1229,7 @@ jump_map_obj_place(struct pl_map *map, uint32_t layout_version, struct daos_obj_ daos_obj_id_t oid; struct pool_domain *root; enum layout_gen_mode gen_mode = CURRENT; + bool mode_dependent = false; int rc; jmap = pl_map2jmap(map); @@ -1046,12 +1254,11 @@ jump_map_obj_place(struct pl_map *map, uint32_t layout_version, struct daos_obj_ gen_mode = PRE_REBUILD; rc = obj_layout_alloc_and_get(jmap, layout_version, &jmop, md, md->omd_ver, gen_mode, - &layout); + &layout, &mode_dependent); if (rc != 0) { D_ERROR("get_layout_alloc failed, rc "DF_RC"\n", DP_RC(rc)); D_GOTO(out, rc); } - obj_layout_dump(oid, layout); rc = pool_map_find_domain(jmap->jmp_map.pl_poolmap, PO_COMP_TP_ROOT, PO_COMP_ID_ALL, &root); D_ASSERT(rc == 1); @@ -1059,6 +1266,22 @@ jump_map_obj_place(struct pl_map *map, uint32_t layout_version, struct daos_obj_ if (is_pool_map_adding(jmap->jmp_map.pl_poolmap)) layout->ol_shard_peers++; /* may or may not, have to check */ + + /* + * Without a mode-dependent decision, CURRENT and PRE_REBUILD agree. Otherwise compare + * them even when no initial target needs PRE_REBUILD remapping: during extension, a + * group can exceed the number of old domains and reuse a domain in the initial pass. + * A new failure can then change initial targets through reset_dom_cur_grp_v1(), moving + * a healthy read source without any shard having been on a historical spare. + */ + if (gen_mode == CURRENT && mode_dependent) { + rc = layout_keep_relocated_source(jmap, layout_version, &jmop, md, layout, + layout->ol_shard_peers > 0); + if (rc != 0) + D_GOTO(out, rc); + } + + obj_layout_dump(oid, layout); /** * If the layout is being extended or drained, it need recreate the layout * strictly by rebuild version to make sure both new and old shards being @@ -1134,13 +1357,14 @@ jump_map_obj_find_diff(struct pl_map *map, uint32_t layout_ver, struct daos_obj_ } D_INIT_LIST_HEAD(&reint_list); - rc = obj_layout_alloc_and_get(jmap, layout_ver, &jop, md, reint_ver, PRE_REBUILD, &layout); + rc = obj_layout_alloc_and_get(jmap, layout_ver, &jop, md, reint_ver, PRE_REBUILD, &layout, + NULL); if (rc < 0) D_GOTO(out, rc); obj_layout_dump(md->omd_id, layout); rc = obj_layout_alloc_and_get(jmap, layout_ver, &jop, md, reint_ver, POST_REBUILD, - &reint_layout); + &reint_layout, NULL); if (rc < 0) D_GOTO(out, rc); diff --git a/src/placement/jump_map.h b/src/placement/jump_map.h index 330872d842f..03acfbc9ba7 100644 --- a/src/placement/jump_map.h +++ b/src/placement/jump_map.h @@ -1,6 +1,6 @@ /** * (C) Copyright 2022-2023 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -61,7 +61,8 @@ get_target(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t layou struct pool_target **target, struct pool_domain **dom, uint64_t key, uint8_t *dom_used, uint8_t *dom_full, uint8_t *dom_cur_grp_used, uint8_t *dom_cur_grp_real, uint8_t *tgts_used, int shard_num, uint32_t allow_version, enum layout_gen_mode gen_mode, - pool_comp_type_t fdom_lvl, uint32_t grp_size, uint32_t *spare_left, bool *spare_avail); + pool_comp_type_t fdom_lvl, uint32_t grp_size, uint32_t *spare_left, bool *spare_avail, + bool *mode_dependent); static inline uint64_t jm_crc(uint64_t val0, uint64_t val1, uint64_t val2) diff --git a/src/placement/jump_map_versions.c b/src/placement/jump_map_versions.c index dd42512994f..2abdbfd87d5 100644 --- a/src/placement/jump_map_versions.c +++ b/src/placement/jump_map_versions.c @@ -672,7 +672,8 @@ dom_reset_full(struct pool_domain *dom, uint8_t *dom_bits, uint8_t *tgts_used, } static bool -dom_tgts_are_avaible(struct pool_domain *dom, uint32_t allow_version, enum layout_gen_mode gen_mode) +dom_tgts_are_avaible(struct pool_domain *dom, uint32_t allow_version, enum layout_gen_mode gen_mode, + bool *mode_dependent) { int i; @@ -680,7 +681,7 @@ dom_tgts_are_avaible(struct pool_domain *dom, uint32_t allow_version, enum layou struct pool_target *tgt; tgt = &dom->do_targets[i]; - if (!comp_need_remap(&tgt->ta_comp, allow_version, gen_mode, NULL)) + if (!comp_need_remap(&tgt->ta_comp, allow_version, gen_mode, NULL, mode_dependent)) return true; } return false; @@ -691,7 +692,7 @@ static void reset_dom_cur_grp_v1(struct pool_domain *root, struct pool_domain *curr_pd, uint8_t *dom_cur_grp_used, uint8_t *dom_cur_grp_real, uint8_t *dom_full, uint8_t *tgts_used, uint32_t fdom_lvl, uint32_t allow_version, - enum layout_gen_mode gen_mode) + enum layout_gen_mode gen_mode, bool *mode_dependent) { struct pool_domain *tree; uint32_t dom_nr; @@ -779,7 +780,7 @@ reset_dom_cur_grp_v1(struct pool_domain *root, struct pool_domain *curr_pd, struct pool_domain *dom = &root[start_dom + i]; if (!isset(dom_cur_grp_real, start_dom + i) && - dom_tgts_are_avaible(dom, allow_version, gen_mode)) { + dom_tgts_are_avaible(dom, allow_version, gen_mode, mode_dependent)) { dom_reset_full(dom, dom_full, tgts_used, root, allow_version, gen_mode, fdom_lvl); dom_reset_bit(dom, dom_cur_grp_used, root, @@ -816,7 +817,8 @@ get_target_new(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t l struct pool_target **target, struct pool_domain **dom, uint64_t key, uint8_t *dom_used, uint8_t *dom_full, uint8_t *dom_cur_grp_used, uint8_t *dom_cur_grp_real, uint8_t *tgts_used, int shard_num, uint32_t allow_version, - enum layout_gen_mode gen_mode, pool_comp_type_t fdom_lvl, uint32_t grp_size) + enum layout_gen_mode gen_mode, pool_comp_type_t fdom_lvl, uint32_t grp_size, + bool *mode_dependent) { struct pool_target *found = NULL; bool pd_ignored; @@ -845,11 +847,11 @@ get_target_new(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t l if (pd_ignored) reset_dom_cur_grp_v1(root, root, dom_cur_grp_used, dom_cur_grp_real, dom_full, tgts_used, fdom_lvl, allow_version, - gen_mode); + gen_mode, mode_dependent); else - reset_dom_cur_grp_v1(root, curr_pd, dom_cur_grp_used, - dom_cur_grp_real, dom_full, tgts_used, - fdom_lvl, allow_version, gen_mode); + reset_dom_cur_grp_v1( + root, curr_pd, dom_cur_grp_used, dom_cur_grp_real, dom_full, + tgts_used, fdom_lvl, allow_version, gen_mode, mode_dependent); } } *target = found; @@ -1091,7 +1093,8 @@ get_target(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t layou struct pool_target **target, struct pool_domain **dom, uint64_t key, uint8_t *dom_used, uint8_t *dom_full, uint8_t *dom_cur_grp_used, uint8_t *dom_cur_grp_real, uint8_t *tgts_used, int shard_num, uint32_t allow_version, enum layout_gen_mode gen_mode, - pool_comp_type_t fdom_lvl, uint32_t grp_size, uint32_t *spare_left, bool *spare_avail) + pool_comp_type_t fdom_lvl, uint32_t grp_size, uint32_t *spare_left, bool *spare_avail, + bool *mode_dependent) { switch(layout_ver) { case 0: @@ -1124,7 +1127,7 @@ get_target(struct pool_domain *root, struct pool_domain *curr_pd, uint32_t layou */ get_target_new(root, curr_pd, layout_ver, target, dom, key, dom_used, dom_full, dom_cur_grp_used, dom_cur_grp_real, tgts_used, shard_num, - allow_version, gen_mode, fdom_lvl, grp_size); + allow_version, gen_mode, fdom_lvl, grp_size, mode_dependent); if (spare_avail) *spare_avail = true; break; diff --git a/src/placement/pl_map.h b/src/placement/pl_map.h index ef2dd748705..19aa0998b22 100644 --- a/src/placement/pl_map.h +++ b/src/placement/pl_map.h @@ -139,7 +139,8 @@ remap_list_fill(struct pl_map *map, struct daos_obj_md *md, struct daos_obj_shar int determine_valid_spares(struct pool_target *spare_tgt, struct daos_obj_md *md, bool spare_avail, d_list_t *remap_list, uint32_t allow_version, enum layout_gen_mode gen_mode, - struct failed_shard *f_shard, struct pl_obj_layout *layout); + struct failed_shard *f_shard, struct pl_obj_layout *layout, + bool *mode_dependent); int spec_place_rank_get(unsigned int *pos, daos_obj_id_t oid, @@ -150,7 +151,7 @@ pl_map_extend(struct pl_obj_layout *layout, d_list_t *extended_list); bool comp_need_remap(struct pool_component *comp, uint32_t allow_status, enum layout_gen_mode gen_mode, - unsigned int *remap_flags); + unsigned int *remap_flags, bool *mode_dependent); enum { /* rebuilding this shard */ diff --git a/src/placement/pl_map_common.c b/src/placement/pl_map_common.c index 8bd657b0dd5..fadf1b3aaae 100644 --- a/src/placement/pl_map_common.c +++ b/src/placement/pl_map_common.c @@ -257,10 +257,15 @@ remap_list_fill(struct pl_map *map, struct daos_obj_md *md, struct daos_obj_shar * IS_REBUILDING → po_rebuilding=1 skip shard on read * IS_REINTEGRATING → po_reintegrating=1 shard is being reintegrated * HAS_PEER → ol_shard_peers++ triggers extend_layout (dual-write to old+new) + * + * @mode_dependent, when passed, is set as soon as the answer above depends on @gen_mode, i.e. + * for every status but UPIN and DOWNOUT. It lets a caller which would otherwise have to + * generate a second layout in another mode and diff it tell upfront that both modes must + * agree, see layout_keep_relocated_source(). */ bool comp_need_remap(struct pool_component *comp, uint32_t allow_version, enum layout_gen_mode gen_mode, - unsigned int *remap_flags) + unsigned int *remap_flags, bool *mode_dependent) { unsigned int status = comp->co_status; unsigned int flags = 0; @@ -272,6 +277,12 @@ comp_need_remap(struct pool_component *comp, uint32_t allow_version, enum layout if (status == PO_COMP_ST_DOWNOUT) return true; /* always remap */ + /* Everything below is reached only by DOWN, DRAIN and UP, and every one of those + * branches keys off @gen_mode. + */ + if (mode_dependent != NULL) + *mode_dependent = true; + /* NB: if @remap is false, @flags is applied to the current shard/target, otherwise * it's applied to the remapped shard/target. */ @@ -338,7 +349,8 @@ comp_need_remap(struct pool_component *comp, uint32_t allow_version, enum layout int determine_valid_spares(struct pool_target *spare_tgt, struct daos_obj_md *md, bool spare_avail, d_list_t *remap_list, uint32_t allow_version, enum layout_gen_mode gen_mode, - struct failed_shard *f_shard, struct pl_obj_layout *layout) + struct failed_shard *f_shard, struct pl_obj_layout *layout, + bool *mode_dependent) { struct pl_obj_shard *l_shard = &layout->ol_shards[f_shard->fs_shard_idx]; @@ -346,8 +358,8 @@ determine_valid_spares(struct pool_target *spare_tgt, struct daos_obj_md *md, bo goto next_fail; /* The selected spare target is down as well */ - if (comp_need_remap(&spare_tgt->ta_comp, allow_version, gen_mode, - &f_shard->fs_remap_flags)) { + if (comp_need_remap(&spare_tgt->ta_comp, allow_version, gen_mode, &f_shard->fs_remap_flags, + mode_dependent)) { D_DEBUG(DB_PL, "Spare target is also unavailable " DF_TARGET ".\n", DP_TARGET(spare_tgt)); diff --git a/src/placement/ring_map.c b/src/placement/ring_map.c index 0c3a2b16b6f..3e9c76c6a2e 100644 --- a/src/placement/ring_map.c +++ b/src/placement/ring_map.c @@ -1019,7 +1019,7 @@ ring_obj_remap_shards(struct pl_ring_map *rimap, struct daos_obj_md *md, spare_tgt = &tgts[plts[spare_idx].pt_pos]; determine_valid_spares(spare_tgt, md, spare_avail, remap_list, -1, -1, f_shard, - layout); + layout, NULL); } remap_dump(remap_list, md, "after remap:"); diff --git a/src/placement/tests/SConscript b/src/placement/tests/SConscript index b60893f268f..4bc12ba73c4 100644 --- a/src/placement/tests/SConscript +++ b/src/placement/tests/SConscript @@ -26,7 +26,7 @@ def scons(): 'placement_test.c']) pl_bench_tgt = denv.SharedObject(['pl_bench.c', 'place_obj_common.c']) - libraries = ['daos', 'daos_common', 'gurt', 'uuid', 'cmocka', 'isal', 'm'] + libraries = ['daos', 'daos_common', 'gurt', 'uuid', 'cmocka', 'isal', 'm', 'pthread'] ring_pl_test = denv.d_program('ring_pl_map', ring_test_tgt, LIBS=libraries) diff --git a/src/placement/tests/jump_map_place_obj.c b/src/placement/tests/jump_map_place_obj.c index 0734d1972b8..8dd6d4d21eb 100644 --- a/src/placement/tests/jump_map_place_obj.c +++ b/src/placement/tests/jump_map_place_obj.c @@ -570,6 +570,38 @@ jtc_layout_has_duplicate(struct jm_test_ctx *ctx) return result; } +/* + * Like jtc_layout_has_duplicate(), but only readable entries count: an extended layout can + * name the same target as the readable primary of one shard and the write-only peer of + * another one of the same group, which is fine as long as no target is read for two shards. + */ +static bool +jtc_layout_has_readable_duplicate(struct jm_test_ctx *ctx) +{ + const uint32_t total_targets = pool_map_target_nr(ctx->po_map); + bool *target_set; + bool result = false; + int i; + + D_ALLOC_ARRAY(target_set, total_targets); + D_ASSERT(target_set != NULL); + + for (i = 0; i < ctx->layout->ol_nr; i++) { + struct pl_obj_shard *shard = &ctx->layout->ol_shards[i]; + + if (shard->po_target == (uint32_t)-1 || shard->po_rebuilding) + continue; + if (target_set[shard->po_target]) { + print_message("Found readable duplicate target: %d\n", shard->po_target); + result = true; + } + target_set[shard->po_target] = true; + } + D_FREE(target_set); + + return result; +} + static void jtc_enable_debug(struct jm_test_ctx *ctx) { @@ -1672,13 +1704,21 @@ placement_handles_multiple_states_with_addition(void **state) assert_success(jtc_create_layout(&ctx)); - is_false(jtc_layout_has_duplicate(&ctx)); + /* + * The new failure relocates the reintegrating shard off the spare it sits on: the + * spare walk is shared, so failing another target shifts shards that have nothing + * to do with it. That spare stays the readable primary of the shard and, being free + * in the POST_REBUILD layout, becomes the write-only peer the failed shard is + * rebuilt to as well, see layout_keep_relocated_source(). So the same target can + * show up twice, but never readable twice. + */ + is_false(jtc_layout_has_readable_duplicate(&ctx)); jtc_scan(&ctx); rebuilding = jtc_get_layout_rebuild_count(&ctx); - /* 1 each for down, up, new ... maybe? */ - assert_true(rebuilding == 2 || rebuilding == 3 || rebuilding == 4); + /* 1 each for down, up, new ... maybe? Plus the peers of the relocated shards. */ + assert_true(rebuilding >= 2 && rebuilding <= 5); /* Both DOWN and UP target will be remapped during remap */ assert_int_equal(ctx.rebuild.out_nr, 3); @@ -2305,12 +2345,588 @@ fail_multiple_ranks(void **state) jtc_fini(&ctx); } +/** + * Generate the layout of @oid with a specific daos_obj_open() mode. DAOS_OO_RO selects the + * PRE_REBUILD layout, which is what the rebuild migration fetch and every read only client + * use (see jump_map_obj_place()). + */ +static int +jtc_place_mode(struct jm_test_ctx *ctx, daos_obj_id_t oid, unsigned int mode, + struct pl_obj_layout **layout) +{ + struct daos_obj_md md = {0}; + + md.omd_id = oid; + md.omd_ver = pool_map_get_version(ctx->po_map); + *layout = NULL; + + return pl_obj_place(ctx->pl_map, PLT_LAYOUT_VERSION, &md, mode, NULL, layout); +} + +/* place the way the rebuild scanner and the EC aggregation do: read only, up to one group */ +static int +jtc_place_grp(struct jm_test_ctx *ctx, daos_obj_id_t oid, uint32_t grp, + struct pl_obj_layout **layout) +{ + struct daos_obj_md md = {0}; + + md.omd_id = oid; + md.omd_ver = pool_map_get_version(ctx->po_map); + md.omd_flags = PL_FL_GRP_SPEC; + md.omd_grp_spec = grp; + *layout = NULL; + + return pl_obj_place(ctx->pl_map, PLT_LAYOUT_VERSION, &md, DAOS_OO_RO, NULL, layout); +} + +/* + * The layout of a single group, as the rebuild scanner and the EC aggregation ask for it, is + * compared against a reference layout truncated the same way. That group has to come out the + * same as in the full layout, targets and rebuilding flags alike. + */ +static void +jtc_assert_grp_layout(struct jm_test_ctx *ctx, daos_obj_id_t oid, struct pl_obj_layout *full) +{ + uint32_t g; + + for (g = 0; g < full->ol_grp_nr; g++) { + struct pl_obj_layout *part; + uint32_t i; + + assert_success(jtc_place_grp(ctx, oid, g, &part)); + assert_int_equal(part->ol_grp_size, full->ol_grp_size); + assert_int_equal(part->ol_grp_nr, g + 1); + for (i = g * full->ol_grp_size; i < (g + 1) * full->ol_grp_size; i++) { + assert_int_equal(part->ol_shards[i].po_target, + full->ol_shards[i].po_target); + assert_int_equal(part->ol_shards[i].po_rebuilding, + full->ol_shards[i].po_rebuilding); + } + pl_obj_layout_free(part); + } +} + +/* + * A shard which was already remapped by an earlier rebuild can be relocated to a different + * spare when a new and otherwise unrelated target fails: the spares are handed out + * sequentially over a shared used-target bitmap, and a shard whose spare candidate is + * unavailable is requeued with the fseq of that candidate, see determine_valid_spares(). + * + * Left alone, the CURRENT layout points at the new target, which has never been written to, + * while the data is on the old one, which the write path does not use any more from the new + * pool map version on: the migration fetch and every read only client, which both use the + * PRE_REBUILD layout, read stale data from the old target, and a read-write client is offered + * the empty new target, unless the remap happened to flag it along the way. + * + * layout_keep_relocated_source() handles such a shard the way a drain is handled: the old + * target stays the readable primary and the new one is appended as a write-only peer, so + * that both are written to and the migration copies the old one to the new one. + * + * The checks below state what a client can rely on for any layout pair, and are shared by + * the scenarios: + * + * 1. Every readable entry of the PRE_REBUILD layout which sits on a serving target is still + * written to, i.e. is in the write set of the same shard in the CURRENT layout. + * + * 2. Every readable entry of the CURRENT layout holds the data: its target was already in the + * write set of the shard before the new failure, if the shard had a readable copy then. + * + * 3. No target carries two readable entries of one group. The same target can be readable for + * one shard and write-only for another, which a drain or a reintegration produce today. + * + * 4. Every migration destination the rebuild scanner comes up with is in the write set. + * + * 5. The layout of a single group, as the scanner asks for it, matches the full layout. + */ +struct jtc_wb_stats { + uint32_t kept; /* PRE_REBUILD read sources kept in the write set */ + uint32_t relocated; /* shards given a write-only peer on a fresh, healthy spare */ + uint32_t peered; /* CURRENT entries beyond the base group size */ +}; + +static bool +jtc_tgt_is_serving(struct jm_test_ctx *ctx, uint32_t tgt_id) +{ + struct pool_target *tgt; + + assert_int_equal(1, pool_map_find_target(ctx->po_map, tgt_id, &tgt)); + + return pool_target_avail(tgt, PO_COMP_ST_UPIN | PO_COMP_ST_DRAIN); +} + +/* is @tgt in the write set of @shard, i.e. anywhere in the (extended) layout for that shard */ +static bool +jtc_in_write_set(struct pl_obj_layout *layout, uint32_t shard, uint32_t tgt) +{ + int i; + + for (i = 0; i < layout->ol_nr; i++) + if (layout->ol_shards[i].po_shard == shard && layout->ol_shards[i].po_target == tgt) + return true; + + return false; +} + +static void +jtc_assert_write_both(struct jm_test_ctx *ctx, daos_obj_id_t oid, struct pl_obj_layout *before, + struct jtc_wb_stats *st) +{ + struct pl_obj_layout *cur; + struct pl_obj_layout *ro; + struct daos_obj_md md = {0}; + uint32_t tgts[64]; + uint32_t shards[64]; + uint32_t g, i, j; + int nr; + + assert_success(jtc_place_mode(ctx, oid, 0, &cur)); + assert_success(jtc_place_mode(ctx, oid, DAOS_OO_RO, &ro)); + assert_int_equal(cur->ol_grp_nr, ro->ol_grp_nr); + assert_true(cur->ol_grp_size >= ro->ol_grp_size); + + /* 1. read sources of the PRE_REBUILD layout are still written to */ + for (i = 0; i < ro->ol_nr; i++) { + struct pl_obj_shard *s = &ro->ol_shards[i]; + + if (s->po_target == (uint32_t)-1 || s->po_rebuilding || + !jtc_tgt_is_serving(ctx, s->po_target)) + continue; + if (!jtc_in_write_set(cur, s->po_shard, s->po_target)) + fail_msg("shard %u: PRE_REBUILD reads tgt %u, CURRENT does not write it", + s->po_shard, s->po_target); + st->kept++; + } + + /* 2. readable entries of the CURRENT layout hold the data */ + for (i = 0; i < cur->ol_nr; i++) { + struct pl_obj_shard *s = &cur->ol_shards[i]; + bool had_readable = false; + + if (s->po_target == (uint32_t)-1) + continue; + if (i % cur->ol_grp_size >= ro->ol_grp_size) + st->peered++; + if (s->po_rebuilding || before == NULL) + continue; + for (j = 0; j < before->ol_nr; j++) { + struct pl_obj_shard *b = &before->ol_shards[j]; + + if (b->po_shard == s->po_shard && b->po_target != (uint32_t)-1 && + !b->po_rebuilding) + had_readable = true; + } + if (had_readable && !jtc_in_write_set(before, s->po_shard, s->po_target)) + fail_msg("shard %u: CURRENT reads tgt %u, which was never written to", + s->po_shard, s->po_target); + } + + /* 3. one readable entry per target and group */ + for (g = 0; g < cur->ol_grp_nr; g++) { + for (i = g * cur->ol_grp_size; i < (g + 1) * cur->ol_grp_size; i++) { + struct pl_obj_shard *a = &cur->ol_shards[i]; + + if (a->po_target == (uint32_t)-1 || a->po_rebuilding) + continue; + for (j = i + 1; j < (g + 1) * cur->ol_grp_size; j++) { + struct pl_obj_shard *b = &cur->ol_shards[j]; + + if (b->po_target == a->po_target && !b->po_rebuilding) + fail_msg( + "tgt %u readable twice in group %u: shard %u and %u", + a->po_target, g, a->po_shard, b->po_shard); + } + } + } + + /* 4. migration destinations are in the write set */ + md.omd_id = oid; + md.omd_ver = pool_map_get_version(ctx->po_map); + nr = pl_obj_find_rebuild(ctx->pl_map, PLT_LAYOUT_VERSION, &md, NULL, md.omd_ver, tgts, + shards, ARRAY_SIZE(tgts)); + assert_true(nr >= 0); + for (i = 0; i < (uint32_t)nr; i++) + if (!jtc_in_write_set(cur, shards[i], tgts[i])) + fail_msg("shard %u: migrated to tgt %u, which CURRENT does not write", + shards[i], tgts[i]); + + /* 5. the single group layout matches */ + jtc_assert_grp_layout(ctx, oid, ro); + + /* + * A relocated shard is one whose readable primary sits on a spare of an earlier + * rebuild (po_fseq > 1) and which got a write-only peer on a target which has never + * failed and is not being reintegrated or extended. + */ + for (i = 0; i < cur->ol_nr; i++) { + struct pl_obj_shard *s = &cur->ol_shards[i]; + struct pool_target *tgt; + + if (s->po_target == (uint32_t)-1 || !s->po_rebuilding || s->po_reintegrating || + i % cur->ol_grp_size < ro->ol_grp_size) + continue; + assert_int_equal(1, pool_map_find_target(ctx->po_map, s->po_target, &tgt)); + if (tgt->ta_comp.co_status != PO_COMP_ST_UPIN || tgt->ta_comp.co_fseq > 1) + continue; + for (j = 0; j < cur->ol_nr; j++) { + struct pl_obj_shard *p = &cur->ol_shards[j]; + + if (p->po_shard != s->po_shard || p->po_rebuilding || p->po_fseq <= 1 || + p->po_target == (uint32_t)-1) + continue; + assert_int_equal(1, pool_map_find_target(ctx->po_map, p->po_target, &tgt)); + if (tgt->ta_comp.co_status == PO_COMP_ST_UPIN) + st->relocated++; + } + } + + pl_obj_layout_free(cur); + pl_obj_layout_free(ro); +} + +static void +jtc_wb_gen_oid(daos_obj_id_t *oid, uint32_t o, daos_oclass_id_t oc) +{ + gen_oid(oid, o + 1, UINT64_MAX, oc); +} + +/* + * A completed rebuild has left shards on spares, then a new failure hits. Nothing else is in + * flight, so any peer entry in the CURRENT layout is one the relocation handling appended. + */ +static void +_no_stale_read_source(uint32_t domain_nr, uint32_t target_nr, daos_oclass_id_t oc, + uint32_t out_domains, uint32_t down_domains, uint32_t obj_nr) +{ + struct jm_test_ctx ctx; + struct pl_obj_layout **before; + struct jtc_wb_stats st = {0}; + uint32_t down_lo = out_domains * target_nr; + uint32_t down_hi = down_lo + down_domains * target_nr; + uint32_t i, o; + + jtc_init(&ctx, domain_nr, 1, target_nr, oc, g_verbose); + + /* the first failures, whose rebuild has completed */ + for (i = 0; i < down_lo; i++) + jtc_set_status_on_target(&ctx, DOWN, i); + for (i = 0; i < down_lo; i++) + jtc_set_status_on_target(&ctx, DOWNOUT, i); + + /* remember where the data is written before the next, unrelated failure */ + D_ALLOC_ARRAY(before, obj_nr); + assert_non_null(before); + for (o = 0; o < obj_nr; o++) { + daos_obj_id_t oid; + + jtc_wb_gen_oid(&oid, o, oc); + assert_success(jtc_place_mode(&ctx, oid, 0, &before[o])); + assert_int_equal(before[o]->ol_shard_peers, 0); + } + + /* the new failure, its rebuild is the one which is ongoing */ + for (i = down_lo; i < down_hi; i++) + jtc_set_status_on_target(&ctx, DOWN, i); + + for (o = 0; o < obj_nr; o++) { + daos_obj_id_t oid; + + jtc_wb_gen_oid(&oid, o, oc); + jtc_assert_write_both(&ctx, oid, before[o], &st); + } + + /* the scenario has to actually relocate something, otherwise the checks above are + * not testing anything + */ + print_message("%u read sources kept, %u shards relocated, %u peer entries\n", st.kept, + st.relocated, st.peered); + assert_true(st.relocated > 0); + assert_int_equal(st.peered, st.relocated); + + for (o = 0; o < obj_nr; o++) + pl_obj_layout_free(before[o]); + D_FREE(before); + jtc_fini(&ctx); +} + +static void +no_stale_read_source(void **state) +{ + /* 4 ranks x 4 targets, replicated: the pool shape this was first seen on */ + _no_stale_read_source(4, 4, OC_RP_4G2, 1, 1, 500); + /* EC over a pool with less spare room, where the current layout loses the flag too */ + _no_stale_read_source(6, 4, OC_EC_4P2G2, 2, 1, 500); +} + +/* + * NEW domains count towards the object class limit but are not initial placement candidates. + * A five-shard group over four old ranks can therefore reuse a domain during the initial + * pass. After a failure, CURRENT and PRE_REBUILD can choose different initial targets even + * though no target is DOWNOUT or undergoing ordinary reintegration. + */ +static void +extension_preserves_read_sources(void **state) +{ + const struct { + daos_oclass_id_t oc; + uint32_t targets; + } cases[] = {{OC_RP_5G1, 2}, {OC_RP_5G2, 4}, {OC_EC_4P1G1, 2}, {OC_EC_4P1G2, 4}}; + uint32_t c, lv, adding, o, i; + + for (c = 0; c < ARRAY_SIZE(cases); c++) { + for (lv = 1; lv <= 2; lv++) { + for (adding = 0; adding <= 1; adding++) { + struct jm_test_ctx ctx; + struct pl_obj_layout *before[32]; + daos_obj_id_t oids[ARRAY_SIZE(before)]; + struct pl_map_init_attr attr = { + .ia_type = PL_TYPE_JUMP_MAP, + .ia_jump_map.domain = PO_COMP_TP_RANK, + }; + + jtc_init(&ctx, 4, 1, cases[c].targets, cases[c].oc, g_verbose); + assert_success(jtc_pool_map_extend(&ctx, 1, 1, cases[c].targets)); + if (adding) + for (i = 4 * cases[c].targets; i < 5 * cases[c].targets; + i++) + jtc_set_status_on_target(&ctx, POOL_EXTEND, i); + pl_map_decref(ctx.pl_map); + assert_success(pl_map_create(ctx.po_map, &attr, &ctx.pl_map)); + + for (o = 0; o < ARRAY_SIZE(before); o++) { + uint64_t lo = 0x9e3779b97f4a7c15ULL * (o + 1); + struct daos_obj_md md = {0}; + + gen_oid(&oids[o], lo, lo >> 32, cases[c].oc); + oids[o].hi = + (oids[o].hi & ~OID_FMT_TYPE_MASK) | + ((uint64_t)DAOS_OT_AKEY_LEXICAL << OID_FMT_TYPE_SHIFT); + md.omd_id = oids[o]; + md.omd_ver = pool_map_get_version(ctx.po_map); + assert_success(pl_obj_place(ctx.pl_map, lv, &md, DAOS_OO_RW, + NULL, &before[o])); + } + + jtc_set_status_on_target(&ctx, DOWN, 4 * cases[c].targets - 1); + for (o = 0; o < ARRAY_SIZE(before); o++) { + struct daos_obj_md md = {.omd_id = oids[o]}; + struct pl_obj_layout *cur; + struct pl_obj_layout *pre; + + md.omd_ver = pool_map_get_version(ctx.po_map); + assert_success(pl_obj_place(ctx.pl_map, lv, &md, DAOS_OO_RW, + NULL, &cur)); + assert_success(pl_obj_place(ctx.pl_map, lv, &md, DAOS_OO_RO, + NULL, &pre)); + for (i = 0; i < pre->ol_nr; i++) { + struct pl_obj_shard *s = &pre->ol_shards[i]; + + if (s->po_target == (uint32_t)-1 || + s->po_rebuilding || + !jtc_tgt_is_serving(&ctx, s->po_target)) + continue; + assert_true(jtc_in_write_set(before[o], s->po_shard, + s->po_target)); + if (!jtc_in_write_set(cur, s->po_shard, + s->po_target)) + fail_msg( + "class %u layout v%u adding %u " + "oid " DF_OID + ": shard %u PRE_REBUILD reads " + "tgt %u, CURRENT does not write it", + cases[c].oc, lv, adding, + DP_OID(oids[o]), s->po_shard, + s->po_target); + } + for (i = 0; i < cur->ol_nr; i++) { + struct pl_obj_shard *s = &cur->ol_shards[i]; + + if (s->po_target != (uint32_t)-1 && + !s->po_rebuilding) + assert_true(jtc_in_write_set( + before[o], s->po_shard, s->po_target)); + } + pl_obj_layout_free(before[o]); + pl_obj_layout_free(cur); + pl_obj_layout_free(pre); + } + jtc_fini(&ctx); + } + } + } +} + +static void +relocation_peer_alloc_failure(void **state) +{ + struct jm_test_ctx ctx; + struct pl_obj_layout *layout = NULL; + struct d_fault_attr_t attr = { + .fa_interval = UINT32_MAX, + .fa_probability_x = 1, + .fa_probability_y = 1, + .fa_max_faults = 1, + }; + struct d_fault_attr_t *saved_mem; + unsigned int saved_inject; + daos_obj_id_t oid; + uint32_t i; + int rc; + + rc = d_fault_inject_init(); + if (rc == -DER_NOSYS) + skip(); + assert_success(rc); + jtc_init(&ctx, 4, 1, 4, OC_RP_4G2, g_verbose); + for (i = 0; i < 4; i++) + jtc_set_status_on_target(&ctx, DOWN, i); + for (i = 0; i < 4; i++) + jtc_set_status_on_target(&ctx, DOWNOUT, i); + for (i = 4; i < 8; i++) + jtc_set_status_on_target(&ctx, DOWN, i); + + /* With no drain/reintegration, extra entries can only be relocation peers. */ + for (i = 0; i < 500; i++) { + bool extended; + + jtc_wb_gen_oid(&oid, i, OC_RP_4G2); + assert_success(jtc_place_mode(&ctx, oid, 0, &layout)); + extended = layout->ol_grp_size > 4; + pl_obj_layout_free(layout); + layout = NULL; + if (extended) + break; + } + assert_true(i < 500); + + assert_success(D_SPIN_INIT(&attr.fa_lock, PTHREAD_PROCESS_PRIVATE)); + saved_mem = d_fault_attr_mem; + saved_inject = d_fault_inject; + d_fault_attr_mem = &attr; + d_fault_inject = 1; + rc = jtc_place_mode(&ctx, oid, 0, &layout); + d_fault_inject = saved_inject; + d_fault_attr_mem = saved_mem; + assert_success(rc); + pl_obj_layout_free(layout); + layout = NULL; + assert_true(attr.fa_num_hits > 0); + assert_int_equal(attr.fa_num_faults, 0); + + /* + * The final allocation is pl_map_extend()'s peer array. Count it rather than + * depending on the number of allocations needed to generate the two layouts. + */ + attr.fa_interval = attr.fa_num_hits; + attr.fa_num_hits = 0; + d_fault_attr_mem = &attr; + d_fault_inject = 1; + rc = jtc_place_mode(&ctx, oid, 0, &layout); + d_fault_inject = saved_inject; + d_fault_attr_mem = saved_mem; + D_SPIN_DESTROY(&attr.fa_lock); + jtc_fini(&ctx); + assert_success(d_fault_inject_fini()); + + /* Never return a layout with restored primaries but missing write-only peers. */ + assert_int_equal(attr.fa_num_faults, 1); + assert_int_equal(rc, -DER_NOMEM); + assert_null(layout); +} + +/* + * The same, with a drain, a reintegration or an extension in flight as well, so that the + * layouts carry peers of their own already and the relocation peer has to come out of the + * diff against the POST_REBUILD layout, see jump_map_obj_extend_layout(). That diff names the + * target the migration really goes to, which, with a reintegration in flight, is not the spare + * the CURRENT remap came up with. + */ +static void +_no_hidden_peer_source(uint32_t domain_nr, uint32_t target_nr, daos_oclass_id_t oc, + uint32_t out_domains, bool drain, uint32_t obj_nr, struct jtc_wb_stats *st) +{ + struct jm_test_ctx ctx; + struct pl_obj_layout **before; + uint32_t out_hi = out_domains * target_nr; + uint32_t mid_lo = out_hi; + uint32_t mid_hi = mid_lo + target_nr; + uint32_t down_lo = (domain_nr - 1) * target_nr; + uint32_t i, o; + + jtc_init(&ctx, domain_nr, 1, target_nr, oc, g_verbose); + + /* a first failure whose rebuild has completed, so shards end up sitting on spares */ + for (i = 0; i < out_hi; i++) + jtc_set_status_on_target(&ctx, DOWN, i); + for (i = 0; i < out_hi; i++) + jtc_set_status_on_target(&ctx, DOWNOUT, i); + + /* a drain or a reintegration on another domain, both of which add peer targets */ + if (drain) { + for (i = mid_lo; i < mid_hi; i++) + jtc_set_status_on_target(&ctx, DRAIN, i); + } else { + for (i = mid_lo; i < mid_hi; i++) + jtc_set_status_on_target(&ctx, DOWN, i); + for (i = mid_lo; i < mid_hi; i++) + jtc_set_status_on_target(&ctx, DOWNOUT, i); + for (i = mid_lo; i < mid_hi; i++) + jtc_set_status_on_target(&ctx, UP, i); + } + + D_ALLOC_ARRAY(before, obj_nr); + assert_non_null(before); + for (o = 0; o < obj_nr; o++) { + daos_obj_id_t oid; + + jtc_wb_gen_oid(&oid, o, oc); + assert_success(jtc_place_mode(&ctx, oid, 0, &before[o])); + } + + /* and a failure on top of it, so that the two generation modes really do differ */ + for (i = down_lo; i < domain_nr * target_nr; i++) + jtc_set_status_on_target(&ctx, DOWN, i); + + for (o = 0; o < obj_nr; o++) { + daos_obj_id_t oid; + + jtc_wb_gen_oid(&oid, o, oc); + jtc_assert_write_both(&ctx, oid, before[o], st); + } + + print_message("%u read sources kept, %u shards relocated, %u peer entries\n", st->kept, + st->relocated, st->peered); + + for (o = 0; o < obj_nr; o++) + pl_obj_layout_free(before[o]); + D_FREE(before); + jtc_fini(&ctx); +} + +static void +no_hidden_peer_source(void **state) +{ + struct jtc_wb_stats st = {0}; + + /* drain in flight on top of a completed rebuild, plus a new failure */ + _no_hidden_peer_source(4, 4, OC_EC_2P2G2, 1, true, 500, &st); + /* the same with a reintegration, which adds peers through regular UP targets */ + _no_hidden_peer_source(4, 4, OC_EC_2P2G2, 1, false, 500, &st); + _no_hidden_peer_source(5, 4, OC_EC_2P2G2, 1, false, 500, &st); + + /* + * The scenarios have to actually produce peers of their own and relocation peers, + * otherwise nothing was tested. With a reintegration in flight the relocation peer + * is the reintegrating target itself, which is not counted as relocated. + */ + assert_true(st.relocated > 0); + assert_true(st.peered > st.relocated); +} + /* * ------------------------------------------------ * End Test Cases * ------------------------------------------------ */ - static int placement_test_setup(void **state) { @@ -3521,6 +4137,11 @@ static const struct CMUnitTest tests[] = { pool_map_compat_all_downout_extend), T("reintegrating a creation-time DOWNOUT rank keeps existing layouts stable", creation_downout_reint_layout_stable), + T("no stale read source", no_stale_read_source), + T("extension preserves read sources without historical spares", + extension_preserves_read_sources), + T("no hidden peer source", no_hidden_peer_source), + T("relocation peer allocation failure", relocation_peer_alloc_failure), }; int diff --git a/src/placement/tests/place_obj_common.c b/src/placement/tests/place_obj_common.c index a266a997fe9..96be9707fd0 100644 --- a/src/placement/tests/place_obj_common.c +++ b/src/placement/tests/place_obj_common.c @@ -627,6 +627,8 @@ gen_pool_and_placement_map(int num_pds, int fdoms_per_pd, int nodes_per_domain, comp->co_rank = i / vos_per_target; comp->co_index = i % vos_per_target; comp->co_ver = 1; + /* gen_pool_buf() gives every target of a new pool fseq 1, not 0 */ + comp->co_fseq = 1; comp->co_nr = 1; } @@ -717,6 +719,7 @@ gen_pool_and_placement_map_non_standard(int num_domains, comp->co_rank = node_idx; comp->co_ver = 1; + comp->co_fseq = 1; comp->co_nr = 1; }