Tepid: only update indexes that have changed attributes

contrib/amcheck/expected/check_heap.out

@@ -231,6 +231,46 @@ SELECT * FROM verify_heapam('test_foreign_table',
 							endblock := NULL);
 ERROR:  cannot check relation "test_foreign_table"
 DETAIL:  This operation is not supported for foreign tables.
+-- HOT-indexed (HOT/SIU) on-page artifacts:
+--
+-- A HOT-indexed UPDATE keeps the new tuple on the same page as a heap-only
+-- tuple marked HEAP_INDEXED_UPDATED and plants index entries pointing at its
+-- own TID.  Pruning a chain of such updates collapses dead members to
+-- LP_REDIRECT forwarders and preserves the LP of a live HOT-indexed member
+-- whose index entries may not yet be swept.  verify_heapam must treat all of
+-- these as legitimate.  This scenario exercises them and asserts that
+-- verify_heapam reports zero corruption against legitimate HOT-indexed
+-- activity.
+CREATE TABLE hot_indexed_check (id int PRIMARY KEY, c1 int, c2 int, c3 int)
+	WITH (fillfactor = 70);
+CREATE INDEX hot_indexed_check_c1 ON hot_indexed_check (c1);
+CREATE INDEX hot_indexed_check_c2 ON hot_indexed_check (c2);
+INSERT INTO hot_indexed_check
+	SELECT g, g, g, g FROM generate_series(1, 200) g;
+-- Single-step UPDATEs: each row gets one HOT-indexed update.  Each
+-- successful HOT-indexed update keeps its new tuple on-page and inserts an
+-- entry only into the index whose attribute changed.
+UPDATE hot_indexed_check SET c1 = c1 + 1000;
+-- Multi-step UPDATEs: drive several successive HOT-indexed updates against
+-- the same rows so prune sees a chain of dead intermediates and collapses
+-- them to LP_REDIRECT forwarders.  An explicit VACUUM runs the prune path
+-- and exercises chain collapse.
+UPDATE hot_indexed_check SET c2 = c2 + 1 WHERE id <= 50;
+UPDATE hot_indexed_check SET c2 = c2 + 1 WHERE id <= 50;
+UPDATE hot_indexed_check SET c2 = c2 + 1 WHERE id <= 50;
+VACUUM (INDEX_CLEANUP off) hot_indexed_check;
+-- verify_heapam must not report any corruption against legitimate HOT-
+-- indexed artifacts.  Selecting the corrupting message makes any
+-- regression unmistakable in the regress diff.
+SELECT blkno, offnum, attnum, msg
+	FROM verify_heapam('hot_indexed_check',
+					   startblock := NULL,
+					   endblock := NULL);
+ blkno | offnum | attnum | msg 
+-------+--------+--------+-----
+(0 rows)
+
+DROP TABLE hot_indexed_check;
 -- cleanup
 DROP TABLE heaptest;
 DROP TABLESPACE regress_test_stats_tblspc;

contrib/amcheck/sql/check_heap.sql

@@ -138,6 +138,43 @@ SELECT * FROM verify_heapam('test_foreign_table',
 							startblock := NULL,
 							endblock := NULL);
 
+-- HOT-indexed (HOT/SIU) on-page artifacts:
+--
+-- A HOT-indexed UPDATE keeps the new tuple on the same page as a heap-only
+-- tuple marked HEAP_INDEXED_UPDATED and plants index entries pointing at its
+-- own TID.  Pruning a chain of such updates collapses dead members to
+-- LP_REDIRECT forwarders and preserves the LP of a live HOT-indexed member
+-- whose index entries may not yet be swept.  verify_heapam must treat all of
+-- these as legitimate.  This scenario exercises them and asserts that
+-- verify_heapam reports zero corruption against legitimate HOT-indexed
+-- activity.
+CREATE TABLE hot_indexed_check (id int PRIMARY KEY, c1 int, c2 int, c3 int)
+	WITH (fillfactor = 70);
+CREATE INDEX hot_indexed_check_c1 ON hot_indexed_check (c1);
+CREATE INDEX hot_indexed_check_c2 ON hot_indexed_check (c2);
+INSERT INTO hot_indexed_check
+	SELECT g, g, g, g FROM generate_series(1, 200) g;
+-- Single-step UPDATEs: each row gets one HOT-indexed update.  Each
+-- successful HOT-indexed update keeps its new tuple on-page and inserts an
+-- entry only into the index whose attribute changed.
+UPDATE hot_indexed_check SET c1 = c1 + 1000;
+-- Multi-step UPDATEs: drive several successive HOT-indexed updates against
+-- the same rows so prune sees a chain of dead intermediates and collapses
+-- them to LP_REDIRECT forwarders.  An explicit VACUUM runs the prune path
+-- and exercises chain collapse.
+UPDATE hot_indexed_check SET c2 = c2 + 1 WHERE id <= 50;
+UPDATE hot_indexed_check SET c2 = c2 + 1 WHERE id <= 50;
+UPDATE hot_indexed_check SET c2 = c2 + 1 WHERE id <= 50;
+VACUUM (INDEX_CLEANUP off) hot_indexed_check;
+-- verify_heapam must not report any corruption against legitimate HOT-
+-- indexed artifacts.  Selecting the corrupting message makes any
+-- regression unmistakable in the regress diff.
+SELECT blkno, offnum, attnum, msg
+	FROM verify_heapam('hot_indexed_check',
+					   startblock := NULL,
+					   endblock := NULL);
+DROP TABLE hot_indexed_check;
+
 -- cleanup
 DROP TABLE heaptest;
 DROP TABLESPACE regress_test_stats_tblspc;

contrib/amcheck/verify_heapam.c

@@ -13,6 +13,7 @@
 #include "access/detoast.h"
 #include "access/genam.h"
 #include "access/heaptoast.h"
+#include "access/hot_indexed.h"
 #include "access/multixact.h"
 #include "access/relation.h"
 #include "access/table.h"
@@ -522,9 +523,12 @@ verify_heapam(PG_FUNCTION_ARGS)
 			 */
 			if (ItemIdIsRedirected(ctx.itemid))
 			{
-				OffsetNumber rdoffnum = ItemIdGetRedirect(ctx.itemid);
+				OffsetNumber rdoffnum;
 				ItemId		rditem;
 
+				/* Resolve the redirect's target offset. */
+				rdoffnum = ItemIdGetRedirect(ctx.itemid);
+
 				if (rdoffnum < FirstOffsetNumber)
 				{
 					report_corruption(&ctx,
@@ -615,18 +619,47 @@ verify_heapam(PG_FUNCTION_ARGS)
 			ctx.tuphdr = (HeapTupleHeader) PageGetItem(ctx.page, ctx.itemid);
 			ctx.natts = HeapTupleHeaderGetNatts(ctx.tuphdr);
 
-			/* Ok, ready to check this next tuple */
-			check_tuple(&ctx,
-						&xmin_commit_status_ok[ctx.offnum],
-						&xmin_commit_status[ctx.offnum]);
+			/*
+			 * A HOT-selectively-updated collapse-survivor stub is an
+			 * LP_NORMAL item that is not a real tuple: HEAP_INDEXED_UPDATED
+			 * with natts == 0, permanently invisible (HEAP_XMIN_INVALID),
+			 * carrying a forward link and a modified-attrs bitmap.  The
+			 * per-tuple checks assume a real tuple and would misreport it, so
+			 * skip them; the update-chain pass below still records its
+			 * forward edge and treats it like a redirect (a forwarding node).
+			 */
+			if (!HotIndexedHeaderIsStub(ctx.tuphdr))
+				check_tuple(&ctx,
+							&xmin_commit_status_ok[ctx.offnum],
+							&xmin_commit_status[ctx.offnum]);
 
 			/*
 			 * If the CTID field of this tuple seems to point to another tuple
 			 * on the same page, record that tuple as the successor of this
-			 * one.
+			 * one.  A collapse-survivor stub stores its forward link in the
+			 * t_ctid offset only (the block half is repurposed to hold the
+			 * stub's write-time natts), so resolve its successor via the stub
+			 * accessor; the forward target is always on the same page.
 			 */
-			nextblkno = ItemPointerGetBlockNumber(&(ctx.tuphdr)->t_ctid);
-			nextoffnum = ItemPointerGetOffsetNumber(&(ctx.tuphdr)->t_ctid);
+			if (HotIndexedHeaderIsStub(ctx.tuphdr))
+			{
+				nextblkno = ctx.blkno;
+				nextoffnum = HotIndexedStubGetForward(ctx.tuphdr);
+				if (nextoffnum == ctx.offnum ||
+					nextoffnum < FirstOffsetNumber || nextoffnum > maxoff)
+				{
+					report_corruption(&ctx,
+									  psprintf("HOT-indexed stub forward link to item at offset %d is out of range or self-referential (valid range %d..%d)",
+											   nextoffnum,
+											   FirstOffsetNumber, maxoff));
+					continue;
+				}
+			}
+			else
+			{
+				nextblkno = ItemPointerGetBlockNumber(&(ctx.tuphdr)->t_ctid);
+				nextoffnum = ItemPointerGetOffsetNumber(&(ctx.tuphdr)->t_ctid);
+			}
 			if (nextblkno == ctx.blkno && nextoffnum != ctx.offnum &&
 				nextoffnum >= FirstOffsetNumber && nextoffnum <= maxoff)
 				successor[ctx.offnum] = nextoffnum;
@@ -675,7 +708,7 @@ verify_heapam(PG_FUNCTION_ARGS)
 				 */
 				Assert(ItemIdIsNormal(next_lp));
 
-				/* Can only redirect to a HOT tuple. */
+				/* A redirect targets the first surviving chain member. */
 				next_htup = (HeapTupleHeader) PageGetItem(ctx.page, next_lp);
 				if (!HeapTupleHeaderIsHeapOnly(next_htup))
 				{
@@ -687,6 +720,19 @@ verify_heapam(PG_FUNCTION_ARGS)
 				/* HOT chains should not intersect. */
 				if (predecessor[nextoffnum] != InvalidOffsetNumber)
 				{
+					/*
+					 * In the HOT/SIU model several redirects legitimately
+					 * forward to the same live tuple: when a chain collapses,
+					 * the root and each entry-bearing dead member become a
+					 * redirect to first_live so every stale btree entry still
+					 * resolves there (the read path then rechecks the leaf
+					 * key).  Multiple predecessors are therefore expected
+					 * when the target is HOT-selectively-updated; keep the
+					 * first predecessor and do not report it as corruption.
+					 */
+					if ((next_htup->t_infomask2 & HEAP_INDEXED_UPDATED) != 0)
+						continue;
+
 					report_corruption(&ctx,
 									  psprintf("redirect line pointer points to offset %d, but offset %d also points there",
 											   nextoffnum, predecessor[nextoffnum]));
@@ -701,6 +747,30 @@ verify_heapam(PG_FUNCTION_ARGS)
 				continue;
 			}
 
+			/*
+			 * A collapse-survivor stub forwards like a redirect: it is not a
+			 * real tuple, so don't apply the tuple-to-tuple update-chain
+			 * checks, but do record the predecessor edge to its target so the
+			 * live tuple it ultimately forwards to is not mistaken for a
+			 * chain root.  Its target must be heap-only (another stub or the
+			 * live heap-only tuple).
+			 */
+			curr_htup = (HeapTupleHeader) PageGetItem(ctx.page, curr_lp);
+			if (HotIndexedHeaderIsStub(curr_htup))
+			{
+				if (ItemIdIsNormal(next_lp))
+				{
+					next_htup = (HeapTupleHeader) PageGetItem(ctx.page, next_lp);
+					if (!HeapTupleHeaderIsHeapOnly(next_htup))
+						report_corruption(&ctx,
+										  psprintf("HOT-indexed stub forwards to a non-heap-only tuple at offset %d",
+												   nextoffnum));
+					else if (predecessor[nextoffnum] == InvalidOffsetNumber)
+						predecessor[nextoffnum] = ctx.offnum;
+				}
+				continue;
+			}
+
 			/*
 			 * If the next line pointer is a redirect, or if it's a tuple but
 			 * the XMAX of this tuple doesn't match the XMIN of the next
@@ -709,7 +779,6 @@ verify_heapam(PG_FUNCTION_ARGS)
 			 */
 			if (ItemIdIsRedirected(next_lp))
 				continue;
-			curr_htup = (HeapTupleHeader) PageGetItem(ctx.page, curr_lp);
 			curr_xmax = HeapTupleHeaderGetUpdateXid(curr_htup);
 			next_htup = (HeapTupleHeader) PageGetItem(ctx.page, next_lp);
 			next_xmin = HeapTupleHeaderGetXmin(next_htup);

contrib/amcheck/verify_nbtree.c

@@ -1489,6 +1489,29 @@ bt_target_page_check(BtreeCheckState *state)
 		if (state->heapallindexed && P_ISLEAF(topaque) && !ItemIdIsDead(itemid))
 		{
 			IndexTuple	norm;
+			ItemPointerData saved_tid;
+			bool		siu_stripped = false;
+
+			/*
+			 * A HOT-indexed (SIU) fresh entry stores its heap TID with the
+			 * ItemPointerSIUMaybeStaleFlag marker bit set in the offset field
+			 * (see storage/itemptr.h).  The marker is a bitmap-scan hint, not
+			 * part of the tuple's heap address, and the heapallindexed callback
+			 * (bt_tuple_present_callback) fingerprints the plain heap TID it
+			 * gets from the heap scan.  Strip the marker from this leaf tuple's
+			 * TID for the duration of fingerprinting so the two normalized
+			 * images match; restore it immediately afterward, since later
+			 * checks in this loop still read itup from the (immutable) page
+			 * image.  Only plain leaf tuples can carry the marker.
+			 */
+			if (!BTreeTupleIsPosting(itup) &&
+				ItemPointerIsSIUMaybeStale(&itup->t_tid))
+			{
+				saved_tid = itup->t_tid;
+				ItemPointerSetOffsetNumber(&itup->t_tid,
+										   ItemPointerGetOffsetNumber(&itup->t_tid));
+				siu_stripped = true;
+			}
 
 			if (BTreeTupleIsPosting(itup))
 			{
@@ -1516,6 +1539,10 @@ bt_target_page_check(BtreeCheckState *state)
 				if (norm != itup)
 					pfree(norm);
 			}
+
+			/* Restore the SIU marker bit stripped for fingerprinting, if any. */
+			if (siu_stripped)
+				itup->t_tid = saved_tid;
 		}
 
 		/*

contrib/bloom/expected/bloom.out

@@ -228,3 +228,35 @@ CREATE INDEX bloomidx2 ON tst USING bloom (i, t) WITH (length=0);
 ERROR:  value 0 out of bounds for option "length"
 CREATE INDEX bloomidx2 ON tst USING bloom (i, t) WITH (col1=0);
 ERROR:  value 0 out of bounds for option "col1"
+--
+-- HOT-indexed (SIU) BitmapAnd correctness with a bloom index.
+--
+-- A bloom index is bitmap-scan-only and non-summarizing, so it is eligible to
+-- be the "unchanged" side of a HOT-indexed update: the update changes a
+-- btree-indexed column while leaving the bloom-indexed column alone.  The
+-- btree fresh entry then points at the new heap-only tuple and the bloom entry
+-- still points at the chain root; a BitmapAnd of the two must not drop the
+-- matching row (the SIU fresh entry's TID carries an internal may-be-stale
+-- marker that tbm_add_tuples degrades to a lossy page, so the intersection
+-- keeps the row and the heap-side recheck resolves it).  This is handled
+-- centrally in tbm_add_tuples, so bloom -- like every other index AM -- needs
+-- no bloom-specific code for it; this test guards that.
+CREATE TABLE bloom_siu (id int PRIMARY KEY, bcol int, changed int) WITH (fillfactor = 50);
+CREATE INDEX bloom_siu_b ON bloom_siu USING bloom (bcol);
+CREATE INDEX bloom_siu_c ON bloom_siu (changed);
+INSERT INTO bloom_siu VALUES (1, 11, 21);
+UPDATE bloom_siu SET changed = 22 WHERE id = 1;   -- HOT-indexed: only "changed"
+SET enable_seqscan = off;
+SET enable_indexscan = off;
+SET enable_bitmapscan = on;
+-- Must return the row (previously a false negative before the SIU bitmap fix).
+SELECT count(*) AS bloom_bitmapand FROM bloom_siu WHERE bcol = 11 AND changed = 22;
+ bloom_bitmapand 
+-----------------
+               1
+(1 row)
+
+RESET enable_seqscan;
+RESET enable_indexscan;
+RESET enable_bitmapscan;
+DROP TABLE bloom_siu;

contrib/bloom/sql/bloom.sql

@@ -93,3 +93,31 @@ SELECT reloptions FROM pg_class WHERE oid = 'bloomidx'::regclass;
 \set VERBOSITY terse
 CREATE INDEX bloomidx2 ON tst USING bloom (i, t) WITH (length=0);
 CREATE INDEX bloomidx2 ON tst USING bloom (i, t) WITH (col1=0);
+
+--
+-- HOT-indexed (SIU) BitmapAnd correctness with a bloom index.
+--
+-- A bloom index is bitmap-scan-only and non-summarizing, so it is eligible to
+-- be the "unchanged" side of a HOT-indexed update: the update changes a
+-- btree-indexed column while leaving the bloom-indexed column alone.  The
+-- btree fresh entry then points at the new heap-only tuple and the bloom entry
+-- still points at the chain root; a BitmapAnd of the two must not drop the
+-- matching row (the SIU fresh entry's TID carries an internal may-be-stale
+-- marker that tbm_add_tuples degrades to a lossy page, so the intersection
+-- keeps the row and the heap-side recheck resolves it).  This is handled
+-- centrally in tbm_add_tuples, so bloom -- like every other index AM -- needs
+-- no bloom-specific code for it; this test guards that.
+CREATE TABLE bloom_siu (id int PRIMARY KEY, bcol int, changed int) WITH (fillfactor = 50);
+CREATE INDEX bloom_siu_b ON bloom_siu USING bloom (bcol);
+CREATE INDEX bloom_siu_c ON bloom_siu (changed);
+INSERT INTO bloom_siu VALUES (1, 11, 21);
+UPDATE bloom_siu SET changed = 22 WHERE id = 1;   -- HOT-indexed: only "changed"
+SET enable_seqscan = off;
+SET enable_indexscan = off;
+SET enable_bitmapscan = on;
+-- Must return the row (previously a false negative before the SIU bitmap fix).
+SELECT count(*) AS bloom_bitmapand FROM bloom_siu WHERE bcol = 11 AND changed = 22;
+RESET enable_seqscan;
+RESET enable_indexscan;
+RESET enable_bitmapscan;
+DROP TABLE bloom_siu;

contrib/pageinspect/Makefile

@@ -13,7 +13,7 @@ OBJS = \
 	rawpage.o
 
 EXTENSION = pageinspect
-DATA =  pageinspect--1.12--1.13.sql \
+DATA =  pageinspect--1.13--1.14.sql pageinspect--1.12--1.13.sql \
 	pageinspect--1.11--1.12.sql pageinspect--1.10--1.11.sql \
 	pageinspect--1.9--1.10.sql pageinspect--1.8--1.9.sql \
 	pageinspect--1.7--1.8.sql pageinspect--1.6--1.7.sql \