Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit172259a

Browse files
committed
Verify roundtrip dump/restore of regression database
Add a test to pg_upgrade's test suite that verifies thatdump-restore-dump of regression database produces equivalent output todumping it directly. This was already being tested by runningpg_upgrade itself, but non-binary-upgrade mode was not being covered.The regression database has accrued, over time, a sufficient collectionof interesting objects to ensure good coverage, but there hasn't been aconcerted effort to be completely exhaustive, so it is likely stillpossible to have more.This'd belong more naturally in the pg_dump test suite, but we chose toput it in src/bin/pg_upgrade/t/002_pg_upgrade.pl because we need a runof the regression tests which is already done here, so this has lesstotal test runtime impact. Also, experiments have shown that usingparallel dump/restore is slightly faster, so we use --format=directory -j2.This test has already reported pg_dump bugs, as fixed infd41ba9,74563f6,d611f8b,4694aed.Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com>Reviewed-by: Michael Paquier <michael@paquier.xyz>Reviewed-by: Daniel Gustafsson <daniel@yesql.se>Reviewed-by: Tom Lane <tgl@sss.pgh.pa.us>Reviewed-by: Álvaro Herrera <alvherre@alvh.no-ip.org>Discussion:https://www.postgresql.org/message-id/CAExHW5uF5V=Cjecx3_Z=7xfh4rg2Wf61PT+hfquzjBqouRzQJQ@mail.gmail.com
1 parent764d501 commit172259a

File tree

4 files changed

+254
-3
lines changed

4 files changed

+254
-3
lines changed

‎src/bin/pg_upgrade/t/002_pg_upgrade.pl

Lines changed: 94 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,15 @@
1111

1212
use PostgreSQL::Test::Cluster;
1313
use PostgreSQL::Test::Utils;
14+
use PostgreSQL::Test::AdjustDump;
1415
use PostgreSQL::Test::AdjustUpgrade;
1516
use Test::More;
1617

1718
# Can be changed to test the other modes.
1819
my$mode =$ENV{PG_TEST_PG_UPGRADE_MODE} ||'--copy';
1920

21+
my$tempdir = PostgreSQL::Test::Utils::tempdir;
22+
2023
# Generate a database with a name made of a range of ASCII characters.
2124
subgenerate_db
2225
{
@@ -35,8 +38,8 @@ sub generate_db
3538
"created database with ASCII characters from$from_char to$to_char");
3639
}
3740

38-
# Filter the contents of a dump before its use in a content comparison.
39-
# This returns the path to the filtered dump.
41+
# Filter the contents of a dump before its use in a content comparison for
42+
#upgrade testing.This returns the path to the filtered dump.
4043
subfilter_dump
4144
{
4245
my ($is_old,$old_version,$dump_file) =@_;
@@ -60,6 +63,41 @@ sub filter_dump
6063
return$dump_file_filtered;
6164
}
6265

66+
# Dump database db from the given node in plain format and adjust it for
67+
# comparing dumps from the original and the restored database.
68+
#
69+
# file_prefix is used to create unique names for all dump files so that they
70+
# remain available for debugging in case the test fails.
71+
#
72+
# adjust_child_columns is passed to adjust_regress_dumpfile() which actually
73+
# adjusts the dump output.
74+
#
75+
# The name of the file containting adjusted dump is returned.
76+
subget_dump_for_comparison
77+
{
78+
my ($node,$db,$file_prefix,$adjust_child_columns) =@_;
79+
80+
my$dumpfile =$tempdir .'/' .$file_prefix .'.sql';
81+
my$dump_adjusted ="${dumpfile}_adjusted";
82+
83+
open(my$dh,'>',$dump_adjusted)
84+
||die"could not open$dump_adjusted for writing$!";
85+
86+
# Don't dump statistics, because there are still some bugs.
87+
$node->run_log(
88+
[
89+
'pg_dump','--no-sync','--no-statistics',
90+
'-d'=>$node->connstr($db),
91+
'-f'=>$dumpfile
92+
]);
93+
94+
print$dh adjust_regress_dumpfile(slurp_file($dumpfile),
95+
$adjust_child_columns);
96+
close($dh);
97+
98+
return$dump_adjusted;
99+
}
100+
63101
# The test of pg_upgrade requires two clusters, an old one and a new one
64102
# that gets upgraded. Before running the upgrade, a logical dump of the
65103
# old cluster is taken, and a second logical dump of the new one is taken
@@ -80,7 +118,6 @@ sub filter_dump
80118
}
81119

82120
# Paths to the dumps taken during the tests.
83-
my$tempdir = PostgreSQL::Test::Utils::tempdir;
84121
my$dump1_file ="$tempdir/dump1.sql";
85122
my$dump2_file ="$tempdir/dump2.sql";
86123

@@ -264,6 +301,60 @@ sub filter_dump
264301
is($rc, 0,'regression tests pass');
265302
}
266303

304+
# Test that dump/restore of the regression database roundtrips cleanly. This
305+
# doesn't work well when the nodes are different versions, so skip it in that
306+
# case. Note that this isn't a pg_restore test, but it's convenient to do it
307+
# here because we've gone to the trouble of creating the regression database.
308+
#
309+
# Do this while the old cluster is running before it is shut down by the
310+
# upgrade test.
311+
SKIP:
312+
{
313+
my$dstnode = PostgreSQL::Test::Cluster->new('dst_node');
314+
315+
skip"different Postgres versions"
316+
if ($oldnode->pg_version !=$dstnode->pg_version);
317+
skip"source node not using default install"
318+
if (defined$oldnode->install_path);
319+
320+
# Dump the original database for comparison later.
321+
my$src_dump =
322+
get_dump_for_comparison($oldnode,'regression','src_dump', 1);
323+
324+
# Setup destination database cluster
325+
$dstnode->init(%node_params);
326+
# Stabilize stats for comparison.
327+
$dstnode->append_conf('postgresql.conf','autovacuum = off');
328+
$dstnode->start;
329+
330+
my$dump_file ="$tempdir/regression.dump";
331+
332+
# Use --create in dump and restore commands so that the restored
333+
# database has the same configurable variable settings as the original
334+
# database so that the dumps taken from both databases taken do not
335+
# differ because of locale changes. Additionally this provides test
336+
# coverage for --create option.
337+
#
338+
# Use directory format so that we can use parallel dump/restore.
339+
$oldnode->command_ok(
340+
[
341+
'pg_dump','-Fd','-j2','--no-sync',
342+
'-d'=>$oldnode->connstr('regression'),
343+
'--create','-f'=>$dump_file
344+
],
345+
'pg_dump on source instance');
346+
347+
$dstnode->command_ok(
348+
['pg_restore','--create','-j2','-d'=>'postgres',$dump_file ],
349+
'pg_restore to destination instance');
350+
351+
my$dst_dump =
352+
get_dump_for_comparison($dstnode,'regression','dest_dump', 0);
353+
354+
compare_files($src_dump,$dst_dump,
355+
'dump outputs from original and restored regression databases match');
356+
}
357+
267358
# Initialize a new node for the upgrade.
268359
my$newnode = PostgreSQL::Test::Cluster->new('new_node');
269360

‎src/test/perl/Makefile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ install: all installdirs
2525
$(INSTALL_DATA)$(srcdir)/PostgreSQL/Test/Kerberos.pm'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Kerberos.pm'
2626
$(INSTALL_DATA)$(srcdir)/PostgreSQL/Test/Cluster.pm'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Cluster.pm'
2727
$(INSTALL_DATA)$(srcdir)/PostgreSQL/Test/BackgroundPsql.pm'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/BackgroundPsql.pm'
28+
$(INSTALL_DATA)$(srcdir)/PostgreSQL/Test/AdjustDump.pm'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustDump.pm'
2829
$(INSTALL_DATA)$(srcdir)/PostgreSQL/Test/AdjustUpgrade.pm'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustUpgrade.pm'
2930
$(INSTALL_DATA)$(srcdir)/PostgreSQL/Version.pm'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Version.pm'
3031

@@ -35,6 +36,7 @@ uninstall:
3536
rm -f'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Kerberos.pm'
3637
rm -f'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Cluster.pm'
3738
rm -f'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/BackgroundPsql.pm'
39+
rm -f'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustDump.pm'
3840
rm -f'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustUpgrade.pm'
3941
rm -f'$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Version.pm'
4042

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
# Copyright (c) 2024-2025, PostgreSQL Global Development Group
2+
3+
=pod
4+
5+
=head1NAME
6+
7+
PostgreSQL::Test::AdjustDump - helper module for dump/restore tests
8+
9+
=head1SYNOPSIS
10+
11+
use PostgreSQL::Test::AdjustDump;
12+
13+
# Adjust contents of dump output file so that dump output from original
14+
# regression database and that from the restored regression database match
15+
$dump = adjust_regress_dumpfile($dump, $adjust_child_columns);
16+
17+
=head1DESCRIPTION
18+
19+
C<PostgreSQL::Test::AdjustDump> encapsulates various hacks needed to
20+
compare the results of dump/restore tests.
21+
22+
=cut
23+
24+
packagePostgreSQL::Test::AdjustDump;
25+
26+
use strict;
27+
use warningsFATAL=>'all';
28+
29+
use Exporter'import';
30+
use Test::More;
31+
32+
our@EXPORT =qw(
33+
adjust_regress_dumpfile
34+
);
35+
36+
=pod
37+
38+
=head1ROUTINES
39+
40+
=over
41+
42+
=item$dump = adjust_regress_dumpfile($dump, $adjust_child_columns)
43+
44+
Edit a dump output file, taken from the source regression database,
45+
to remove the known differences to a dump taken after restoring the
46+
same database.
47+
48+
Arguments:
49+
50+
=over
51+
52+
=itemC<dump>: Contents of dump file
53+
54+
=itemC<adjust_child_columns>: 1 indicates that the given dump file requires
55+
adjusting columns in the child tables; usually when the dump is from original
56+
database. 0 indicates no such adjustment is needed; usually when the dump is
57+
from restored database.
58+
59+
=back
60+
61+
Returns the adjusted dump text.
62+
63+
Adjustments Applied:
64+
65+
=over
66+
67+
=itemColumn reordering on child table creation
68+
69+
This rearranges the column declarations in theC<CREATE TABLE... INHERITS>
70+
statements in the dump file from original database so that they match those
71+
from the restored database.
72+
73+
Only executed ifC<adjust_child_columns> is true.
74+
75+
Context: some regression tests purposefully create child tables in such a way
76+
that the order of their inherited columns differ from column orders of their
77+
respective parents. In the restored database, however, the order of their
78+
inherited columns are same as that of their respective parents. Thus the column
79+
orders of these child tables in the original database and those in the restored
80+
database differ, causing difference in the dump outputs. See
81+
C<MergeAttributes()> andC<dumpTableSchema()> for details.
82+
83+
=itemRemoval of problematicC<COPY> statements
84+
85+
Remove COPY statements to abnormal children tables.
86+
87+
Context: This difference is caused because of columns that are added to parent
88+
tables that already have children; because recreating the children tables puts
89+
the columns from the parent ahead of columns declared locally in children,
90+
these extra columns are in earlier position compared to the original database.
91+
Reordering columns on the entireC<COPY> data is impractical, so we just remove
92+
them.
93+
94+
=itemNewline adjustment
95+
96+
Windows-style newlines are changed to Unix-style. Empty lines are trimmed.
97+
98+
=back
99+
100+
=cut
101+
102+
subadjust_regress_dumpfile
103+
{
104+
my ($dump,$adjust_child_columns) =@_;
105+
106+
# use Unix newlines
107+
$dump =~s/\r\n/\n/g;
108+
109+
# Adjust the CREATE TABLE ... INHERITS statements.
110+
if ($adjust_child_columns)
111+
{
112+
$dump =~s/(^CREATE\sTABLE\sgenerated_stored_tests\.gtestxx_4\s\()
113+
(\n\s+b\sinteger),
114+
(\n\s+a\sinteger\sNOT\sNULL)/$1$3,$2/mgx;
115+
116+
$dump =~s/(^CREATE\sTABLE\sgenerated_virtual_tests\.gtestxx_4\s\()
117+
(\n\s+b\sinteger),
118+
(\n\s+a\sinteger\sNOT\sNULL)/$1$3,$2/mgx;
119+
120+
$dump =~s/(^CREATE\sTABLE\spublic\.test_type_diff2_c1\s\()
121+
(\n\s+int_four\sbigint),
122+
(\n\s+int_eight\sbigint),
123+
(\n\s+int_two\ssmallint)/$1$4,$2,$3/mgx;
124+
125+
$dump =~s/(^CREATE\sTABLE\spublic\.test_type_diff2_c2\s\()
126+
(\n\s+int_eight\sbigint),
127+
(\n\s+int_two\ssmallint),
128+
(\n\s+int_four\sbigint)/$1$3,$4,$2/mgx;
129+
}
130+
131+
# Remove COPY statements with differing column order
132+
formy$table (
133+
'public\.b_star','public\.c_star',
134+
'public\.cc2','public\.d_star',
135+
'public\.e_star','public\.f_star',
136+
'public\.renamecolumnanother','public\.renamecolumnchild',
137+
'public\.test_type_diff2_c1','public\.test_type_diff2_c2',
138+
'public\.test_type_diff_c')
139+
{
140+
# This multiline pattern matches the whole COPY, up to the
141+
# terminating "\."
142+
$dump =~s/^COPY $table\(.+?^\\\.$//sm;
143+
}
144+
145+
# Suppress blank lines, as some places in pg_dump emit more or fewer.
146+
$dump =~s/\n\n+/\n/g;
147+
148+
return$dump;
149+
}
150+
151+
=pod
152+
153+
=back
154+
155+
=cut
156+
157+
1;

‎src/test/perl/meson.build

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@ install_data(
1313
'PostgreSQL/Test/Kerberos.pm',
1414
'PostgreSQL/Test/Cluster.pm',
1515
'PostgreSQL/Test/BackgroundPsql.pm',
16+
'PostgreSQL/Test/AdjustDump.pm',
1617
'PostgreSQL/Test/AdjustUpgrade.pm',
1718
install_dir: dir_pgxs/'src/test/perl/PostgreSQL/Test')

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp