96bef763-76aa-4452-b071-5b849b31cd38
/
PolarDBforPostgreSQL
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
PolarDBforPostgreSQL/polardb_build.sh

948 lines
30 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
#
# Copyright (c) 2020, Alibaba Group Holding Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##################### PHASE 1: set up parameter #####################
usage () {
cat <<EOF
This script is to be used to compile PG core source code (PG engine code without files within contrib and external)
It can be called with following options:
--basedir=<temp dir for PG installation>, specifies which dir to install PG to, note this dir would be cleaned up before being used
--datadir=<temp dir for databases>], specifies which dir to store database cluster, note this dir would be cleaned up before being used
--conf=<file path for postgresql.conf>, specifies the configure file to use
--user=<user to start PG>, specifies which user to run PG as
--port=<port to run PG on>, specifies which port to run PG on
--debug=[on|off], specifies whether to compile PG with debug mode (affecting gcc flags)
-c,--coverage, specifies whether to build PG with coverage option
--nc,--nocompile, prevents re-compilation, re-installation, and re-initialization
-t,-r,--regress, runs regression test after compilation and installation.
-m --minimal compile with minimal extention set
--withrep init the database with a hot standby replica
--withstandby init the database with a hot standby replica
--pg_bld_rep_port=<port to run PG rep on>, specifies which port to run PG replica on
--pg_bld_standby_port=<port to run PG standby on>, specifies which port to run PG standby on
--repdir=<temp dir for databases>], specifies which dir to store replica data, note this dir would be cleaned up before being used
--storage=localfs, specify storage type
-e,--extension, run extension test
--with-tde, TDE enable
--with-dma, DMA enable
--with-pfsd, PFSD enable
--fault-injector, faultinjector enable
--without-fbl, run without flashback log
--extra-conf, add an extra conf file
Please lookup the following secion to find the default values for above options.
Typical command patterns to kick off this script:
1) To just cleanup, re-compile, re-install and get PG restart:
polardb_build.sh
2) To run all steps included 1), as well as run the ALL regression test cases:
polardb_build.sh -t
3) To cleanup and re-compile with code coverage option:
polardb_build.sh -c
4) To run the tests besides 3).
polardb_build.sh -c -t
5) To run with specific port, user, and/or configuration file
polardb_build.sh --port=5501 --user=pg001 --conf=/root/data/postgresql.conf
6) To run on local pfs
polardb_build.sh --storage=localfs
7) To run with a replica (it also works with --storage=localfs)
polardb_build.sh --withrep
8) To run with a standby (it also works with --storage=localfs)
polardb_build.sh --withstandby
9) To run all the tests (make check)(include src/test,src/pl,src/interfaces/ecpg,contrib,external)
polardb_build.sh -r-check-all
10) To run all the tests (make installcheck)(include src/test,src/pl,src/interfaces/ecpg,contrib,external)
polardb_build.sh -r-installcheck-all
EOF
exit 0;
}
su_eval() {
if [[ -z $su_str ]];
then
eval "$1"
else
eval "$su_str \"$1\" "
fi
}
function del_cov() {
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_master_dir restart -w -c -o '-p $pg_bld_port'"
if [[ $withrep == "yes" ]];
then
for i in $(seq 1 $repnum)
do
if [[ $repnum == "1" ]];
then
pg_bld_replica_dir_n=${pg_bld_replica_dir}
else
pg_bld_replica_dir_n=${pg_bld_replica_dir}${i}
fi
pg_bld_rep_port_n=$(($pg_bld_rep_port+$i-1))
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_replica_dir_n restart -w -c -o '-p $pg_bld_rep_port_n'"
done
fi
if [[ $withstandby == "yes" ]];
then
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_standby_dir restart -w -c -o '-p $pg_bld_standby_port'"
fi
make coverage-html -sj`getconf _NPROCESSORS_ONLN`
perl delta_coverage.pl -p $prev_commit
cd coverage
sed -i -E "s/Line Coverage/Line Coverage\/<a href=\"delta_coverage.txt\">Delta Coverage<\/a>/" index.html
sed -i -E "s/Line Coverage/Line Coverage\/<a href=\"delta_coverage.txt\">Delta Coverage<\/a>/" index-sort-l.html
cd ..
}
function px_init() {
echo "################################ px_init ################################"
echo "polar_enable_px=0" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_enable_check_workers=0" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_enable_replay_wait=1" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_dop_per_node=3" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_max_workers_number=0" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_enable_cte_shared_scan=1" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_enable_partition=1" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_enable_left_index_nestloop_join=1" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_wait_lock_timeout=1800000" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_enable_partitionwise_join=1" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_optimizer_multilevel_partitioning=1" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_max_slices=1000000" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_enable_adps=1" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_enable_adps_explain_analyze=1" >> $pg_bld_master_dir/postgresql.conf
echo "polar_trace_heap_scan_flow=1" >> $pg_bld_master_dir/postgresql.conf
echo "polar_px_enable_spi_read_all_namespaces=1" >> $pg_bld_master_dir/postgresql.conf
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_master_dir reload"
}
function check_core() {
pg_bld_master_dir_n=${pg_bld_master_dir}
corefile=`find ${pg_bld_master_dir_n} -name core.*`
if [[ -n $corefile ]];
then
echo "find core in master" >> testlog
exit
fi
if [[ $withrep == "yes" ]];
then
for i in $(seq 1 $repnum)
do
if [[ $repnum == "1" ]];
then
pg_bld_replica_dir_n=${pg_bld_replica_dir}
corefile=`find ${pg_bld_replica_dir_n} -name core.*`
if [[ -n $corefile ]];
then
echo "find core in replica" >> testlog
exit
fi
else
pg_bld_replica_dir_n=${pg_bld_replica_dir}${i}
corefile=`find ${pg_bld_replica_dir_n} -name core.*`
if [[ -n $corefile ]];
then
echo "find core in replica" >> testlog
exit
fi
fi
done
fi
}
function check_failed_case() {
failcases=`grep "\.\. FAILED" testlog | head -n 1`
failcases=`grep "\.\. Failed" testlog | head -n 1`$failcases
failcases=`grep "CMake Error at" testlog| head -n 1`$failcases
failcases=`grep "could not connect to server" testlog | head -n 1`$failcases
failcases=`grep "terminating connection due to administrator command" testlog | head -n 1`$failcases
failcases=`grep "FAIL" testlog | head -n 1`$failcases
failcases=`grep "Dubious, test returned 255" testlog | head -n 1`$failcases
failcases=`grep "FATAL" testlog | head -n 1`$failcases
failcases=`grep "Error 2" testlog | head -n 1`$failcases
check_core
if [[ -n $failcases ]];
then
echo ""
echo ""
echo "============================"
echo "Suspected failing records:"
echo "============================"
echo ""
grep "\.\. FAILED" testlog
grep "\.\. Failed" testlog
grep "CMake Error at" testlog
grep "could not connect to server" testlog
grep "terminating connection due to administrator command" testlog
grep "FATAL" testlog
grep "Dubious, test returned 255" testlog
else
rm -f testlog
echo ""
echo ""
echo "==========================="
echo "Congratulations!! All cases" $1 "passed!"
echo "==========================="
echo ""
fi
}
pg_bld_prefix=$HOME
pg_bld_basedir=$pg_bld_prefix/tmp_basedir_polardb_pg_1100_bld
pg_bld_data_dir=$pg_bld_prefix/tmp_datadir_polardb_pg_1100_bld
pg_bld_master_dir=$pg_bld_prefix/tmp_master_dir_polardb_pg_1100_bld
pg_bld_replica_dir=$pg_bld_prefix/tmp_replica_dir_polardb_pg_1100_bld
pg_bld_standby_dir=$pg_bld_prefix/tmp_standby_dir_polardb_pg_1100_bld
pg_bld_standby_data_dir=$pg_bld_prefix/tmp_standby_datadir_polardb_pg_1100_bld
pg_bld_user=`whoami`
pg_bld_port=5432
pg_bld_rep_port=5433
pg_bld_standby_port=5434
pg_db_user=postgres
current_branch=`git rev-parse --abbrev-ref HEAD`
if [[ $current_branch == "HEAD" ]];
then
current_branch=`git rev-parse HEAD~0`
fi
prev_commit=`git rev-parse HEAD~1`
next_commit=`git rev-parse HEAD~0`
nocompile=off
debug_mode=off
coverage=off
regress_test=off
regress_test_ng=off
px_debug_mode=off
regress_test_quick=off
make_installcheck_world=off
make_check_world=off
minimal_compile=off
static_check=off
tap_tests=off
delta_cov=off
enable_asan=off
tde=off
dma=off
fault_injector=off
enable_flashback_log=on
pfsd=off
enc_func="aes-256"
SRCDIR=`pwd` # 源码文件目录
# unset PG* envs
unset PGPORT
unset PGDATA
unset PGHOST
unset PGDATABASE
unset PGUSER
function set_env() {
export PGPORT=$pg_bld_port
export PGUSER=postgres
export PGHOST=$pg_bld_master_dir
export PGDATABASE=postgres
}
# is data dir path specified by user
data_dir_specified=no
rep_dir_specified=no
standby_dir_specified=no
withrep=no
withstandby=no
need_initdb=no
need_initrep=no
noclean=no
normbasedir=no
noinit=no
withpx=no
initpx=no
repnum=1
for arg do
# the parameter after "=", or the whole $arg if no match
val=`echo "$arg" | sed -e 's;^--[^=]*=;;'`
#
# Note we use eval for dir assignment to expand ~
#
case "$arg" in
--basedir=*) eval pg_bld_basedir="$val" ;;
--datadir=*) eval pg_bld_master_dir="$val"; data_dir_specified=yes ;;
--repdir=*) eval pg_bld_replica_dir="$val"; rep_dir_specified=yes ;;
--standbydir=*) eval pg_bld_standby_dir="$val"; standby_dir_specified=yes ;;
--user=*) pg_bld_user="$val" ;;
--port=*) pg_bld_port="$val" ;;
--repport=*) pg_bld_rep_port="$val" ;;
--standbyport=*) pg_bld_standby_port="$val" ;;
-h|--help) usage ;;
--nc|--nocompile) nocompile=on ;;
--debug=*) debug_mode="$val" ;;
--px-debug=*) px_debug_mode="$val" ;;
--prev_commit=*) prev_commit="$val" ;;
--next_commit=*) next_commit="$val" ;;
--withrep*) withrep=yes ;;
--withstandby*) withstandby=yes ;;
--withpx*) withpx=yes ;;
--initpx*) withpx=yes;
initpx=yes ;;
--repnum*) repnum="$val" ;;
-c|--coverage) debug_mode=on;
coverage=on
;;
-r|-t|--regress) regress_test=on
;;
-r-ng) regress_test_ng=on
tap_tests=on
;;
-r-quick|-t-quick) regress_test_quick=on
;;
-r-px) regress_test=on;
withpx=yes;
initpx=yes
;;
-r-check-all) make_check_world=on;
tap_tests=on
;;
-r-installcheck-all) make_installcheck_world=on;
tap_tests=on
;;
--noclean) noclean=yes ;; #do not make distclean
--normbasedir) normbasedir=yes ;; #do not remove data and base dirs
-m|--minimal) minimal_compile=on ;;
--storage=*) storage="$val" ;; #default localfs, don't delete it for habit
--tap) tap_tests=on;;
--delta_coverage) delta_coverage=on
;;
-e|--extension) extension_test=on
;;
-r-external) external_test=on
;;
-r-contrib) contrib_test=on
;;
-r-pl) pl_test=on
;;
--asan) enable_asan=on;;
--with-tde) tde=on
;;
--enc=*) eval enc_func="$val" #default aes-256 ,support aes-256、aes-128、sm4
;;
--with-dma) dma=on
;;
--with-pfsd) pfsd=on
;;
--fault-injector) fault_injector=on
;;
--without-fbl) enable_flashback_log=off
;;
--noinit) noinit=yes
;;
--extra-conf=*) extra_conf="$val"
;;
*) echo "wrong options : $arg";
exit 1
;;
esac
done
set_env
if [[ $delta_coverage == "on" ]];
then
del_cov;
exit;
fi
su_str=
if [[ "$EUID" == 0 ]];
then
echo "Running with user $pg_bld_user!"
su_str="su $pg_bld_user -c "
fi
if [[ $withpx == "yes" ]];
then
echo "################################ build with px ################################"
configure_flag+=" --enable-polar-px"
if [[ $noinit == "no" ]];
then
withrep="yes"
if [[ $repnum == "1" ]];
then
repnum=2
fi
fi
fi
if [[ $initpx == "yes" ]];
then
export PG_REGRESS_PX_MODE="--polar-parallel-execution --load-extension=polar_px"
fi
####### PHASE 2: cleanup dir and env, and set new dir and env #######
# stop server maybe failed, just ignore
set +e
# cleanup dirs and PG process
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_master_dir stop -m i"
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_replica_dir stop -m i"
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_standby_dir stop -m i"
# reset
set -e
# must clean before dir remove, some module depend on base_dir/bin/pg_config
if [[ $noclean == "no" ]];
then
make -s distclean || true
fi
mkdir -p $pg_bld_basedir
if [[ $noinit == "no" ]];
then
mkdir -p $pg_bld_master_dir
mkdir -p $pg_bld_replica_dir
mkdir -p $pg_bld_data_dir
fi
if [[ $normbasedir == "no" ]] && [[ -n $pg_bld_basedir ]] && [[ -d $pg_bld_basedir/bin ]];
then
rm -fr $pg_bld_basedir/*
fi
# cleanup datadir only if it is not specified explicitly by user (so a default trival one)
if [[ $noinit == "no" ]] && [[ $data_dir_specified == "no" ]] && [[ -n $pg_bld_master_dir ]];
then
rm -fr $pg_bld_master_dir/*
rm -fr $pg_bld_data_dir/*
need_initdb=yes
fi
gcc_opt_level_flag="-g -pipe -Wall -grecord-gcc-switches -I/usr/include/et"
tde_initdb_args=""
if [[ $debug_mode == "on" ]];
then
gcc_opt_level_flag+=" -ggdb -O0 -g3 -fno-omit-frame-pointer -fstack-protector-strong --param=ssp-buffer-size=4"
configure_flag+=" --enable-debug --enable-cassert --enable-tap-tests"
export COPT="-Werror"
else
gcc_opt_level_flag+=" -O3"
if [[ $withpx == "yes" ]];
then
gcc_opt_level_flag+=" -Wp,-D_FORTIFY_SOURCE=2"
fi
fi
if [[ $px_debug_mode == "on" ]];
then
configure_flag+=" --enable-gpos-debug"
fi
if [[ $enable_asan == "on" ]];
then
gcc_opt_level_flag+=" -fsanitize=address -fno-omit-frame-pointer"
#Asan core file is huge (>15T), so we do not allow coredump.
export ASAN_OPTIONS="alloc_dealloc_mismatch=0:"
fi
if [[ $coverage == "on" ]];
then
configure_flag+=" --enable-coverage"
fi
if [[ $tap_tests == "on" ]];
then
configure_flag+=" --enable-tap-tests"
fi
if [[ $fault_injector == "on" ]]
then
configure_flag+=" --enable-inject-faults"
fi
if [[ $minimal_compile == "on" ]];
then
configure_flag+=" --without-readline --without-zlib --enable-polar-minimal"
else
configure_flag+=" --with-openssl --with-libxml --with-perl --with-python --with-tcl --with-pam --with-gssapi --enable-nls --with-libxslt --with-ldap --with-uuid=e2fs --with-icu --with-llvm"
fi
if [[ $pfsd == "on" ]];
then
configure_flag+=" --with-pfsd"
fi
dma_regress_opt=
if [[ $dma == "on" ]];
then
dma_regress_opt="DMA_OPTS=cluster"
configure_flag+=" --with-dma"
fi
# setup env
export CFLAGS=" $gcc_opt_level_flag"
export CXXFLAGS="$gcc_opt_level_flag"
export LDFLAGS="-Wl,-rpath,'\$\$ORIGIN/../lib'"
# to get rid of postgis db creation error
export LANG=en_US.UTF-8
# need to set LD_LIBRARY_PATH for ecpg test cases to link correct libraries when compiled and run by its pg_regress
export LD_LIBRARY_PATH=$pg_bld_basedir/lib:$LD_LIBRARY_PATH
# may be needed by some module Makefile to get the correct pg_config
export PATH=$pg_bld_basedir/bin:$PATH
#################### PHASE 3: compile and install ###################
if [[ $nocompile == "off" ]];
then
./configure --prefix=$pg_bld_basedir --with-pgport=$pg_bld_port $common_configure_flag $configure_flag
for target in . contrib external
do
make -sj`getconf _NPROCESSORS_ONLN` -C $target
make install -s -C $target
done
fi ##nocoimpile == off
###################### PHASE 4 Test: run test cases in non-polar ######################
if [[ $make_check_world == "on" ]];
then
unset COPT
su_eval "make $dma_regress_opt check-world PG_TEST_EXTRA='kerberos ldap ssl' 2>&1" | tee -a testlog
su_eval "make $dma_regress_opt polar-check 2>&1" | tee -a testlog
check_failed_case "check-world"
fi
if [[ $regress_test == "on" ]] && [[ $initpx != "yes" ]];
then
su_eval "make $dma_regress_opt polar-check 2>&1" | tee -a testlog
if [ -e ./src/test/regress/regression.diffs ]
then
cat ./src/test/regress/regression.diffs
exit 1
fi
fi
###################### PHASE 5: init and start ######################
# to protect us from running from wrong dir, use ../polardb_pg etc. here
if [[ $noinit == "no" ]] && [[ "$EUID" == 0 ]];
then
chown -R $pg_bld_user ../polardb_pg
chown -R $pg_bld_user $pg_bld_basedir
chown -R $pg_bld_user $pg_bld_master_dir
fi
if [[ $tde == "on" ]];
then
tde_initdb_args="--cluster-passphrase-command 'echo \"adfadsfadssssssssfa123123123123123123123123123123123123111313123\"' -e $enc_func"
fi
if [[ $need_initdb == "yes" ]];
then
su_eval "$pg_bld_basedir/bin/initdb -k -U $pg_db_user -D $pg_bld_master_dir $tde_initdb_args"
# common configs
echo "polar_enable_shared_storage_mode = on
polar_hostid = 1
max_connections = 100
polar_wal_pipeline_enable = true
polar_create_table_with_full_replica_identity = off
logging_collector = on
log_directory = 'pg_log'
unix_socket_directories='.'
shared_buffers = 128MB
synchronous_commit = on
full_page_writes = off
#random_page_cost = 1.1
autovacuum_naptime = 10min
max_worker_processes = 32
polar_use_statistical_relpages = off
polar_enable_persisted_buffer_pool = off
polar_nblocks_cache_mode = 'all'
polar_enable_replica_use_smgr_cache = on
polar_enable_standby_use_smgr_cache = on" >> $pg_bld_master_dir/postgresql.conf
if [[ $enable_flashback_log == "on" ]];
then
echo "polar_enable_flashback_log = on
polar_enable_fast_recovery_area = on" >> $pg_bld_master_dir/postgresql.conf
fi
# echo "max_wal_size = 16GB
# min_wal_size = 4GB" >> $pg_bld_master_dir/postgresql.conf
disk_name=`echo $pg_bld_data_dir | cut -d '/' -f2`
echo "polar_vfs.localfs_mode = true
polar_enable_localfs_test_mode = on
polar_enable_shared_storage_mode = on
listen_addresses = '*'
polar_disk_name = '${disk_name}'
polar_datadir = 'file-dio://${pg_bld_data_dir}'" >> $pg_bld_master_dir/postgresql.conf
echo "shared_preload_libraries = '\$libdir/polar_px,\$libdir/polar_vfs,\$libdir/polar_worker,\$libdir/pg_stat_statements,\$libdir/auth_delay,\$libdir/auto_explain,\$libdir/polar_monitor_preload,\$libdir/polar_stat_sql'" >> $pg_bld_master_dir/postgresql.conf
su_eval "bash $pg_bld_basedir/bin/polar-initdb.sh ${pg_bld_master_dir}/ ${pg_bld_data_dir}/ localfs"
if [[ $dma == "on" ]];
then
echo "polar_logindex_mem_size = 0
polar_enable_flashback_log = off" >> $pg_bld_master_dir/postgresql.conf
echo "polar_enable_dma = on
polar_dma_repl_user = $pg_db_user" >> $pg_bld_master_dir/polar_dma.conf
su_eval "$pg_bld_basedir/bin/postgres -D $pg_bld_master_dir -p $pg_bld_port -c polar_dma_init_meta=ON -c polar_dma_members_info=\"localhost:$pg_bld_port@1\""
fi
fi
# init the config of the replica
if [[ $rep_dir_specified == "no" ]] && [[ -n $pg_bld_replica_dir ]] && [[ $withrep == "yes" ]];
then
rm -fr $pg_bld_replica_dir
cp -frp $pg_bld_master_dir $pg_bld_replica_dir
echo "polar_hostid = 2" >> $pg_bld_replica_dir/postgresql.conf
echo "synchronous_standby_names='replica1'" >> $pg_bld_master_dir/postgresql.conf
echo "primary_conninfo = 'host=localhost port=$pg_bld_port user=$pg_db_user dbname=postgres application_name=replica1'" >> $pg_bld_replica_dir/recovery.conf
echo "polar_replica = on" >> $pg_bld_replica_dir/recovery.conf
echo "recovery_target_timeline = 'latest'" >> $pg_bld_replica_dir/recovery.conf
echo "primary_slot_name = 'replica1'" >> $pg_bld_replica_dir/recovery.conf
if [[ -n $extra_conf ]];
then
if [[ -f $extra_conf ]];
then
cat $extra_conf >> $pg_bld_replica_dir/postgresql.conf
echo >> $pg_bld_replica_dir/postgresql.conf
else
echo "invalid extra conf file : $extra_conf";
exit 1
fi
fi
fi
# init the config of the standby
if [[ $standby_dir_specified == "no" ]] && [[ -n $pg_bld_standby_dir ]] && [[ $withstandby == "yes" ]];
then
rm -fr $pg_bld_standby_dir
cp -frp $pg_bld_master_dir $pg_bld_standby_dir
echo "polar_hostid = 3" >> $pg_bld_standby_dir/postgresql.conf
if [[ $enable_flashback_log == "on" ]];
then
echo "polar_enable_flashback_log = on
polar_enable_lazy_checkpoint = off" >> $pg_bld_standby_dir/postgresql.conf
fi
echo "primary_conninfo = 'host=localhost port=$pg_bld_port user=$pg_db_user dbname=postgres application_name=standby1'" >> $pg_bld_standby_dir/recovery.conf
echo "standby_mode = on" >> $pg_bld_standby_dir/recovery.conf
echo "recovery_target_timeline = 'latest'" >> $pg_bld_standby_dir/recovery.conf
echo "primary_slot_name = 'standby1'" >> $pg_bld_standby_dir/recovery.conf
if [[ -n $extra_conf ]];
then
if [[ -f $extra_conf ]];
then
cat $extra_conf >> $pg_bld_standby_dir/postgresql.conf
echo >> $pg_bld_standby_dir/postgresql.conf
else
echo "invalid extra conf file : $extra_conf";
exit 1
fi
fi
fi
if [[ $noinit == "no" ]];
then
# start master
echo "port = ${pg_bld_port}
polar_hostid = 100
full_page_writes = off" >> $pg_bld_master_dir/postgresql.conf
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_master_dir start -w -c"
fi
# create a replication_slot and start
if [[ $withrep == "yes" ]];
then
for i in $(seq 1 $repnum)
do
if [[ $repnum == "1" ]];
then
pg_bld_replica_dir_n=${pg_bld_replica_dir}
else
pg_bld_replica_dir_n=${pg_bld_replica_dir}${i}
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_replica_dir_n stop -m i || true"
rm -fr $pg_bld_replica_dir_n
cp -frp $pg_bld_replica_dir $pg_bld_replica_dir_n
fi
pg_bld_rep_port_n=$(($pg_bld_rep_port+$i-1))
echo "port = $pg_bld_rep_port_n
polar_hostid = $i" >> $pg_bld_replica_dir_n/postgresql.conf
if [[ -n $extra_conf ]];
then
if [[ -f $extra_conf ]];
then
if [[ $pg_bld_replica_dir_n != ${pg_bld_replica_dir} ]]
then
cat $extra_conf >> $pg_bld_replica_dir_n/postgresql.conf
echo >> $pg_bld_replica_dir_n/postgresql.conf
fi
else
echo "invalid extra conf file : $extra_conf";
exit 1
fi
fi
echo "primary_conninfo = 'host=localhost port=$pg_bld_port user=$pg_db_user dbname=postgres application_name=replica${i}'" >> $pg_bld_replica_dir_n/recovery.conf
echo "primary_slot_name = 'replica${i}'" >> $pg_bld_replica_dir_n/recovery.conf
# su_eval "env $pg_bld_basedir/bin/psql -h 127.0.0.1 -d postgres -U $pg_db_user -c \"SELECT * FROM pg_create_physical_replication_slot('replica${i}')\""
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_replica_dir_n start -w -c"
done
fi
if [[ $dma == "on" ]];
then
while true
do
$pg_bld_basedir/bin/psql -h 127.0.0.1 -U $pg_db_user -p $pg_bld_port -d postgres -c "do \$\$
DECLARE
counter integer := 60;
BEGIN
LOOP
PERFORM 1 where pg_is_in_recovery() = false;
IF FOUND THEN
RAISE NOTICE 'became leader';
EXIT;
END IF;
PERFORM pg_sleep(1);
counter := counter - 1;
IF counter = 0 THEN
RAISE EXCEPTION 'became leader timeout';
EXIT;
END IF;
END LOOP;
END\$\$;"
if [ $? -eq 0 ]; then
break;
else
echo "retry in 1 second .........."
sleep 1
fi
done
fi
if [[ -n $extra_conf ]];
then
if [[ -f $extra_conf ]];
then
cat $extra_conf >> $pg_bld_master_dir/postgresql.conf
echo >> $pg_bld_master_dir/postgresql.conf
else
echo "invalid extra conf file : $extra_conf";
exit 1
fi
fi
if [[ $noinit == "no" ]];
then
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_master_dir reload"
fi
# create a replication_slot and user $pg_bld_user
if [[ $withrep == "yes" ]];
then
for i in $(seq 1 $repnum)
do
su_eval "env $pg_bld_basedir/bin/psql -h 127.0.0.1 -d postgres -p $pg_bld_port -U $pg_db_user -c \"SELECT * FROM pg_create_physical_replication_slot('replica${i}')\""
sleep 2
done
fi
# build standby data dir, then start standby
if [[ $withstandby == "yes" ]];
then
su_eval "env $pg_bld_basedir/bin/psql -h 127.0.0.1 -d postgres -p $pg_bld_port -U $pg_db_user -c \"SELECT * FROM pg_create_physical_replication_slot('standby1')\""
sleep 2
rm -fr $pg_bld_standby_data_dir
cp -frp $pg_bld_data_dir $pg_bld_standby_data_dir
sed -i -E "s/${pg_bld_data_dir//\//\\/}/${pg_bld_standby_data_dir//\//\\/}/" $pg_bld_standby_dir/postgresql.conf
su_eval "$pg_bld_basedir/bin/pg_ctl -D $pg_bld_standby_dir start -w -c -o '-p $pg_bld_standby_port'"
fi
if [[ $noinit == "no" ]];
then
echo "Following command can be used to connect to PG:"
echo ""
echo su $pg_bld_user -c \"$pg_bld_basedir/bin/psql -h 127.0.0.1 -p $pg_bld_port postgres\"
echo su $pg_bld_user -c \"$pg_bld_basedir/bin/psql -h 127.0.0.1 -p $pg_bld_rep_port postgres\"
echo su $pg_bld_user -c \"$pg_bld_basedir/bin/psql -h 127.0.0.1 -p $pg_bld_standby_port postgres\"
echo ""
fi
######################### PHASE 6 Test: test for polar ###########################
if [[ $initpx == "yes" ]];
then
px_init
fi
if [[ $regress_test == "on" ]] && [[ $regress_test_quick == "off" ]];
then
# some cases describe using "polar-ignore" when "make polar-installcheck", for polar model
su_eval "make installcheck 2>&1" | tee -a testlog
if [ -e ./src/test/regress/regression.diffs ]
then
cat ./src/test/regress/regression.diffs
cp -frp ./src/test/regress/regression.diffs ./src/test/regress/regression_installcheck.diffs
fi
if [[ $initpx == "yes" ]];
then
su_eval "make polar-installcheck-px 2>&1" | tee -a testlog
else
su_eval "make polar-installcheck 2>&1" | tee -a testlog
fi
if [ -e ./src/test/regress/regression.diffs ]
then
cat ./src/test/regress/regression.diffs
cp -frp ./src/test/regress/regression.diffs ./src/test/regress/regression_polar.diffs
fi
if [[ -e ./src/test/regress/regression_installcheck.diffs ]] || [[ -e ./src/test/regress/regression_polar.diffs ]]
then
exit 1
fi
fi
if [[ $extension_test == "on" ]];
then
if [[ $contrib_test == "on" ]];
then
# make installcheck the extensions
set +x
echo "============================"
echo "Check the contrib extensions:"
echo "============================"
supported_extensions=`su_eval "$pg_bld_basedir/bin/psql -h 127.0.0.1 -d postgres -U $pg_db_user -c 'show polar_supported_extensions;' -t"`
extension_arr=(${supported_extensions//+/ })
contrib_extension_array=`ls contrib/`
for i in ${extension_arr[@]};
do
if [[ $i =~ plperl || $i =~ plpgsql || $i =~ pltcl ]] ; then
echo "extensions $i in src/pl has checked already."
elif [[ $i =~ citext || $i =~ btree_gin || $i =~ btree_gist ]] ; then
: # intentionally empty statement
su_eval "make -C contrib/$i installcheck 2>&1 " | tee -a testlog
elif [[ "${contrib_extension_array[@]}" =~ $i ]] ; then
su_eval "make -C contrib/$i installcheck 2>&1 " | tee -a testlog
elif [[ $i =~ "uuid-ossp" ]] ; then
su_eval "make -C contrib/uuid-ossp installcheck 2>&1 " | tee -a testlog
fi
done
set -x
fi
if [[ $pl_test == "on" ]];
then
set +x
echo "============================"
echo "Check the pl extensions:"
echo "============================"
: # intentionally empty statement
su_eval "make -C src/pl installcheck 2>&1 " | tee -a testlog
set -x
fi
if [[ $external_test == "on" ]];
then
set +x
echo "============================"
echo "Check the external extensions:"
echo "============================"
su_eval "make -C external installcheck 2>&1 " | tee -a testlog
set -x
fi
check_failed_case
fi
if [[ $coverage == "on" ]];
then
del_cov
fi
if [[ $make_installcheck_world == "on" ]] && [[ $regress_test_quick == "off" ]];
then
export PGHOST=$pg_bld_master_dir
su_eval "make installcheck-world PG_TEST_EXTRA='kerberos ldap ssl' 2>&1" | tee -a testlog
su_eval "make polar-installcheck 2>&1" | tee -a testlog
check_failed_case "installcheck-world"
unset PGHOST
fi
if [[ $regress_test_ng == "on" ]]
then
echo "regress_test_ng begin"
if [[ $regress_test_quick == "off" ]];
then
su_eval "make -C src/test installcheck 2>&1" | tee -a testlog
su_eval "make -C src/test/regress polar-installcheck 2>&1" | tee -a testlog
check_failed_case
else
killall postgres || true
killall polar-postgres || true
unset COPT
su_eval "make $dma_regress_opt -C src/test check 2>&1" | tee -a testlog
su_eval "make $dma_regress_opt -C src/test/regress polar-check 2>&1" | tee -a testlog
check_failed_case
fi
echo "regress_test_ng end"
fi
echo "rds build done"
exit 0
Reference in New Issue View Git Blame Copy Permalink