406 lines
11 KiB
Go
406 lines
11 KiB
Go
/*
|
|
Copyright 2022 Alibaba Group Holding Limited.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package knobs
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/go-logr/logr"
|
|
corev1 "k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
|
|
polardbxv1 "github.com/alibaba/polardbx-operator/api/v1"
|
|
k8shelper "github.com/alibaba/polardbx-operator/pkg/k8s/helper"
|
|
"github.com/alibaba/polardbx-operator/pkg/meta/core/gms"
|
|
"github.com/alibaba/polardbx-operator/pkg/operator/v1/polardbx/convention"
|
|
xstoreconvention "github.com/alibaba/polardbx-operator/pkg/operator/v1/xstore/convention"
|
|
dbutil "github.com/alibaba/polardbx-operator/pkg/util/database"
|
|
"github.com/alibaba/polardbx-operator/pkg/webhook/extension"
|
|
)
|
|
|
|
type Sync struct {
|
|
client.Reader
|
|
logr.Logger
|
|
}
|
|
|
|
func (s *Sync) openGMS(ctx context.Context, polardbx *polardbxv1.PolarDBXCluster) (*sql.DB, error) {
|
|
gmsName := convention.NewGMSName(polardbx)
|
|
if polardbx.Spec.ShareGMS {
|
|
gmsName = convention.NewDNName(polardbx, 0)
|
|
}
|
|
var xstoreGms polardbxv1.XStore
|
|
if err := s.Get(ctx, types.NamespacedName{
|
|
Namespace: polardbx.Namespace,
|
|
Name: gmsName,
|
|
}, &xstoreGms); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var gmsSecret corev1.Secret
|
|
if err := s.Get(ctx, types.NamespacedName{
|
|
Namespace: polardbx.Namespace,
|
|
Name: xstoreconvention.NewSecretName(&xstoreGms),
|
|
}, &gmsSecret); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
gmsSvcName := xstoreconvention.NewServiceName(&xstoreGms, xstoreconvention.ServiceTypeReadWrite)
|
|
var gmsSvc corev1.Service
|
|
if err := s.Get(ctx, types.NamespacedName{
|
|
Namespace: polardbx.Namespace,
|
|
Name: gmsSvcName,
|
|
}, &gmsSvc); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return dbutil.OpenMySQLDB(&dbutil.MySQLDataSource{
|
|
Host: k8shelper.GetServiceDNSRecordWithSvc(&gmsSvc, false),
|
|
Port: int(k8shelper.MustGetPortFromService(&gmsSvc, xstoreconvention.PortAccess).Port),
|
|
Username: xstoreconvention.SuperAccount,
|
|
Password: string(gmsSecret.Data[xstoreconvention.SuperAccount]),
|
|
Database: gms.MetaDBName,
|
|
Timeout: 2 * time.Second,
|
|
SSL: "",
|
|
})
|
|
}
|
|
|
|
func (s *Sync) operateGmsInTransaction(ctx context.Context, polardbx *polardbxv1.PolarDBXCluster, op func(ctx context.Context, tx *sql.Tx) error) error {
|
|
// Open the DB.
|
|
db, err := s.openGMS(ctx, polardbx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
|
|
// Begin a transaction.
|
|
tx, err := db.BeginTx(ctx, &sql.TxOptions{
|
|
Isolation: sql.LevelRepeatableRead,
|
|
ReadOnly: false,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Execute the operation.
|
|
if err := op(ctx, tx); err != nil {
|
|
_ = tx.Rollback()
|
|
return err
|
|
}
|
|
|
|
// Commit.
|
|
return tx.Commit()
|
|
}
|
|
|
|
const clConfigDataIdFormat = "polardbx.inst.config.%s" // {inst_id}
|
|
|
|
func autoConvertToIntStr(val string) intstr.IntOrString {
|
|
intVal, err := strconv.Atoi(val)
|
|
if err == nil && (intVal <= math.MaxInt32 && intVal >= math.MinInt32) {
|
|
return intstr.FromInt(intVal)
|
|
} else {
|
|
return intstr.FromString(val)
|
|
}
|
|
}
|
|
|
|
//goland:noinspection SqlDialectInspection,SqlNoDataSourceInspection,SqlResolve
|
|
func loadKnobsWithinTx(ctx context.Context, tx *sql.Tx, knobs *polardbxv1.PolarDBXClusterKnobs) error {
|
|
// Query the current version.
|
|
var version int64
|
|
row := tx.QueryRowContext(ctx, "SELECT op_version FROM config_listener WHERE data_id = ?",
|
|
fmt.Sprintf(clConfigDataIdFormat, knobs.Spec.ClusterName))
|
|
|
|
if err := row.Scan(&version); err != nil {
|
|
return err
|
|
}
|
|
knobs.Status.Version = version
|
|
|
|
// Load the knobs if force or the recorded version is outdated.
|
|
{
|
|
rs, err := tx.QueryContext(ctx, "SELECT param_key, param_val FROM inst_config WHERE inst_id = ?",
|
|
knobs.Spec.ClusterName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rs.Close()
|
|
|
|
// Scan the rows.
|
|
knobs.Spec.Knobs = make(map[string]intstr.IntOrString)
|
|
var key, val string
|
|
for rs.Next() {
|
|
if err := rs.Scan(&key, &val); err != nil {
|
|
return err
|
|
}
|
|
|
|
knobs.Spec.Knobs[key] = autoConvertToIntStr(val)
|
|
}
|
|
}
|
|
|
|
// Update the update time.
|
|
knobs.Status.LastUpdated = metav1.Now()
|
|
knobs.Status.Size = int32(len(knobs.Spec.Knobs))
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *Sync) MutateOnCreate(ctx context.Context, obj runtime.Object) error {
|
|
knobs := obj.(*polardbxv1.PolarDBXClusterKnobs)
|
|
|
|
if knobs.Status.LastUpdated.IsZero() {
|
|
knobs.Status.Version = -1
|
|
knobs.Status.LastUpdated = metav1.Now()
|
|
}
|
|
|
|
// Do nothing if there's no cluster name specified.
|
|
if len(knobs.Spec.ClusterName) == 0 {
|
|
return nil
|
|
}
|
|
|
|
var polardbxcluster polardbxv1.PolarDBXCluster
|
|
if err := s.Get(ctx, types.NamespacedName{
|
|
Namespace: knobs.Namespace,
|
|
Name: knobs.Spec.ClusterName,
|
|
}, &polardbxcluster); err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
s.Logger.Info("Specified polardbxcluster not exists.", "polardbxcluster", knobs.Spec.ClusterName)
|
|
} else {
|
|
s.Logger.Error(err, "Failed to get polardbxcluster.")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Load the initial knobs.
|
|
if err := s.operateGmsInTransaction(ctx, &polardbxcluster,
|
|
func(ctx context.Context, tx *sql.Tx) error {
|
|
return loadKnobsWithinTx(ctx, tx, knobs)
|
|
},
|
|
); err != nil {
|
|
// Failed to load, setup with the default.
|
|
s.Logger.Error(err, "Failed to load knobs within transaction, set with default.")
|
|
|
|
knobs.Status.Version = -1
|
|
knobs.Spec.Knobs = nil
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type operation string
|
|
|
|
const (
|
|
opAdd operation = "Add"
|
|
opDelete operation = "Delete"
|
|
opUpdate operation = "Update"
|
|
)
|
|
|
|
type action struct {
|
|
Key string
|
|
Op operation
|
|
Val string
|
|
OldVal string
|
|
}
|
|
|
|
func newUpdateAction(key string, oldVal, newVal fmt.Stringer) action {
|
|
return action{Key: key, Op: opUpdate, Val: newVal.String(), OldVal: oldVal.String()}
|
|
}
|
|
|
|
func newAddAction(key string, val fmt.Stringer) action {
|
|
return action{Key: key, Op: opAdd, Val: val.String()}
|
|
}
|
|
|
|
func newDeleteAction(key string) action {
|
|
return action{Key: key, Op: opDelete}
|
|
}
|
|
|
|
//goland:noinspection SqlDialectInspection,SqlResolve,SqlNoDataSourceInspection
|
|
const (
|
|
insertOrUpdateConfigSQL = "INSERT INTO inst_config (inst_id, param_key, param_val) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE param_val = VALUES(param_val), gmt_modified = NOW()"
|
|
deleteConfigSQL = "DELETE FROM inst_config WHERE inst_id = ? AND param_key = ?"
|
|
)
|
|
|
|
func (act *action) IsDelete() bool {
|
|
return act.Op == opDelete
|
|
}
|
|
|
|
func (act *action) Apply(knobs map[string]intstr.IntOrString) bool {
|
|
switch act.Op {
|
|
case opAdd:
|
|
val, exists := knobs[act.Key]
|
|
if exists {
|
|
// Convert to update if already exists and value not equals.
|
|
if val.String() != act.Val {
|
|
knobs[act.Key] = autoConvertToIntStr(act.Val)
|
|
return true
|
|
}
|
|
} else {
|
|
knobs[act.Key] = autoConvertToIntStr(act.Val)
|
|
return true
|
|
}
|
|
case opUpdate:
|
|
val, exists := knobs[act.Key]
|
|
if !exists {
|
|
// Convert to add if not exists.
|
|
knobs[act.Key] = autoConvertToIntStr(act.Val)
|
|
return true
|
|
} else {
|
|
if val.String() != act.Val {
|
|
knobs[act.Key] = autoConvertToIntStr(act.Val)
|
|
return true
|
|
}
|
|
}
|
|
case opDelete:
|
|
_, exists := knobs[act.Key]
|
|
if exists {
|
|
delete(knobs, act.Key)
|
|
return true
|
|
}
|
|
default:
|
|
panic("unknown operation: " + act.Op)
|
|
}
|
|
return false
|
|
}
|
|
|
|
func diffKnobs(oldKnobs, newKnobs map[string]intstr.IntOrString) []action {
|
|
diff := make([]action, 0)
|
|
|
|
for key, val := range newKnobs {
|
|
if oldVal, exists := oldKnobs[key]; exists {
|
|
// Update on value change.
|
|
if val.String() != oldVal.String() {
|
|
diff = append(diff, newUpdateAction(key, &oldVal, &val))
|
|
}
|
|
} else {
|
|
diff = append(diff, newAddAction(key, &val))
|
|
}
|
|
}
|
|
|
|
for key := range oldKnobs {
|
|
if _, exists := newKnobs[key]; !exists {
|
|
diff = append(diff, newDeleteAction(key))
|
|
}
|
|
}
|
|
|
|
return diff
|
|
}
|
|
|
|
//goland:noinspection SqlDialectInspection,SqlNoDataSourceInspection,SqlResolve
|
|
func (s *Sync) MutateOnUpdate(ctx context.Context, oldObj, newObj runtime.Object) error {
|
|
knobs := newObj.(*polardbxv1.PolarDBXClusterKnobs)
|
|
|
|
// Do nothing if there's no cluster name specified.
|
|
if len(knobs.Spec.ClusterName) == 0 {
|
|
return nil
|
|
}
|
|
|
|
oldKnobs := oldObj.(*polardbxv1.PolarDBXClusterKnobs)
|
|
|
|
// Calculate the difference.
|
|
knobsDiff := diffKnobs(oldKnobs.Spec.Knobs, knobs.Spec.Knobs)
|
|
|
|
var polardbxcluster polardbxv1.PolarDBXCluster
|
|
if err := s.Get(ctx, types.NamespacedName{
|
|
Namespace: knobs.Namespace,
|
|
Name: knobs.Spec.ClusterName,
|
|
}, &polardbxcluster); err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
s.Logger.Info("Specified polardbxcluster not exists, reset.", "polardbxcluster", knobs.Spec.ClusterName)
|
|
// Reset all if not reset.
|
|
if knobs.Status.Version >= 0 {
|
|
knobs.Spec.Knobs = nil
|
|
knobs.Status.LastUpdated = metav1.Now()
|
|
knobs.Status.Version = -1
|
|
knobs.Status.Size = 0
|
|
} else {
|
|
oldKnobs.DeepCopyInto(knobs)
|
|
}
|
|
} else {
|
|
s.Logger.Error(err, "Failed to get polardbxcluster.")
|
|
// Reject the update.
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
if len(knobsDiff) == 0 {
|
|
// If difference is zero, just do another load.
|
|
return s.operateGmsInTransaction(ctx, &polardbxcluster,
|
|
func(ctx context.Context, tx *sql.Tx) error {
|
|
return loadKnobsWithinTx(ctx, tx, knobs)
|
|
},
|
|
)
|
|
} else {
|
|
return s.operateGmsInTransaction(ctx, &polardbxcluster,
|
|
func(ctx context.Context, tx *sql.Tx) error {
|
|
// Load the knobs in GMS.
|
|
if err := loadKnobsWithinTx(ctx, tx, knobs); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Apply the diff.
|
|
for _, act := range knobsDiff {
|
|
if act.Apply(knobs.Spec.Knobs) {
|
|
if act.IsDelete() {
|
|
_, err := tx.ExecContext(ctx, deleteConfigSQL, knobs.Spec.ClusterName, act.Key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
_, err := tx.ExecContext(ctx, insertOrUpdateConfigSQL, knobs.Spec.ClusterName, act.Key, act.Val)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Update the version.
|
|
rs, err := tx.ExecContext(ctx, "UPDATE config_listener SET op_version = op_version + 1 WHERE data_id = ? AND op_version = ?",
|
|
fmt.Sprintf(clConfigDataIdFormat, knobs.Spec.ClusterName), knobs.Status.Version)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if rowsAffected, err := rs.RowsAffected(); err != nil {
|
|
return err
|
|
} else if rowsAffected == 0 {
|
|
return errors.New("knobs has been updated, abort")
|
|
}
|
|
|
|
knobs.Status.Version++
|
|
knobs.Status.LastUpdated = metav1.Now()
|
|
knobs.Status.Size = int32(len(knobs.Spec.Knobs))
|
|
|
|
return nil
|
|
},
|
|
)
|
|
}
|
|
}
|
|
|
|
func NewSync(r client.Reader, logger logr.Logger) extension.CustomMutator {
|
|
return &Sync{Reader: r, Logger: logger}
|
|
}
|