Migrating databases to the control plane

export PULL_OPENSTACK_CONFIGURATION_DATABASES=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \
    mysql -rsh "$SOURCE_MARIADB_IP" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;')
echo "$PULL_OPENSTACK_CONFIGURATION_DATABASES"

export PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \
    mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root -p"$SOURCE_DB_ROOT_PASSWORD" | grep -v OK)
echo "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK"

export PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \
    mysql -rsh "${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" nova_api -e \
    'select uuid,name,transport_url,database_connection,disabled from cell_mappings;')
echo "$PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS"

export PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \
    mysql -rsh "$SOURCE_MARIADB_IP" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" nova_api -e \
    "select host from nova.services where services.binary='nova-compute' and deleted=0;")
echo "$PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES"

export PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS=$($CONTROLLER1_SSH sudo podman exec -it nova_api nova-manage cell_v2 list_cells)
echo "$PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS"

cat >~/.source_cloud_exported_variables << EOF
PULL_OPENSTACK_CONFIGURATION_DATABASES="$PULL_OPENSTACK_CONFIGURATION_DATABASES"
PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK="$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK"
PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS="$PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS"
PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES="$PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES"
PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS="$PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS"
SRIOV_AGENTS="$SRIOV_AGENTS"
EOF

oc project openstack

cd /home/lab-user/labrepo/content/files
oc apply -f osp-ng-ctlplane-secret.yaml

oc set data secret/osp-secret "AdminPassword=$ADMIN_PASSWORD"

oc set data secret/osp-secret "AodhPassword=$AODH_PASSWORD"
oc set data secret/osp-secret "BarbicanPassword=$BARBICAN_PASSWORD"
oc set data secret/osp-secret "CeilometerPassword=$CEILOMETER_PASSWORD"
oc set data secret/osp-secret "CinderPassword=$CINDER_PASSWORD"
oc set data secret/osp-secret "GlancePassword=$GLANCE_PASSWORD"
oc set data secret/osp-secret "HeatAuthEncryptionKey=$HEAT_AUTH_ENCRYPTION_KEY"
oc set data secret/osp-secret "HeatPassword=$HEAT_PASSWORD"
oc set data secret/osp-secret "HeatStackDomainAdminPassword=$HEAT_STACK_DOMAIN_ADMIN_PASSWORD"
oc set data secret/osp-secret "IronicPassword=$IRONIC_PASSWORD"
oc set data secret/osp-secret "IronicInspectorPassword=$IRONIC_PASSWORD"
oc set data secret/osp-secret "ManilaPassword=$MANILA_PASSWORD"
oc set data secret/osp-secret "MetadataSecret=$METADATA_SECRET"
oc set data secret/osp-secret "NeutronPassword=$NEUTRON_PASSWORD"
oc set data secret/osp-secret "NovaPassword=$NOVA_PASSWORD"
oc set data secret/osp-secret "OctaviaPassword=$OCTAVIA_PASSWORD"
oc set data secret/osp-secret "PlacementPassword=$PLACEMENT_PASSWORD"
oc set data secret/osp-secret "SwiftPassword=$SWIFT_PASSWORD"

cd /home/lab-user/labrepo/content/files
oc apply -f osp-ng-ctlplane-deploy-backend.yaml

# Update the services list to be stopped
ServicesToStop=("tripleo_aodh_api.service"
                "tripleo_aodh_api_cron.service"
                "tripleo_aodh_evaluator.service"
                "tripleo_aodh_listener.service"
                "tripleo_aodh_notifier.service"
                "tripleo_ceilometer_agent_central.service"
                "tripleo_ceilometer_agent_notification.service"
                "tripleo_horizon.service"
                "tripleo_keystone.service"
                "tripleo_barbican_api.service"
                "tripleo_barbican_worker.service"
                "tripleo_barbican_keystone_listener.service"
                "tripleo_cinder_api.service"
                "tripleo_cinder_api_cron.service"
                "tripleo_cinder_scheduler.service"
                "tripleo_cinder_volume.service"
                "tripleo_cinder_backup.service"
                "tripleo_collectd.service"
                "tripleo_glance_api.service"
                "tripleo_gnocchi_api.service"
                "tripleo_gnocchi_metricd.service"
                "tripleo_gnocchi_statsd.service"
                "tripleo_manila_api.service"
                "tripleo_manila_api_cron.service"
                "tripleo_manila_scheduler.service"
                "tripleo_neutron_api.service"
                "tripleo_placement_api.service"
                "tripleo_nova_api_cron.service"
                "tripleo_nova_api.service"
                "tripleo_nova_conductor.service"
                "tripleo_nova_metadata.service"
                "tripleo_nova_scheduler.service"
                "tripleo_nova_vnc_proxy.service"
                "tripleo_aodh_api.service"
                "tripleo_aodh_api_cron.service"
                "tripleo_aodh_evaluator.service"
                "tripleo_aodh_listener.service"
                "tripleo_aodh_notifier.service"
                "tripleo_ceilometer_agent_central.service"
                "tripleo_ceilometer_agent_compute.service"
                "tripleo_ceilometer_agent_ipmi.service"
                "tripleo_ceilometer_agent_notification.service"
                "tripleo_ovn_cluster_northd.service"
                "tripleo_ironic_neutron_agent.service"
                "tripleo_ironic_api.service"
                "tripleo_ironic_inspector.service"
                "tripleo_ironic_conductor.service")

PacemakerResourcesToStop=("openstack-cinder-volume"
                          "openstack-cinder-backup"
                          "openstack-manila-share")

echo "Stopping systemd OpenStack services"
for service in ${ServicesToStop[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            echo "Stopping the $service in controller $i"
            if ${!SSH_CMD} sudo systemctl is-active $service; then
                ${!SSH_CMD} sudo systemctl stop $service
            fi
        fi
    done
done

echo "Checking systemd OpenStack services"
for service in ${ServicesToStop[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            if ! ${!SSH_CMD} systemctl show $service | grep ActiveState=inactive >/dev/null; then
                echo "ERROR: Service $service still running on controller $i"
            else
                echo "OK: Service $service is not running on controller $i"
            fi
        fi
    done
done

echo "Stopping pacemaker OpenStack services"
for i in {1..3}; do
    SSH_CMD=CONTROLLER${i}_SSH
    if [ ! -z "${!SSH_CMD}" ]; then
        echo "Using controller $i to run pacemaker commands"
        for resource in ${PacemakerResourcesToStop[*]}; do
            if ${!SSH_CMD} sudo pcs resource config $resource &>/dev/null; then
                echo "Stopping $resource"
                ${!SSH_CMD} sudo pcs resource disable $resource
            else
                echo "Service $resource not present"
            fi
        done
        break
    fi
done

echo "Checking pacemaker OpenStack services"
for i in {1..3}; do
    SSH_CMD=CONTROLLER${i}_SSH
    if [ ! -z "${!SSH_CMD}" ]; then
        echo "Using controller $i to run pacemaker commands"
        for resource in ${PacemakerResourcesToStop[*]}; do
            if ${!SSH_CMD} sudo pcs resource config $resource &>/dev/null; then
                if ! ${!SSH_CMD} sudo pcs resource status $resource | grep Started; then
                    echo "OK: Service $resource is stopped"
                else
                    echo "ERROR: Service $resource is started"
                fi
            fi
        done
        break
    fi
done

oc rsh mariadb-copy-data mysql -h "${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" -e "SHOW databases;"

. ~/.source_cloud_exported_variables
test -z "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK"  || [ "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" = " " ] && echo "OK" || echo "CHECK FAILED"

oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
    mysql -rsh "$PODIFIED_MARIADB_IP" -uroot -p"$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;'
oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
    mysql -rsh "$PODIFIED_CELL1_MARIADB_IP" -uroot -p"$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;'

oc rsh mariadb-copy-data << EOF
  mysql -h"${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" \
  -N -e "show databases" | grep -E -v "schema|mysql|gnocchi|aodh" | \
  while read dbname; do
    echo "Dumping \${dbname}";
    mysqldump -h"${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" \
      --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \
      "\${dbname}" > /backup/"\${dbname}".sql;
   done
EOF

oc rsh mariadb-copy-data << EOF
  # db schemas to rename on import
  declare -A db_name_map
  db_name_map['nova']='nova_cell1'
  db_name_map['ovs_neutron']='neutron'
  db_name_map['ironic-inspector']='ironic_inspector'

  # db servers to import into
  declare -A db_server_map
  db_server_map['default']=${PODIFIED_MARIADB_IP}
  db_server_map['nova_cell1']=${PODIFIED_CELL1_MARIADB_IP}

  # db server root password map
  declare -A db_server_password_map
  db_server_password_map['default']=${PODIFIED_DB_ROOT_PASSWORD}
  db_server_password_map['nova_cell1']=${PODIFIED_DB_ROOT_PASSWORD}

  cd /backup
  for db_file in \$(ls *.sql); do
    db_name=\$(echo \${db_file} | awk -F'.' '{ print \$1; }')
    if [[ -v "db_name_map[\${db_name}]" ]]; then
      echo "renaming \${db_name} to \${db_name_map[\${db_name}]}"
      db_name=\${db_name_map[\${db_name}]}
    fi
    db_server=\${db_server_map["default"]}
    if [[ -v "db_server_map[\${db_name}]" ]]; then
      db_server=\${db_server_map[\${db_name}]}
    fi
    db_password=\${db_server_password_map['default']}
    if [[ -v "db_server_password_map[\${db_name}]" ]]; then
      db_password=\${db_server_password_map[\${db_name}]}
    fi
    echo "creating \${db_name} in \${db_server}"
    mysql -h"\${db_server}" -uroot "-p\${db_password}" -e \
      "CREATE DATABASE IF NOT EXISTS \${db_name} DEFAULT \
      CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION};"
    echo "importing \${db_name} into \${db_server}"
    mysql -h "\${db_server}" -uroot "-p\${db_password}" "\${db_name}" < "\${db_file}"
  done

  mysql -h "\${db_server_map['default']}" -uroot -p"\${db_server_password_map['default']}" -e \
    "update nova_api.cell_mappings set name='cell1' where name='default';"
  mysql -h "\${db_server_map['nova_cell1']}" -uroot -p"\${db_server_password_map['nova_cell1']}" -e \
    "delete from nova_cell1.services where host not like '%nova-cell1-%' and services.binary != 'nova-compute';"
EOF

. ~/.source_cloud_exported_variables

# use 'oc exec' and 'mysql -rs' to maintain formatting
dbs=$(oc exec openstack-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;')
echo $dbs | grep -Eq '\bkeystone\b' && echo "OK" || echo "CHECK FAILED"

# ensure neutron db is renamed from ovs_neutron
echo $dbs | grep -Eq '\bneutron\b'
echo $PULL_OPENSTACK_CONFIGURATION_DATABASES | grep -Eq '\bovs_neutron\b' && echo "OK" || echo "CHECK FAILED"

# ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1
c1dbs=$(oc exec openstack-cell1-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;')
echo $c1dbs | grep -Eq '\bnova_cell1\b' && echo "OK" || echo "CHECK FAILED"

# ensure default cell renamed to cell1, and the cell UUIDs retained intact
novadb_mapped_cells=$(oc exec openstack-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \
  nova_api -e 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;')
uuidf='\S{8,}-\S{4,}-\S{4,}-\S{4,}-\S{12,}'
left_behind=$(comm -23 \
  <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \
  <(echo $novadb_mapped_cells | tr -s "| " " " | grep -oE " $uuidf \S+"))
changed=$(comm -13 \
  <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \
  <(echo $novadb_mapped_cells | tr -s "| " " " | grep -oE " $uuidf \S+"))
test $(grep -Ec ' \S+$' <<<$left_behind) -eq 1 && echo "OK" || echo "CHECK FAILED"
default=$(grep -E ' default$' <<<$left_behind)
test $(grep -Ec ' \S+$' <<<$changed) -eq 1 && echo "OK" || echo "CHECK FAILED"
grep -qE " $(awk '{print $1}' <<<$default) cell1$" <<<$changed && echo "OK" || echo "CHECK FAILED"

# ensure the registered Compute service name has not changed
novadb_svc_records=$(oc exec openstack-cell1-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \
  nova_cell1 -e "select host from services where services.binary='nova-compute' and deleted=0 order by host asc;")
diff -Z <(echo $novadb_svc_records) <(echo $PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES) && echo "OK" || echo "CHECK FAILED"

oc delete pod mariadb-copy-data
oc delete pvc mariadb-data

oc apply -f - <<EOF
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: ovn-data-cert
  namespace: openstack
spec:
  commonName: ovn-data-cert
  secretName: ovn-data-cert
  issuerRef:
    name: rootca-internal
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: ovn-data
spec:
  storageClassName: $STORAGE_CLASS
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
---
apiVersion: v1
kind: Pod
metadata:
  name: ovn-copy-data
  annotations:
    openshift.io/scc: anyuid
    k8s.v1.cni.cncf.io/networks: internalapi
  labels:
    app: adoption
spec:
  containers:
  - image: $OVSDB_IMAGE
    command: [ "sh", "-c", "sleep infinity"]
    name: adoption
    volumeMounts:
    - mountPath: /backup
      name: ovn-data
    - mountPath: /etc/pki/tls/misc
      name: ovn-data-cert
      readOnly: true
  securityContext:
    allowPrivilegeEscalation: false
    capabilities:
      drop: ALL
    runAsNonRoot: true
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - name: ovn-data
    persistentVolumeClaim:
      claimName: ovn-data
  - name: ovn-data-cert
    secret:
      secretName: ovn-data-cert
EOF

oc wait --for=condition=Ready pod/ovn-copy-data --timeout=30s

oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  ovn:
    enabled: true
    template:
      ovnDBCluster:
        ovndbcluster-nb:
          replicas: 3
          dbType: NB
          storageRequest: 10G
          networkAttachment: internalapi
        ovndbcluster-sb:
          replicas: 3
          dbType: SB
          storageRequest: 10G
          networkAttachment: internalapi
      ovnNorthd:
        replicas: 3
      ovnController:
        networkAttachment: tenant
        nodeSelector:
'

oc wait --for=jsonpath='{.status.phase}'=Running pod --selector=service=ovsdbserver-nb
oc wait --for=jsonpath='{.status.phase}'=Running pod --selector=service=ovsdbserver-sb

PODIFIED_OVSDB_NB_IP=$(oc get svc --selector "statefulset.kubernetes.io/pod-name=ovsdbserver-nb-0" -ojsonpath='{.items[0].spec.clusterIP}')
PODIFIED_OVSDB_SB_IP=$(oc get svc --selector "statefulset.kubernetes.io/pod-name=ovsdbserver-sb-0" -ojsonpath='{.items[0].spec.clusterIP}')

oc exec -it ovsdbserver-nb-0 -- ovn-nbctl show
oc exec -it ovsdbserver-sb-0 -- ovn-sbctl list Chassis

oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  ovn:
    enabled: true
    template:
      ovnNorthd:
        replicas: 1
'

oc patch openstackcontrolplane openstack --type=merge --patch '
spec:
  ovn:
    enabled: true
    template:
      ovnController:
        networkAttachment: tenant
'

oc delete --ignore-not-found=true pod ovn-copy-data
oc delete --ignore-not-found=true pvc ovn-data

ServicesToStop=("tripleo_ovn_cluster_north_db_server.service"
                "tripleo_ovn_cluster_south_db_server.service")

echo "Stopping systemd OpenStack services"
for service in ${ServicesToStop[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            echo "Stopping the $service in controller $i"
            if ${!SSH_CMD} sudo systemctl is-active $service; then
                ${!SSH_CMD} sudo systemctl stop $service
            fi
        fi
    done
done

echo "Checking systemd OpenStack services"
for service in ${ServicesToStop[*]}; do
    for i in {1..3}; do
        SSH_CMD=CONTROLLER${i}_SSH
        if [ ! -z "${!SSH_CMD}" ]; then
            if ! ${!SSH_CMD} systemctl show $service | grep ActiveState=inactive >/dev/null; then
                echo "ERROR: Service $service still running on controller $i"
            else
                echo "OK: Service $service is not running on controller $i"
            fi
        fi
    done
done