#!/bin/bash
#-----------------------
# Testing neutron-api
#-----------------------
set -e

# Dump diagnostics for a systemd unit, so a failure in the daemon loop below is
# debuggable. A bare "service restart" under "set -e" otherwise aborts the test
# before any logs are captured (the unit journal is not part of the autopkgtest
# artifacts), which hides the actual cause of a start failure.
dump_service() {
    local unit="$1"
    echo "----- systemctl status ${unit} -----"
    systemctl --no-pager --full status "$unit" 2>&1 || true
    echo "----- journalctl -xeu ${unit} (tail) -----"
    journalctl --no-pager -xeu "$unit" 2>&1 | tail -n 200 || true
    echo "----- /var/log/neutron/*.log (tail) -----"
    tail -n 200 /var/log/neutron/*.log 2>/dev/null || true
}

mysql -u root << EOF
DROP DATABASE IF EXISTS neutron;
DROP USER IF EXISTS 'neutron'@'localhost';
DROP USER IF EXISTS 'neutron'@'%';
CREATE DATABASE neutron;
CREATE USER 'neutron'@'localhost' IDENTIFIED BY 'changeme';
CREATE USER 'neutron'@'%'         IDENTIFIED BY 'changeme';
GRANT ALL PRIVILEGES ON neutron.* TO 'neutron'@'localhost';
GRANT ALL PRIVILEGES ON neutron.* TO 'neutron'@'%';
EOF

crudini --set /etc/neutron/neutron.conf database connection mysql+pymysql://neutron:changeme@localhost/neutron
# NOTE(Ubuntu LP: #2150285): exercise the Apache mod_wsgi worker shim. The API
# is served by apache2 in daemon mode with several workers (see
# debian/neutron-api.conf). Enable a geneve type driver with a configured VNI
# range plus the network_segment_range service plugin so that exactly one
# worker is expected to seed a default segment range, and the OVN mechanism
# driver registers one hash-ring node per worker sharing a single start time.
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 type_drivers local,geneve
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 mechanism_drivers ovn
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 tenant_network_types geneve
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2_type_geneve vni_ranges 1:65536
# OVN requires a geneve max_header_size of at least 38; the neutron default of
# 30 makes the OVN mechanism driver abort at startup ("max_header_size set too
# low for OVN").
crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2_type_geneve max_header_size 38
crudini --set /etc/neutron/neutron.conf DEFAULT service_plugins network_segment_range
neutron-db-manage upgrade head

# Permit remote database access
ovn-nbctl set-connection ptcp:6641:0.0.0.0 -- \
          set connection . inactivity_probe=60000
ovn-sbctl set-connection ptcp:6642:0.0.0.0 -- \
          set connection . inactivity_probe=60000

DAEMONS=(
    'apache2'
    'neutron-ovn-maintenance-worker'
    'neutron-periodic-workers'
    'neutron-rpc-server'
)
for daemon in "${DAEMONS[@]}"; do
    echo -n "Checking ${daemon} ... "
    # Use an if-condition so "set -e" does not abort here on a failed restart;
    # capture the unit's journal and neutron logs before exiting instead.
    if ! service "$daemon" restart; then
        echo "ERROR: ${daemon} failed to start"
        dump_service "$daemon"
        exit 1
    fi
    TIMEOUT=50
    while [ "$TIMEOUT" -gt 0 ]; do
        if service "$daemon" status > /dev/null 2>&1; then
            echo "OK"
            break
        fi
        TIMEOUT=$((TIMEOUT - 1))
        sleep 0.5
    done

    if [ "$TIMEOUT" -le 0 ]; then
        echo "ERROR: ${daemon} IS NOT RUNNING"
        dump_service "$daemon"
        exit 1
    fi
done

echo -n "Checking neutron-api endpoint ... "
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:9696/)

if [ "$HTTP_STATUS" -eq 200 ]; then
    echo "OK (received HTTP $HTTP_STATUS)"
else
    echo "ERROR: Unexpected response code ($HTTP_STATUS)"
    exit 1
fi

#---------------------------------------------------------------------------
# LP: #2150285 - keep Apache mod_wsgi operational for one more cycle
#
# The checks below fail on a build where the keep-mod-wsgi-operational patch
# is absent: get_api_worker_id() returns None in every mod_wsgi worker, so the
# first-worker guard never seeds default segment ranges, and get_start_time()
# falls through to a per-worker current time, so the OVN hash-ring workers race
# into deleting each other's rows.
#---------------------------------------------------------------------------
ML2_GROUP="mechanism_driver"
EXPECTED_WORKERS=$(grep -rhoP 'WSGIDaemonProcess\s+neutron-api.*processes=\K[0-9]+' \
                   /etc/apache2/ 2>/dev/null | head -1)
EXPECTED_WORKERS=${EXPECTED_WORKERS:-4}

mysqlq() { mysql -N -B -u root neutron -e "$1"; }

# Spread requests across every mod_wsgi daemon process so each one imports the
# app (running post-fork init: hash-ring registration on API workers) and the
# elected first worker seeds the default segment range.
warm_up() {
    for _ in $(seq 1 60); do
        curl -s -o /dev/null http://localhost:9696/ || true
    done
}

# Poll until the host has registered one hash-ring node per worker, or bail.
wait_for_nodes() {
    local timeout=60 nodes
    while [ "$timeout" -gt 0 ]; do
        warm_up
        nodes=$(mysqlq "SELECT COUNT(*) FROM ovn_hash_ring WHERE group_name='${ML2_GROUP}';")
        if [ "${nodes:-0}" -ge "$EXPECTED_WORKERS" ]; then
            return 0
        fi
        timeout=$((timeout - 1))
        sleep 1
    done
    return 1
}

dump_and_fail() {
    echo "ERROR: $1"
    echo "--- ovn_hash_ring ---"
    mysql -u root neutron -e "SELECT node_uuid, group_name, created_at FROM ovn_hash_ring;" || true
    echo "--- network_segment_ranges ---"
    mysql -u root neutron -e "SELECT id, network_type, minimum, maximum, \`default\` FROM network_segment_ranges;" || true
    tail -n 200 /var/log/neutron/*.log || true
    exit 1
}

echo -n "Waiting for ${EXPECTED_WORKERS} OVN hash-ring nodes ... "
if ! wait_for_nodes; then
    dump_and_fail "fewer than ${EXPECTED_WORKERS} hash-ring nodes registered (workers racing or no worker IDs)"
fi
echo "OK"

# 1. Default network segment range must be seeded exactly once.
echo -n "Checking default segment range was seeded ... "
RANGES=$(mysqlq "SELECT COUNT(*) FROM network_segment_ranges WHERE \`default\`=1;")
if [ "${RANGES:-0}" -lt 1 ]; then
    dump_and_fail "no default network segment range was seeded (LP: #2150285)"
fi
echo "OK (${RANGES} default range(s))"

# 2. All workers must share one start time (created_at) and none may have been
#    deleted by a sibling.
echo -n "Checking hash-ring workers share one start time ... "
NODES=$(mysqlq "SELECT COUNT(*) FROM ovn_hash_ring WHERE group_name='${ML2_GROUP}';")
DISTINCT_CREATED=$(mysqlq "SELECT COUNT(DISTINCT created_at) FROM ovn_hash_ring WHERE group_name='${ML2_GROUP}';")
DISTINCT_UUID=$(mysqlq "SELECT COUNT(DISTINCT node_uuid) FROM ovn_hash_ring WHERE group_name='${ML2_GROUP}';")
if [ "${DISTINCT_CREATED:-0}" -ne 1 ] || [ "${NODES:-0}" -ne "$EXPECTED_WORKERS" ] \
        || [ "${DISTINCT_UUID:-0}" -ne "$EXPECTED_WORKERS" ]; then
    dump_and_fail "expected ${EXPECTED_WORKERS} distinct nodes sharing 1 created_at, got nodes=${NODES} uuids=${DISTINCT_UUID} created_at=${DISTINCT_CREATED}"
fi
echo "OK (${NODES} nodes, 1 shared start time)"

# 3. The shared start time must advance on restart (it is not a frozen sentinel
#    mtime). After restart the old rows must be gone and no PK collision logged.
echo -n "Checking start time advances across restart ... "
OLD_CREATED=$(mysqlq "SELECT MAX(created_at) FROM ovn_hash_ring WHERE group_name='${ML2_GROUP}';")
: > /var/log/neutron/neutron-api.log || true
sleep 2
service apache2 restart
if ! wait_for_nodes; then
    dump_and_fail "workers did not re-register after restart"
fi
NEW_CREATED=$(mysqlq "SELECT MAX(created_at) FROM ovn_hash_ring WHERE group_name='${ML2_GROUP}';")
NEW_DISTINCT=$(mysqlq "SELECT COUNT(DISTINCT created_at) FROM ovn_hash_ring WHERE group_name='${ML2_GROUP}';")
if [ "${NEW_DISTINCT:-0}" -ne 1 ]; then
    dump_and_fail "stale rows survived restart (created_at not advanced): distinct=${NEW_DISTINCT}"
fi
if [[ ! "$NEW_CREATED" > "$OLD_CREATED" ]]; then
    dump_and_fail "start time did not advance on restart: old='${OLD_CREATED}' new='${NEW_CREATED}'"
fi
if grep -qiE "IntegrityError|Duplicate entry" /var/log/neutron/neutron-api.log; then
    dump_and_fail "hash-ring primary-key collision after restart"
fi
echo "OK (old='${OLD_CREATED}' -> new='${NEW_CREATED}')"

echo "LP: #2150285 mod_wsgi checks PASSED"
