jack/Deploy-Laboratory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

基本框架

Browse Source
This commit is contained in:
jack
2026-03-21 04:36:06 +08:00
commit de1be1dbe5
125 changed files with 10302 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
scripts/diag/entrypath/README.md Normal file
View File

@@ -0,0 +1,113 @@
# entrypath 诊断脚本说明
`entrypath.sh` 用于排查 `client -> worker:80 -> kube-proxy DNAT -> Traefik Pod` 全链路问题。
## 命令
```bash
./scripts/diag/entrypath/entrypath.sh <command> [options]
```
- `run`：完整检查（默认）
- `preflight`：仅检查依赖与参数环境
- `capture`：强制开启抓包/trace能力后执行 run
- `analyze --log <path>`：离线分析日志
## 关键参数
- `--worker-host` / `--client-host`
- `--worker-ssh-key` / `--client-ssh-key`
- `--client-ip` / `--lb-ip`
- `--remote-check y|n`
- `--capture-mode y|n`
- `--nft-trace-mode y|n`
- `--return-trace-mode y|n`
- `--pod-netns-trace-mode y|n`
- `--non-interactive`
## 日志
- root 运行：`/root/netpol-diag-logs/entrypath-*.log`
- 非 root：`~/netpol-diag-logs/entrypath-*.log`
## 典型用法
### 1) 预检查
```bash
./scripts/diag/entrypath/entrypath.sh preflight --non-interactive
```
### 2) 全功能在线诊断（默认值示例）
```bash
./scripts/diag/entrypath/entrypath.sh run \
--worker-host root@192.168.2.62 \
--client-host root@192.168.2.63 \
--worker-ssh-key ~/.ssh/id_ed25519_k3s_diag_worker \
--client-ssh-key ~/.ssh/id_ed25519_k3s_diag_client \
--client-ip 192.168.2.63 \
--lb-ip 192.168.2.62 \
--remote-check y \
--capture-mode y \
--capture-seconds 15 \
--nft-trace-mode y \
--nft-trace-seconds 10 \
--return-trace-mode y \
--return-trace-seconds 12 \
--pod-netns-trace-mode y \
--pod-netns-trace-seconds 12 \
--non-interactive
```
### 3) 离线日志判读
```bash
./scripts/diag/entrypath/entrypath.sh analyze \
--log ~/netpol-diag-logs/entrypath-20260310-195812.log
```
## 常见陷阱与修复
### 1) `62:80` 不通，但 worker 已 DNAT 到 Traefik
若日志同时出现：
- `nft 观测到 KUBE-EXT DNAT: yes`
- `ylc61(any) SYN/SYN-ACK: N/0`
- `filter_FORWARD_POLICIES ... reject with icmpx admin-prohibited`
通常是 `ylc61` 的 firewalld 转发策略阻断 `flannel.1 -> cni0`。
修复（推荐）：
```bash
sudo firewall-cmd --zone=trusted --add-interface=flannel.1
sudo firewall-cmd --zone=trusted --add-interface=cni0
sudo firewall-cmd --permanent --zone=trusted --add-interface=flannel.1
sudo firewall-cmd --permanent --zone=trusted --add-interface=cni0
sudo firewall-cmd --reload
```
### 2) `Worker CNI hostport DNAT 计数未增长` 是否异常
不一定。若 nft trace 明确显示走的是 `KUBE-EXT -> KUBE-SVC -> KUBE-SEP`，则 CNI hostport 计数不增长属于正常路径差异，不应作为故障根因。
### 3) 成功判据
至少满足以下任一组：
- 客户端对 `http://<lb-ip>:80` 返回 `404/200/...`（非连接失败）
- 自动判读中：
- `ylc62(ens18) SYN/SYN-ACK` 为 `N/N`
- `ylc61(any) SYN/SYN-ACK` 为 `N/N`
- `ylc61(cni0) SYN/SYN-ACK` 为 `N/N`
## 模块划分
- `lib/common.sh`：通用工具、参数默认值
- `lib/k8s_checks.sh`：本地 K8s 基线采样
- `lib/remote_checks.sh`：远端 worker 采样与复测
- `lib/capture.sh`：tcpdump / nft / conntrack / pod netns
- `lib/analyze.sh`：实时/离线判读

144
scripts/diag/entrypath/entrypath.sh Normal file
View File

@@ -0,0 +1,144 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LIB_DIR="${SCRIPT_DIR}/lib"
source "${LIB_DIR}/common.sh"
source "${LIB_DIR}/k8s_checks.sh"
source "${LIB_DIR}/remote_checks.sh"
source "${LIB_DIR}/capture.sh"
source "${LIB_DIR}/analyze.sh"
parse_args() {
init_defaults
if [[ $# -gt 0 ]]; then
case "$1" in
run|preflight|capture|analyze)
COMMAND="$1"
shift
;;
esac
fi
while [[ $# -gt 0 ]]; do
case "$1" in
--worker-host) WORKER_HOST="${2:-}"; shift 2 ;;
--client-host) CLIENT_HOST="${2:-}"; shift 2 ;;
--client-ip) CLIENT_IP="${2:-}"; shift 2 ;;
--lb-ip) LB_IP="${2:-}"; shift 2 ;;
--worker-ssh-key) WORKER_SSH_KEY="${2:-}"; shift 2 ;;
--ssh-key) WORKER_SSH_KEY="${2:-}"; shift 2 ;;
--client-ssh-key) CLIENT_SSH_KEY="${2:-}"; shift 2 ;;
--remote-check) DO_REMOTE_ARG="${2:-}"; shift 2 ;;
--capture-mode) CAPTURE_MODE_ARG="${2:-}"; shift 2 ;;
--capture-seconds) CAPTURE_SECONDS="${2:-12}"; shift 2 ;;
--nft-trace-mode) NFT_TRACE_MODE_ARG="${2:-}"; shift 2 ;;
--nft-trace-seconds) NFT_TRACE_SECONDS="${2:-8}"; shift 2 ;;
--return-trace-mode) RETURN_TRACE_MODE_ARG="${2:-}"; shift 2 ;;
--return-trace-seconds) RETURN_TRACE_SECONDS="${2:-10}"; shift 2 ;;
--pod-netns-trace-mode) POD_NETNS_TRACE_MODE_ARG="${2:-}"; shift 2 ;;
--pod-netns-trace-seconds) POD_NETNS_TRACE_SECONDS_ARG="${2:-}"; shift 2 ;;
--non-interactive) NON_INTERACTIVE="1"; shift ;;
--log) ANALYZE_LOG="${2:-}"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*)
echo "[ERR] 未知参数: $1"
usage
exit 1
;;
esac
done
}
cmd_preflight() {
local_preflight_checks
prepare_runtime_context
echo "=== preflight ==="
echo "[OK] 依赖检查通过"
echo "worker_host=${WORKER_HOST}"
echo "client_host=${CLIENT_HOST:-<none>}"
echo "client_ip=${CLIENT_IP}"
echo "lb_ip=${LB_IP}"
echo "worker_ssh_key=${WORKER_SSH_KEY:-<ssh默认>}"
echo "client_ssh_key=${CLIENT_SSH_KEY:-<ssh默认>}"
}
cmd_run() {
local_preflight_checks
echo "K3s 全链路一键检查（入口 -> DNAT -> Service -> Endpoint -> NetPol -> 回包）"
echo "建议在 server 节点执行（例如 ylc61）。"
echo
# 构造针对 IPv4 LB_IP 的 curl 探测命令
local CURL_HTTP CURL_DESC
CURL_HTTP="curl -I --max-time 3 http://${LB_IP}:80"
CURL_DESC="curl -I --max-time 3 http://${LB_IP}:80"
prepare_runtime_context
setup_log_file
say "日志文件: $LOG_FILE"
say "worker SSH key: ${WORKER_SSH_KEY:-<ssh默认>}"
say "client SSH key: ${CLIENT_SSH_KEY:-<ssh默认>}"
collect_local_k8s_state
echo
resolve_runtime_modes
collect_remote_worker_state
echo
echo ">>> 请在第三方客户端(${CLIENT_IP})执行 3 次：${CURL_DESC}"
start_worker_capture
start_worker_nft_trace
start_return_path_trace
start_pod_netns_trace
if [[ -n "${CLIENT_HOST}" ]]; then
say "通过 SSH 自动触发客户端探测: ${CLIENT_HOST}"
run_cmd "Client 自动探测（3次）" ssh "${CLIENT_SSH_OPTS[@]}" "${CLIENT_HOST}" \
"for i in 1 2 3; do ${CURL_HTTP} || true; sleep 1; done"
elif [[ "${NON_INTERACTIVE}" == "0" ]]; then
read -r -p "完成后按回车继续采样..."
else
echo "[WARN] non-interactive 模式且未提供 --client-host：跳过等待直接采样，可能没有新流量。"
fi
flush_worker_capture
post_remote_worker_state
run_cmd "Traefik Pod FW 链复测" sudo iptables -L "${TRAEFIK_CHAIN:-KUBE-ROUTER-FORWARD}" -n -v --line-numbers
run_cmd "本机访问目标LB_IP:80（仅供参考，可能本机被kube-proxy劫持）" bash -lc "${CURL_HTTP}"
print_diag_summary
echo
echo "Traefik pod netns SYN/SYN-ACK: ${POD_NETNS_SYN_COUNT:-0}/${POD_NETNS_SYNACK_COUNT:-0}"
echo
echo "完成。完整日志: ${LOG_FILE}"
}
cmd_capture() {
DO_REMOTE_ARG="y"
CAPTURE_MODE_ARG="y"
NFT_TRACE_MODE_ARG="y"
RETURN_TRACE_MODE_ARG="y"
POD_NETNS_TRACE_MODE_ARG="y"
NON_INTERACTIVE="1"
cmd_run
}
cmd_analyze() {
analyze_log_file "${ANALYZE_LOG}"
}
main() {
parse_args "$@"
case "${COMMAND}" in
run) cmd_run ;;
preflight) cmd_preflight ;;
capture) cmd_capture ;;
analyze) cmd_analyze ;;
*) echo "[ERR] 未知命令: ${COMMAND}"; usage; exit 1 ;;
esac
}
main "$@"

80
scripts/diag/entrypath/lib/analyze.sh Normal file
View File

@@ -0,0 +1,80 @@
#!/usr/bin/env bash
print_diag_summary() {
echo
echo "===== 自动判读（基于当前计数） ====="
echo "- Traefik Pod FW 链: ${TRAEFIK_CHAIN:-N/A}"
echo "- Traefik REJECT 命中: ${REJECT_PKTS:-0}"
echo "- Traefik NFLOG 命中: ${NFLOG_PKTS:-0}"
echo "- Service web 链: ${TRAEFIK_WEB_SVC_CHAIN:-N/A}"
echo "- Service web endpoint 链: ${TRAEFIK_WEB_SEP_CHAIN:-N/A}"
echo "- Worker CNI hostport链: ${WORKER_CNI_DNAT_CHAIN:-N/A}"
echo "- nft 观测到 KUBE-EXT DNAT: ${NFT_DNAT_HIT:-no}"
echo "- ylc61(any) SYN/SYN-ACK: ${RET_LOCAL_SYN_COUNT:-0}/${RET_LOCAL_SYNACK_COUNT:-0}"
echo "- ylc61(cni0) SYN/SYN-ACK: ${RET_CNI0_SYN_COUNT:-0}/${RET_CNI0_SYNACK_COUNT:-0}"
echo "- ylc62(ens18) SYN/SYN-ACK: ${RET_WORKER_SYN_COUNT:-0}/${RET_WORKER_SYNACK_COUNT:-0}"
echo "- Traefik pod netns SYN/SYN-ACK: ${POD_NETNS_SYN_COUNT:-0}/${POD_NETNS_SYNACK_COUNT:-0}"
if [[ "${REJECT_PKTS:-0}" =~ ^[0-9]+$ ]] && [[ "${REJECT_PKTS:-0}" -gt 0 ]]; then
echo "[结论] Traefik Pod 防火墙链出现 REJECT 命中，优先检查 kube-system 下 Traefik 相关 Ingress NetworkPolicy。"
elif [[ "${RET_WORKER_SYNACK_COUNT:-0}" -gt 0 ]] && [[ "${RET_LOCAL_SYNACK_COUNT:-0}" -gt 0 ]] && [[ "${RET_CNI0_SYNACK_COUNT:-0}" -gt 0 ]]; then
echo "[结论] 链路已恢复：ylc62/ylc61/cni0 均观测到 SYN-ACK，62:80 已可达 Traefik。"
elif [[ "${NFT_DNAT_HIT:-no}" == "yes" ]] && [[ "${RET_LOCAL_SYN_COUNT:-0}" -gt 0 ]] && [[ "${RET_LOCAL_SYNACK_COUNT:-0}" -eq 0 ]]; then
echo "[结论] 流量已经在 worker 被 KUBE-EXT/KUBE-SVC DNAT 到 Traefik(10.42.0.12:8000)，但 ylc61 未观察到 SYN-ACK，优先排查 Traefik Pod/宿主转发回包路径。"
elif [[ -n "${WORKER_CNI_HIT_AFTER:-}" && -n "${WORKER_CNI_HIT_BEFORE:-}" ]] && \
[[ "${WORKER_CNI_HIT_AFTER}" == "${WORKER_CNI_HIT_BEFORE}" ]]; then
echo "[结论] Worker CNI hostport DNAT 计数未增长。若 nft trace 显示走 KUBE-EXT/KUBE-SVC，这是正常路径提示，不构成故障根因。"
else
echo "[结论] 未观察到 Traefik REJECT 明确命中，优先检查回包链路（ylc61<->ylc62 flannel / ylc62 ens18 出口）。"
fi
}
analyze_log_file() {
local log_file="$1"
if [[ -z "${log_file}" || ! -f "${log_file}" ]]; then
echo "[ERR] analyze 模式需要有效日志文件: --log <path>"
return 1
fi
local has_worker_dnat="no"
local has_firewalld_reject="no"
local has_traefik_reject="no"
local has_syn_no_synack="no"
local has_synack_recovered="no"
if awk '/KUBE-EXT-.*KUBE-SVC|dnat to 10\.42\./ {hit=1} END{exit !hit}' "${log_file}"; then
has_worker_dnat="yes"
fi
if awk '/filter_FORWARD_POLICIES.*admin-prohibited/ {hit=1} END{exit !hit}' "${log_file}"; then
has_firewalld_reject="yes"
fi
if awk '/Traefik REJECT 命中: [1-9]/ {hit=1} END{exit !hit}' "${log_file}"; then
has_traefik_reject="yes"
fi
if awk '/ylc61\(any\) SYN\/SYN-ACK: [1-9][0-9]*\/0/ {hit=1} END{exit !hit}' "${log_file}"; then
has_syn_no_synack="yes"
fi
if awk '/ylc61\(any\) SYN\/SYN-ACK: [1-9][0-9]*\/[1-9][0-9]*/ {a=1} /ylc62\(ens18\) SYN\/SYN-ACK: [1-9][0-9]*\/[1-9][0-9]*/ {b=1} END{exit !(a&&b)}' "${log_file}"; then
has_synack_recovered="yes"
fi
echo "===== 日志离线判读 ====="
echo "- 日志文件: ${log_file}"
echo "- 观测到 worker DNAT: ${has_worker_dnat}"
echo "- 观测到 firewalld forward reject: ${has_firewalld_reject}"
echo "- 观测到 Traefik Pod REJECT 命中: ${has_traefik_reject}"
echo "- 观测到 ylc61 SYN 无 SYN-ACK: ${has_syn_no_synack}"
echo "- 观测到链路恢复（有 SYN-ACK）: ${has_synack_recovered}"
if [[ "${has_firewalld_reject}" == "yes" ]]; then
echo "[结论] 高概率为 ylc61 firewalld FORWARD 策略阻断 flannel.1 -> cni0。"
elif [[ "${has_synack_recovered}" == "yes" ]]; then
echo "[结论] 链路已恢复，入口到 Traefik 回包路径正常。"
elif [[ "${has_worker_dnat}" == "yes" && "${has_syn_no_synack}" == "yes" ]]; then
echo "[结论] worker 入站与 DNAT 正常，需优先排查 ylc61 到 Traefik Pod 的转发/回包链路。"
elif [[ "${has_traefik_reject}" == "yes" ]]; then
echo "[结论] Traefik Pod NetworkPolicy 命中拒绝，优先检查 kube-system netpol。"
else
echo "[结论] 日志未出现单一确定根因，建议执行 run/capture 模式重新采样。"
fi
}

286
scripts/diag/entrypath/lib/capture.sh Normal file
View File

@@ -0,0 +1,286 @@
#!/usr/bin/env bash
CAPTURE_MODE="N"
CAPTURE_SECONDS="12"
CAPTURE_MODE_ARG=""
CAP_FILE_ENS18=""
CAP_PID_ENS18=""
NFT_TRACE_MODE="N"
NFT_TRACE_SECONDS="8"
NFT_TRACE_MODE_ARG=""
NFT_FILE=""
NFT_PID=""
NFT_TRACE_TABLE="diag_k3s_entrypath"
LOCAL_NFT_TRACE_TABLE="diag61_k3s_entrypath"
RETURN_TRACE_MODE="N"
RETURN_TRACE_SECONDS="10"
RETURN_TRACE_MODE_ARG=""
RET_FILE_LOCAL_8000=""
RET_FILE_LOCAL_CNI0=""
RET_FILE_WORKER_ENS18=""
RET_FILE_WORKER_CONNTRACK=""
RET_PID_LOCAL_8000=""
RET_PID_LOCAL_CNI0=""
RET_PID_WORKER_ENS18=""
RET_PID_WORKER_CONNTRACK=""
RET_FILE_LOCAL_NFT_TRACE=""
RET_PID_LOCAL_NFT_TRACE=""
NFT_DNAT_HIT="no"
RET_LOCAL_SYN_COUNT=0
RET_LOCAL_SYNACK_COUNT=0
RET_CNI0_SYN_COUNT=0
RET_CNI0_SYNACK_COUNT=0
RET_WORKER_SYN_COUNT=0
RET_WORKER_SYNACK_COUNT=0
POD_NETNS_TRACE_MODE="N"
POD_NETNS_TRACE_MODE_ARG=""
POD_NETNS_TRACE_SECONDS=""
POD_NETNS_TRACE_SECONDS_ARG=""
POD_NETNS_PID=""
POD_NETNS_FILE=""
POD_NETNS_SYN_COUNT=0
POD_NETNS_SYNACK_COUNT=0
start_worker_capture() {
if [[ ! "$CAPTURE_MODE" =~ ^[Yy]$ ]]; then
return 0
fi
if [[ ! "$DO_REMOTE" =~ ^[Yy]$ ]] || [[ -z "${WORKER_HOST}" ]]; then
echo "[WARN] 抓包模式已开启，但未启用远端检查或未提供 worker 主机，跳过抓包。"
return 0
fi
CAP_FILE_ENS18="$(mktemp)"
say "启动 worker 抓包（ens18, ${CAPTURE_SECONDS}s）: host ${CLIENT_IP} and tcp port 80"
ssh "${SSH_OPTS[@]}" "${WORKER_HOST}" \
"sudo timeout ${CAPTURE_SECONDS} tcpdump -ni ens18 'host ${CLIENT_IP} and tcp port 80' 2>/dev/null || true" \
>"${CAP_FILE_ENS18}" 2>&1 &
CAP_PID_ENS18="$!"
sleep 1
}
start_worker_nft_trace() {
if [[ ! "$NFT_TRACE_MODE" =~ ^[Yy]$ ]]; then
return 0
fi
if [[ ! "$DO_REMOTE" =~ ^[Yy]$ ]] || [[ -z "${WORKER_HOST}" ]]; then
echo "[WARN] nft trace 已开启，但未启用远端检查或未提供 worker 主机，跳过 nft trace。"
return 0
fi
ssh "${SSH_OPTS[@]}" "${WORKER_HOST}" \
"sudo nft add table inet ${NFT_TRACE_TABLE} 2>/dev/null || true; \
sudo nft 'add chain inet ${NFT_TRACE_TABLE} prerouting { type filter hook prerouting priority -301; policy accept; }' 2>/dev/null || true; \
sudo nft add rule inet ${NFT_TRACE_TABLE} prerouting iif \"ens18\" ip saddr ${CLIENT_IP} ip daddr ${LB_IP} tcp dport 80 meta nftrace set 1 2>/dev/null || true" \
|| true
NFT_FILE="$(mktemp)"
say "启动 worker nft trace（${NFT_TRACE_SECONDS}s）"
ssh "${SSH_OPTS[@]}" "${WORKER_HOST}" \
"sudo timeout ${NFT_TRACE_SECONDS} nft monitor trace 2>/dev/null || true" \
>"${NFT_FILE}" 2>&1 &
NFT_PID="$!"
sleep 1
}
start_return_path_trace() {
if [[ ! "$RETURN_TRACE_MODE" =~ ^[Yy]$ ]]; then
return 0
fi
if [[ ! "$DO_REMOTE" =~ ^[Yy]$ ]] || [[ -z "${WORKER_HOST}" ]]; then
echo "[WARN] 回包链路跟踪已开启，但未启用远端检查或未提供 worker 主机，跳过。"
return 0
fi
RET_FILE_LOCAL_8000="$(mktemp)"
RET_FILE_LOCAL_CNI0="$(mktemp)"
RET_FILE_LOCAL_NFT_TRACE="$(mktemp)"
RET_FILE_WORKER_ENS18="$(mktemp)"
RET_FILE_WORKER_CONNTRACK="$(mktemp)"
say "启动回包链路跟踪（${RETURN_TRACE_SECONDS}s）"
sudo nft add table inet "${LOCAL_NFT_TRACE_TABLE}" 2>/dev/null || true
sudo nft "add chain inet ${LOCAL_NFT_TRACE_TABLE} forward { type filter hook forward priority -301; policy accept; }" 2>/dev/null || true
sudo nft add rule inet "${LOCAL_NFT_TRACE_TABLE}" forward iif "flannel.1" ip daddr "${TRAEFIK_IP}" tcp dport 8000 meta nftrace set 1 2>/dev/null || true
sudo timeout "${RETURN_TRACE_SECONDS}" nft monitor trace 2>/dev/null \
>"${RET_FILE_LOCAL_NFT_TRACE}" 2>&1 &
RET_PID_LOCAL_NFT_TRACE="$!"
sudo timeout "${RETURN_TRACE_SECONDS}" tcpdump -ni any "host ${TRAEFIK_IP} and tcp port 8000" 2>/dev/null \
>"${RET_FILE_LOCAL_8000}" 2>&1 &
RET_PID_LOCAL_8000="$!"
sudo timeout "${RETURN_TRACE_SECONDS}" tcpdump -ni cni0 "host ${TRAEFIK_IP} and tcp port 8000" 2>/dev/null \
>"${RET_FILE_LOCAL_CNI0}" 2>&1 &
RET_PID_LOCAL_CNI0="$!"
ssh "${SSH_OPTS[@]}" "${WORKER_HOST}" \
"sudo timeout ${RETURN_TRACE_SECONDS} tcpdump -ni ens18 'host ${CLIENT_IP} and tcp' 2>/dev/null || true" \
>"${RET_FILE_WORKER_ENS18}" 2>&1 &
RET_PID_WORKER_ENS18="$!"
ssh "${SSH_OPTS[@]}" "${WORKER_HOST}" \
"if command -v conntrack >/dev/null 2>&1; then sudo timeout ${RETURN_TRACE_SECONDS} conntrack -E -p tcp 2>/dev/null || true; else echo 'conntrack: not found'; fi" \
>"${RET_FILE_WORKER_CONNTRACK}" 2>&1 &
RET_PID_WORKER_CONNTRACK="$!"
sleep 1
}
start_pod_netns_trace() {
if [[ ! "${POD_NETNS_TRACE_MODE}" =~ ^[Yy]$ ]]; then
return 0
fi
if ! command -v crictl >/dev/null 2>&1; then
echo "[WARN] 未找到 crictl，跳过 pod netns 抓包。"
return 0
fi
if ! command -v nsenter >/dev/null 2>&1; then
echo "[WARN] 未找到 nsenter，跳过 pod netns 抓包。"
return 0
fi
local sec="${POD_NETNS_TRACE_SECONDS:-$RETURN_TRACE_SECONDS}"
local cid
local pid
local runtime_id=""
runtime_id="$(sudo kubectl -n kube-system get pod "${TRAEFIK_POD}" -o jsonpath='{.status.containerStatuses[?(@.name=="traefik")].containerID}' 2>/dev/null || true)"
runtime_id="${runtime_id#containerd://}"
runtime_id="${runtime_id#cri-o://}"
if [[ -n "${runtime_id}" ]]; then
cid="${runtime_id}"
else
cid="$(sudo crictl ps --name traefik -q 2>/dev/null | awk 'NR==1{print; exit}' || true)"
fi
if [[ -z "${cid}" ]]; then
echo "[WARN] 未解析到 traefik 容器ID，跳过 pod netns 抓包。"
return 0
fi
pid="$(sudo crictl inspect "${cid}" 2>/dev/null | awk -F': ' '/"pid":/ {gsub(/,/, "", $2); print $2; exit}' || true)"
if [[ -z "${pid}" || ! "${pid}" =~ ^[0-9]+$ ]]; then
echo "[WARN] 未解析到 traefik 容器 PID，跳过 pod netns 抓包。"
return 0
fi
POD_NETNS_FILE="$(mktemp)"
say "启动 Traefik Pod netns 抓包（${sec}s, pid=${pid}）"
sudo timeout "${sec}" nsenter -t "${pid}" -n tcpdump -ni any "tcp port 8000" 2>/dev/null \
>"${POD_NETNS_FILE}" 2>&1 &
POD_NETNS_PID="$!"
sleep 1
}
flush_worker_capture() {
if [[ -n "${CAP_PID_ENS18}" ]]; then
wait "${CAP_PID_ENS18}" || true
CAP_PID_ENS18=""
fi
if [[ -n "${CAP_FILE_ENS18}" && -f "${CAP_FILE_ENS18}" ]]; then
echo
echo "===== Worker 抓包结果（ens18） ====="
cat "${CAP_FILE_ENS18}" || true
rm -f "${CAP_FILE_ENS18}" || true
CAP_FILE_ENS18=""
fi
if [[ -n "${NFT_PID}" ]]; then
wait "${NFT_PID}" || true
NFT_PID=""
fi
if [[ -n "${NFT_FILE}" && -f "${NFT_FILE}" ]]; then
if grep -Eq "KUBE-SEP-.*dnat to ${TRAEFIK_IP}:8000|dnat to ${TRAEFIK_IP}:8000" "${NFT_FILE}" >/dev/null 2>&1; then
NFT_DNAT_HIT="yes"
fi
echo
echo "===== Worker nft trace 结果 ====="
cat "${NFT_FILE}" || true
rm -f "${NFT_FILE}" || true
NFT_FILE=""
fi
if [[ "$NFT_TRACE_MODE" =~ ^[Yy]$ ]] && [[ "$DO_REMOTE" =~ ^[Yy]$ ]] && [[ -n "${WORKER_HOST}" ]]; then
ssh "${SSH_OPTS[@]}" "${WORKER_HOST}" "sudo nft delete table inet ${NFT_TRACE_TABLE} 2>/dev/null || true" || true
fi
if [[ -n "${RET_PID_LOCAL_8000}" ]]; then
wait "${RET_PID_LOCAL_8000}" || true
RET_PID_LOCAL_8000=""
fi
if [[ -n "${RET_PID_LOCAL_NFT_TRACE}" ]]; then
wait "${RET_PID_LOCAL_NFT_TRACE}" || true
RET_PID_LOCAL_NFT_TRACE=""
fi
if [[ -n "${RET_PID_LOCAL_CNI0}" ]]; then
wait "${RET_PID_LOCAL_CNI0}" || true
RET_PID_LOCAL_CNI0=""
fi
if [[ -n "${RET_PID_WORKER_ENS18}" ]]; then
wait "${RET_PID_WORKER_ENS18}" || true
RET_PID_WORKER_ENS18=""
fi
if [[ -n "${RET_PID_WORKER_CONNTRACK}" ]]; then
wait "${RET_PID_WORKER_CONNTRACK}" || true
RET_PID_WORKER_CONNTRACK=""
fi
if [[ -n "${RET_FILE_LOCAL_8000}" && -f "${RET_FILE_LOCAL_8000}" ]]; then
RET_LOCAL_SYN_COUNT="$(count_tcpdump_flag "${RET_FILE_LOCAL_8000}" "Flags [S]")"
RET_LOCAL_SYNACK_COUNT="$(count_tcpdump_flag "${RET_FILE_LOCAL_8000}" "Flags [S.]")"
echo
echo "===== 回包链路抓包（ylc61 any -> ${TRAEFIK_IP}:8000） ====="
cat "${RET_FILE_LOCAL_8000}" || true
rm -f "${RET_FILE_LOCAL_8000}" || true
RET_FILE_LOCAL_8000=""
fi
if [[ -n "${RET_FILE_LOCAL_NFT_TRACE}" && -f "${RET_FILE_LOCAL_NFT_TRACE}" ]]; then
echo
echo "===== 本机 nft trace 结果（ylc61 forward） ====="
cat "${RET_FILE_LOCAL_NFT_TRACE}" || true
rm -f "${RET_FILE_LOCAL_NFT_TRACE}" || true
RET_FILE_LOCAL_NFT_TRACE=""
fi
if [[ -n "${RET_FILE_LOCAL_CNI0}" && -f "${RET_FILE_LOCAL_CNI0}" ]]; then
RET_CNI0_SYN_COUNT="$(count_tcpdump_flag "${RET_FILE_LOCAL_CNI0}" "Flags [S]")"
RET_CNI0_SYNACK_COUNT="$(count_tcpdump_flag "${RET_FILE_LOCAL_CNI0}" "Flags [S.]")"
echo
echo "===== 回包链路抓包（ylc61 cni0 -> ${TRAEFIK_IP}:8000） ====="
cat "${RET_FILE_LOCAL_CNI0}" || true
rm -f "${RET_FILE_LOCAL_CNI0}" || true
RET_FILE_LOCAL_CNI0=""
fi
if [[ -n "${RET_FILE_WORKER_ENS18}" && -f "${RET_FILE_WORKER_ENS18}" ]]; then
RET_WORKER_SYN_COUNT="$(count_tcpdump_flag "${RET_FILE_WORKER_ENS18}" "Flags [S]")"
RET_WORKER_SYNACK_COUNT="$(count_tcpdump_flag "${RET_FILE_WORKER_ENS18}" "Flags [S.]")"
echo
echo "===== 回包链路抓包（ylc62 ens18 <-> ${CLIENT_IP}） ====="
cat "${RET_FILE_WORKER_ENS18}" || true
rm -f "${RET_FILE_WORKER_ENS18}" || true
RET_FILE_WORKER_ENS18=""
fi
if [[ -n "${RET_FILE_WORKER_CONNTRACK}" && -f "${RET_FILE_WORKER_CONNTRACK}" ]]; then
echo
echo "===== 回包链路 conntrack 事件（ylc62） ====="
cat "${RET_FILE_WORKER_CONNTRACK}" || true
rm -f "${RET_FILE_WORKER_CONNTRACK}" || true
RET_FILE_WORKER_CONNTRACK=""
fi
sudo nft delete table inet "${LOCAL_NFT_TRACE_TABLE}" 2>/dev/null || true
if [[ -n "${POD_NETNS_PID}" ]]; then
wait "${POD_NETNS_PID}" || true
POD_NETNS_PID=""
fi
if [[ -n "${POD_NETNS_FILE}" && -f "${POD_NETNS_FILE}" ]]; then
POD_NETNS_SYN_COUNT="$(count_tcpdump_flag "${POD_NETNS_FILE}" "Flags [S]")"
POD_NETNS_SYNACK_COUNT="$(count_tcpdump_flag "${POD_NETNS_FILE}" "Flags [S.]")"
echo
echo "===== Traefik Pod netns 抓包（ylc61） ====="
cat "${POD_NETNS_FILE}" || true
rm -f "${POD_NETNS_FILE}" || true
POD_NETNS_FILE=""
fi
}

104
scripts/diag/entrypath/lib/common.sh Normal file
View File

@@ -0,0 +1,104 @@
#!/usr/bin/env bash
now() { date '+%Y-%m-%d %H:%M:%S'; }
say() { echo "[$(now)] $*"; }
usage() {
cat <<'EOF'
用法:
entrypath.sh <command> [选项]
entrypath.sh [选项] # 等价于 run
命令:
run 完整检查（默认）
preflight 仅检查本地依赖与参数环境
capture 强制开启所有抓包/trace能力后执行 run
analyze --log <path> 离线分析日志文件
通用选项:
--worker-host <user@host> 远端 worker SSH 主机（默认 jack@192.168.2.62）
--client-host <user@host> 远端客户端 SSH 主机（可选，用于自动发起 curl）
--client-ip <ip> 第三方客户端 IP（默认 192.168.2.63）
--lb-ip <ip> 待排查 LB 节点 IP（默认 192.168.2.62）
--worker-ssh-key <path> worker SSH 私钥路径（默认 ~/.ssh/id_ed25519_k3s_diag_worker）
--client-ssh-key <path> 客户端 SSH 私钥路径（默认 ~/.ssh/id_ed25519_k3s_diag_client）
--ssh-key <path> 兼容别名，等同 --worker-ssh-key
--remote-check <y|n> 是否启用远端检查（默认 n，交互可覆盖）
--capture-mode <y|n> 抓包模式（worker ens18，默认 n）
--capture-seconds <n> 抓包持续秒数（默认 12）
--nft-trace-mode <y|n> nft trace 模式（worker，默认 n）
--nft-trace-seconds <n> nft trace 持续秒数（默认 8）
--return-trace-mode <y|n> 回包链路跟踪（ylc61/ylc62，默认 n）
--return-trace-seconds <n> 回包链路跟踪持续秒数（默认 10）
--pod-netns-trace-mode <y|n> Traefik Pod netns 抓包（ylc61，默认 n）
--pod-netns-trace-seconds <n> Traefik Pod netns 抓包持续秒数（默认同 return-trace-seconds）
--non-interactive 非交互模式（需配合上面参数）
--log <path> 仅 analyze 子命令使用
-h, --help 显示帮助
EOF
}
run_cmd() {
local desc="$1"
shift
echo
echo "===== ${desc} ====="
"$@" || true
}
require_cmd() {
local c="$1"
if ! command -v "$c" >/dev/null 2>&1; then
echo "[ERR] missing command: $c"
exit 1
fi
}
read_default() {
local prompt="$1"
local def="$2"
local out
printf "%s [%s]: " "$prompt" "$def" >&2
read -r out
echo "${out:-$def}"
}
extract_pkts_for_target() {
local table="$1"
local chain="$2"
local target="$3"
sudo iptables ${table:+-t "$table"} -L "$chain" -n -v -x 2>/dev/null \
| awk -v t="$target" '$3==t {print $1; exit}'
}
extract_first_jump_target() {
local table="$1"
local chain="$2"
sudo iptables ${table:+-t "$table"} -S "$chain" 2>/dev/null \
| awk '/-j KUBE-SEP-/{for(i=1;i<=NF;i++) if($i=="-j"){print $(i+1); exit}}'
}
count_tcpdump_flag() {
local file="$1"
local flag="$2"
if [[ ! -f "$file" ]]; then
echo 0
return 0
fi
awk -v f="$flag" 'BEGIN{c=0} index($0,f){c++} END{print c}' "$file"
}
init_defaults() {
COMMAND="run"
ANALYZE_LOG=""
WORKER_HOST="jack@192.168.2.62"
CLIENT_HOST=""
CLIENT_IP="192.168.2.63"
LB_IP="192.168.2.62"
WORKER_SSH_KEY=""
CLIENT_SSH_KEY=""
DEFAULT_WORKER_SSH_KEY="${HOME}/.ssh/id_ed25519_k3s_diag_worker"
DEFAULT_CLIENT_SSH_KEY="${HOME}/.ssh/id_ed25519_k3s_diag_client"
DO_REMOTE_ARG=""
NON_INTERACTIVE="0"
}

95
scripts/diag/entrypath/lib/k8s_checks.sh Normal file
View File

@@ -0,0 +1,95 @@
#!/usr/bin/env bash
local_preflight_checks() {
require_cmd bash
require_cmd sudo
require_cmd kubectl
require_cmd awk
require_cmd sed
require_cmd grep
}
prepare_runtime_context() {
if [[ "${NON_INTERACTIVE}" == "0" ]]; then
WORKER_HOST="$(read_default "Worker SSH 主机（user@host，留空跳过远端检查）" "${WORKER_HOST}")"
CLIENT_IP="$(read_default "第三方客户端 IP（用于人工发流量）" "${CLIENT_IP}")"
LB_IP="$(read_default "待排查节点对外 IP（如 ylc62）" "${LB_IP}")"
fi
if [[ -z "${WORKER_SSH_KEY}" && -f "${DEFAULT_WORKER_SSH_KEY}" ]]; then
WORKER_SSH_KEY="${DEFAULT_WORKER_SSH_KEY}"
fi
if [[ -z "${CLIENT_SSH_KEY}" && -f "${DEFAULT_CLIENT_SSH_KEY}" ]]; then
CLIENT_SSH_KEY="${DEFAULT_CLIENT_SSH_KEY}"
fi
if [[ -z "${CLIENT_SSH_KEY}" && -n "${WORKER_SSH_KEY}" ]]; then
CLIENT_SSH_KEY="${WORKER_SSH_KEY}"
fi
SSH_OPTS=()
if [[ -n "${WORKER_SSH_KEY}" ]]; then
SSH_OPTS=(-i "${WORKER_SSH_KEY}" -o IdentitiesOnly=yes)
fi
CLIENT_SSH_OPTS=()
if [[ -n "${CLIENT_SSH_KEY}" ]]; then
CLIENT_SSH_OPTS=(-i "${CLIENT_SSH_KEY}" -o IdentitiesOnly=yes)
fi
}
setup_log_file() {
if [[ "${EUID}" -eq 0 ]]; then
LOG_DIR="/root/netpol-diag-logs"
else
LOG_DIR="${HOME}/netpol-diag-logs"
fi
mkdir -p "$LOG_DIR"
LOG_FILE="${LOG_DIR}/entrypath-$(date '+%Y%m%d-%H%M%S').log"
exec > >(tee -a "$LOG_FILE") 2>&1
}
collect_local_k8s_state() {
run_cmd "节点状态" sudo kubectl get nodes -o wide
run_cmd "kube-system 关键组件" sh -c "sudo kubectl -n kube-system get pods -o wide | grep -E 'traefik|svclb|flannel|kube-proxy' || true"
run_cmd "Traefik Service" sudo kubectl -n kube-system get svc traefik -o wide
run_cmd "Traefik Service 关键字段" sh -c "sudo kubectl -n kube-system get svc traefik -o yaml | grep -E 'type:|externalTrafficPolicy|loadBalancerSourceRanges|svccontroller.k3s.cattle.io' || true"
TRAEFIK_POD="$(sudo kubectl -n kube-system get pod -l app.kubernetes.io/name=traefik -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
TRAEFIK_IP="$(sudo kubectl -n kube-system get pod -l app.kubernetes.io/name=traefik -o jsonpath='{.items[0].status.podIP}' 2>/dev/null || true)"
if [[ -z "${TRAEFIK_IP}" ]]; then
echo "[ERR] 无法解析 Traefik Pod IP，终止。"
exit 1
fi
TRAEFIK_CHAIN="$(sudo iptables -L KUBE-ROUTER-FORWARD -n -v --line-numbers \
| awk -v ip="${TRAEFIK_IP}" '$0 ~ ip {print $4; exit}')"
echo
echo "Traefik pod: ${TRAEFIK_POD}"
echo "Traefik ip : ${TRAEFIK_IP}"
echo "Traefik fw : ${TRAEFIK_CHAIN:-N/A}"
if [[ -n "${TRAEFIK_CHAIN}" ]]; then
run_cmd "Traefik Pod FW 链详情" sudo iptables -L "$TRAEFIK_CHAIN" -n -v -x
run_cmd "Traefik Pod FW 链规则" sudo iptables -S "$TRAEFIK_CHAIN"
REJECT_PKTS="$(extract_pkts_for_target "" "$TRAEFIK_CHAIN" REJECT || echo 0)"
NFLOG_PKTS="$(extract_pkts_for_target "" "$TRAEFIK_CHAIN" NFLOG || echo 0)"
else
REJECT_PKTS=0
NFLOG_PKTS=0
fi
TRAEFIK_WEB_SVC_CHAIN="$(sudo iptables -t nat -S KUBE-SERVICES \
| awk '/kube-system\/traefik:web cluster IP/ && /--dport 80/ {for(i=1;i<=NF;i++) if($i=="-j"){print $(i+1); exit}}')"
TRAEFIK_WEB_SEP_CHAIN=""
if [[ -n "${TRAEFIK_WEB_SVC_CHAIN}" ]]; then
run_cmd "Traefik web Service 链" sudo iptables -t nat -L "$TRAEFIK_WEB_SVC_CHAIN" -n -v -x
TRAEFIK_WEB_SEP_CHAIN="$(extract_first_jump_target nat "$TRAEFIK_WEB_SVC_CHAIN" || true)"
fi
if [[ -n "${TRAEFIK_WEB_SEP_CHAIN}" ]]; then
run_cmd "Traefik web Endpoint 链" sudo iptables -t nat -L "$TRAEFIK_WEB_SEP_CHAIN" -n -v -x
fi
run_cmd "KUBE-SERVICES 中目标LB_IP命中" sh -c "sudo iptables -t nat -L KUBE-SERVICES -n -v --line-numbers | grep '${LB_IP}' || true"
}

59
scripts/diag/entrypath/lib/remote_checks.sh Normal file
View File

@@ -0,0 +1,59 @@
#!/usr/bin/env bash
WORKER_CNI_DNAT_CHAIN=""
WORKER_CNI_HIT_BEFORE=""
WORKER_CNI_HIT_AFTER=""
resolve_runtime_modes() {
if [[ -n "${DO_REMOTE_ARG}" ]]; then
DO_REMOTE="${DO_REMOTE_ARG}"
else
if [[ "${NON_INTERACTIVE}" == "1" ]]; then
DO_REMOTE="N"
else
read -r -p "是否通过 SSH 拉取 worker 计数（需要可免交互 sudo）? [y/N]: " DO_REMOTE
DO_REMOTE="${DO_REMOTE:-N}"
fi
fi
if [[ -n "${CAPTURE_MODE_ARG}" ]]; then
CAPTURE_MODE="${CAPTURE_MODE_ARG}"
fi
if [[ -n "${NFT_TRACE_MODE_ARG}" ]]; then
NFT_TRACE_MODE="${NFT_TRACE_MODE_ARG}"
fi
if [[ -n "${RETURN_TRACE_MODE_ARG}" ]]; then
RETURN_TRACE_MODE="${RETURN_TRACE_MODE_ARG}"
fi
if [[ -n "${POD_NETNS_TRACE_MODE_ARG}" ]]; then
POD_NETNS_TRACE_MODE="${POD_NETNS_TRACE_MODE_ARG}"
fi
if [[ -n "${POD_NETNS_TRACE_SECONDS_ARG}" ]]; then
POD_NETNS_TRACE_SECONDS="${POD_NETNS_TRACE_SECONDS_ARG}"
fi
}
collect_remote_worker_state() {
if [[ ! "$DO_REMOTE" =~ ^[Yy]$ ]] || [[ -z "$WORKER_HOST" ]]; then
return 0
fi
say "开始远端检查: ${WORKER_HOST}"
run_cmd "Worker 基础网络状态" ssh "${SSH_OPTS[@]}" "$WORKER_HOST" "ip -br a; ip route"
run_cmd "Worker k3s-agent 状态" ssh "${SSH_OPTS[@]}" "$WORKER_HOST" "sudo systemctl is-active k3s-agent; sudo journalctl -u k3s-agent -n 40 --no-pager"
run_cmd "Worker PREROUTING 关键计数" ssh "${SSH_OPTS[@]}" "$WORKER_HOST" "sudo iptables -t nat -L PREROUTING -n -v --line-numbers | grep -E 'CNI-HOSTPORT-DNAT|KUBE-SERVICES|dpt:80' || true"
run_cmd "Worker CNI-HOSTPORT-DNAT" ssh "${SSH_OPTS[@]}" "$WORKER_HOST" "sudo iptables -t nat -L CNI-HOSTPORT-DNAT -n -v --line-numbers || true"
WORKER_CNI_DNAT_CHAIN="$(ssh "${SSH_OPTS[@]}" "$WORKER_HOST" "sudo iptables -t nat -S CNI-HOSTPORT-DNAT 2>/dev/null | awk '/-j CNI-DN-/{for(i=1;i<=NF;i++) if(\$i==\"-j\"){print \$(i+1); exit}}'")"
if [[ -n "${WORKER_CNI_DNAT_CHAIN}" ]]; then
run_cmd "Worker 具体 CNI-DNAT 链" ssh "${SSH_OPTS[@]}" "$WORKER_HOST" "sudo iptables -t nat -L ${WORKER_CNI_DNAT_CHAIN} -n -v --line-numbers"
WORKER_CNI_HIT_BEFORE="$(ssh "${SSH_OPTS[@]}" "$WORKER_HOST" "sudo iptables -t nat -L ${WORKER_CNI_DNAT_CHAIN} -n -v -x | awk 'BEGIN{v=0} /DNAT/&&/dpt:80/{v=\$1} END{print v}'")"
fi
}
post_remote_worker_state() {
if [[ "$DO_REMOTE" =~ ^[Yy]$ ]] && [[ -n "${WORKER_CNI_DNAT_CHAIN}" ]]; then
WORKER_CNI_HIT_AFTER="$(ssh "${SSH_OPTS[@]}" "$WORKER_HOST" "sudo iptables -t nat -L ${WORKER_CNI_DNAT_CHAIN} -n -v -x | awk 'BEGIN{v=0} /DNAT/&&/dpt:80/{v=\$1} END{print v}'")"
run_cmd "Worker CNI-DNAT 链复测" ssh "${SSH_OPTS[@]}" "$WORKER_HOST" "sudo iptables -t nat -L ${WORKER_CNI_DNAT_CHAIN} -n -v --line-numbers"
fi
}