TCP 线程池连接耗尽故障排查指导

1.1 系统级进程诊断

查看进程网络连接状态

# 定位Python进程的网络连接（将<pid>替换为实际进程ID）
ss -tanp | grep python | grep <pid>

# 统计连接状态分布
cat /proc/<pid>/net/tcp | awk '{print $4}' | sort | uniq -c

# 检查文件描述符中的socket数量
ls -l /proc/<pid>/fd | grep socket | wc -l

# 分析fd使用情况TOP排序
ls -l /proc/<pid>/fd | awk '{print $NF}' | sort | uniq -c | sort -rn

1.2 Python运行时诊断（免重启）

import sys
import threading
import gc
import socket

# 线程状态检查
print(f"当前活跃线程: {threading.active_count()}")
for t in threading.enumerate():
    print(f"  └─ {t.name}: ID={t.ident}")

# 检测内存中未释放的socket对象
sockets = [obj for obj in gc.get_objects() if isinstance(obj, socket.socket)]
print(f"\n内存中socket对象: {len(sockets)}")
for s in sockets[:10]:
    try:
        print(f"  └─ fd={s.fileno()}, {s.getpeername()} → {s.getsockname()}")
    except:
        print(f"  └─ fd={s.fileno()}, [已断开或未连接]")

1.3 py-spy 性能分析工具

# 安装诊断工具
pip install py-spy

# 实时监控线程状态（类似top命令）
py-spy top --pid <pid>

# 生成CPU火焰图
py-spy record -o profile.svg --pid <pid>

# 导出线程堆栈信息（类似jstack）
py-spy dump --pid <pid>

层级一：同步代码优化（requests + ThreadPoolExecutor）

❌ 常见错误模式

import requests
from concurrent.futures import ThreadPoolExecutor

# 错误1：每次请求新建Session，产生大量TIME_WAIT
def fetch_bad(url):
    resp = requests.get(url, timeout=30)
    return resp.text

# 错误2：线程池无界增长且不关闭
def batch_fetch_bad(urls):
    executor = ThreadPoolExecutor(max_workers=100)  # 危险！
    futures = [executor.submit(fetch_bad, url) for url in urls]
    return [f.result() for f in futures]  # 忘记shutdown()

# 错误3：缺少超时配置，可能无限挂起
requests.get("http://slow-api.com")  # 默认无超时

✅ 最佳实践方案

import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from concurrent.futures import ThreadPoolExecutor, as_completed

# 1. 创建可复用的Session（连接池核心）
def create_session(
    pool_connections=10,
    pool_maxsize=100,
    max_retries=3,
    backoff_factor=0.5
):
    session = requests.Session()

    # 配置重试策略
    retry_strategy = Retry(
        total=max_retries,
        backoff_factor=backoff_factor,
        status_forcelist=[429, 500, 502, 503, 504],
        allowed_methods=["HEAD", "GET", "OPTIONS", "POST"]
    )

    # 连接池参数
    adapter = HTTPAdapter(
        pool_connections=pool_connections,
        pool_maxsize=pool_maxsize,
        max_retries=retry_strategy,
        pool_block=True
    )

    session.mount("http://", adapter)
    session.mount("https://", adapter)

    # 设置默认超时
    session.request = lambda *args, **kwargs: requests.Session.request(
        session, *args, **kwargs, timeout=(5, 30)
    )

    return session

# 2. 线程池正确使用（上下文管理器）
def batch_fetch_good(urls, max_workers=10):
    session = create_session(pool_maxsize=max_workers * 2)

    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        future_to_url = {
            executor.submit(session.get, url): url 
            for url in urls
        }

        results = {}
        for future in as_completed(future_to_url, timeout=60):
            url = future_to_url[future]
            try:
                resp = future.result()
                results[url] = resp.text
            except Exception as e:
                results[url] = f"Error: {e}"
                # 确保异常时响应体被释放
                if hasattr(e, 'response') and e.response:
                    e.response.close()

        return results
    # 自动执行 executor.shutdown() + session.close()

# 3. 分批流式处理（控制内存）
def batch_fetch_streaming(urls, max_workers=10, chunk_size=100):
    session = create_session()

    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        for i in range(0, len(urls), chunk_size):
            chunk = urls[i:i + chunk_size]
            futures = [executor.submit(session.get, url) for url in chunk]

            for future in as_completed(futures):
                yield future.result()

            # 定期触发GC
            import gc
            gc.collect()

层级二：异步代码优化（asyncio + aiohttp）

❌ 异步陷阱示例

import asyncio
import requests  # 致命错误：同步库用于异步环境

async def bad_async_fetch(url):
    resp = requests.get(url)  # 阻塞整个事件循环！
    return resp.text

async def main():
    # 无限制并发，瞬间创建数千连接
    tasks = [bad_async_fetch(f"http://api.com/{i}") for i in range(10000)]
    await asyncio.gather(*tasks)  # 资源爆炸

# 无超时保护
asyncio.wait_for(some_coroutine, timeout=None)

✅ 异步最佳实践

import asyncio
import aiohttp
from aiohttp import ClientTimeout, TCPConnector
import asyncpg

# 1. 配置带限流的ClientSession
async def create_async_session(
    limit=100,
    limit_per_host=30,
    ttl_dns_cache=300,
    use_dns_cache=True
):
    connector = TCPConnector(
        limit=limit,
        limit_per_host=limit_per_host,
        ttl_dns_cache=ttl_dns_cache,
        use_dns_cache=use_dns_cache,
        enable_cleanup_closed=True,
        force_close=False,
        ssl=False
    )

    timeout = ClientTimeout(
        total=30,
        connect=5,
        sock_read=10
    )

    session = aiohttp.ClientSession(
        connector=connector,
        timeout=timeout,
        headers={"Connection": "keep-alive"}
    )

    return session

# 2. 信号量控制并发度
async def fetch_with_semaphore(session, url, semaphore):
    async with semaphore:
        try:
            async with session.get(url) as resp:
                resp.raise_for_status()
                return await resp.text()
        except asyncio.TimeoutError:
            return None
        except aiohttp.ClientError as e:
            print(f"Request failed: {e}")
            return None

# 3. 完整并发控制
async def controlled_fetch(urls, max_concurrent=50):
    semaphore = asyncio.Semaphore(max_concurrent)
    session = await create_async_session(limit=max_concurrent * 2)

    try:
        tasks = []
        for url in urls:
            task = asyncio.create_task(
                fetch_with_semaphore(session, url, semaphore)
            )
            tasks.append(task)

            # 背压控制
            if len(tasks) >= 1000:
                done, pending = await asyncio.wait(
                    tasks, 
                    return_when=asyncio.FIRST_COMPLETED
                )
                tasks = list(pending)
                for d in done:
                    yield await d

        if tasks:
            results = await asyncio.gather(*tasks, return_exceptions=True)
            for r in results:
                yield r

    finally:
        await session.close()

# 4. 异步数据库连接池
async def create_db_pool():
    return await asyncpg.create_pool(
        "postgresql://user:pass@localhost/db",
        min_size=5,
        max_size=20,
        max_queries=50000,
        max_inactive_time=300,
        command_timeout=60,
        server_settings={'jit': 'off'}
    )

async def main():
    pool = await create_db_pool()
    try:
        async with pool.acquire() as conn:
            async with conn.transaction():
                result = await conn.fetch("SELECT * FROM users WHERE id = $1", 1)
                return result
    finally:
        await pool.close()

层级三：WSGI/ASGI服务器调优

Gunicorn 生产配置

# gunicorn.conf.py
import multiprocessing
import os

# 工作进程模型选择
# sync: 简单但并发低
# gevent: 协程模型，高并发推荐
# uvicorn.workers.UvicornWorker: ASGI异步
worker_class = "gevent"

# 进程数配置
workers = multiprocessing.cpu_count() * 2 + 1
worker_connections = 1000

# 线程配置（sync模式）
threads = 4

# 超时设置
timeout = 30
graceful_timeout = 10
keepalive = 5

# 防内存泄漏
max_requests = 10000
max_requests_jitter = 1000

# 日志配置
accesslog = "-"
errorlog = "-"
loglevel = "warning"

# 预加载
preload_app = True

# 请求限制
limit_request_line = 4094
limit_request_fields = 100
limit_request_field_size = 8190

启动命令

# gevent高并发模式
gunicorn -c gunicorn.conf.py "app:create_app()"

# 纯异步ASGI
gunicorn -k uvicorn.workers.UvicornWorker -w 4 "app:asgi_app"

# 生产推荐组合
gunicorn -k uvicorn.workers.UvicornH11Worker -w 4 -b 0.0.0.0:8000 "app:asgi_app"

层级四：监控与可观测性

Prometheus指标暴露

from prometheus_client import Counter, Histogram, Gauge, start_http_server
import functools
import time
import asyncio

# 指标定义
ACTIVE_REQUESTS = Gauge(
    'http_requests_active', 
    '当前处理中的请求数',
    ['method', 'endpoint']
)

REQUEST_DURATION = Histogram(
    'http_request_duration_seconds',
    '请求处理耗时',
    ['method', 'endpoint', 'status'],
    buckets=[.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0]
)

CONNECTION_POOL_SIZE = Gauge(
    'connection_pool_size',
    '连接池大小',
    ['pool_name']
)

# 异步监控装饰器
def monitor_async(func):
    @functools.wraps(func)
    async def wrapper(*args, **kwargs):
        method = kwargs.get('method', 'GET')
        endpoint = func.__name__

        with ACTIVE_REQUESTS.labels(method=method, endpoint=endpoint).track_inprogress():
            start = time.time()
            try:
                result = await func(*args, **kwargs)
                status = 200
                return result
            except Exception as e:
                status = 500
                raise
            finally:
                duration = time.time() - start
                REQUEST_DURATION.labels(
                    method=method, 
                    endpoint=endpoint, 
                    status=status
                ).observe(duration)

    return wrapper

# 连接池监控
async def monitor_aiohttp_session(session, name="default"):
    if hasattr(session, '_connector'):
        connector = session._connector
        while True:
            CONNECTION_POOL_SIZE.labels(pool_name=name).set(connector.size)
            CONNECTION_POOL_SIZE.labels(pool_name=f"{name}_limit").set(connector.limit)
            await asyncio.sleep(10)

# 启动指标服务
def start_metrics_server(port=9090):
    start_http_server(port)
    print(f"Metrics server started on port {port}")

Kubernetes健康检查

from fastapi import FastAPI, HTTPException
import psutil

app = FastAPI()

class ConnectionPoolHealth:
    def __init__(self):
        self.pools = {}

    def register_pool(self, name, pool_obj, max_size_attr='maxsize', 
                     used_attr='size', available_attr='available'):
        self.pools[name] = {
            'obj': pool_obj,
            'max': max_size_attr,
            'used': used_attr,
            'avail': available_attr
        }

    def check(self):
        status = {}
        for name, config in self.pools.items():
            pool = config['obj']
            try:
                max_size = getattr(pool, config['max'], 0)
                used = getattr(pool, config['used'], 0)
                available = getattr(pool, config['avail'], 0)

                usage = used / max_size if max_size > 0 else 0
                status[name] = {
                    'healthy': usage < 0.9,
                    'usage': f"{usage:.1%}",
                    'used': used,
                    'max': max_size,
                    'available': available
                }
            except Exception as e:
                status[name] = {'healthy': False, 'error': str(e)}

        return status

pool_health = ConnectionPoolHealth()

@app.get("/health")
async def health_check():
    cpu_percent = psutil.cpu_percent(interval=0.1)
    memory = psutil.virtual_memory()

    pool_status = pool_health.check()
    pools_healthy = all(s.get('healthy', False) for s in pool_status.values())

    healthy = (
        cpu_percent < 90 and 
        memory.percent < 90 and 
        pools_healthy
    )

    status_code = 200 if healthy else 503

    return {
        "status": "healthy" if healthy else "unhealthy",
        "cpu": f"{cpu_percent}%",
        "memory": f"{memory.percent}%",
        "pools": pool_status
    }, status_code

@app.get("/ready")
async def readiness_check():
    try:
        # await check_db_connection()
        return {"ready": True}
    except Exception as e:
        raise HTTPException(status_code=503, detail=str(e))

运行时注入诊断脚本

#!/usr/bin/env python3
# debug_injector.py

import sys
import threading
import gc
import asyncio
import socket

def diagnose():
    print("=" * 50)
    print(f"Python 版本: {sys.version}")
    print(f"活跃线程数: {threading.active_count()}")
    print(f"当前线程: {threading.current_thread().name}")

    # 事件循环检查
    try:
        loop = asyncio.get_running_loop()
        print(f"事件循环运行中: {loop.is_running()}")
        if hasattr(loop, '_scheduled'):
            print(f"计划任务数: {len(loop._scheduled)}")
    except RuntimeError:
        print("无运行中的事件循环")

    # Socket对象检查
    sockets = [obj for obj in gc.get_objects() if isinstance(obj, socket.socket)]
    print(f"\n内存中socket对象: {len(sockets)}")

    # 线程池检查
    from concurrent.futures import ThreadPoolExecutor
    executors = [obj for obj in gc.get_objects() if isinstance(obj, ThreadPoolExecutor)]
    print(f"\nThreadPoolExecutor实例: {len(executors)}")
    for i, exe in enumerate(executors):
        print(f"  Executor {i}: workers={exe._max_workers}, "
              f"active={len(exe._threads)}, "
              f"queue={exe._work_queue.qsize()}")

    # aiohttp会话检查
    try:
        import aiohttp
        sessions = [obj for obj in gc.get_objects() if isinstance(obj, aiohttp.ClientSession)]
        print(f"\naiohttp ClientSession: {len(sessions)}")
        for s in sessions:
            if hasattr(s, '_connector'):
                conn = s._connector
                print(f"  Connector: size={conn.size}, limit={conn.limit}")
    except ImportError:
        pass

    print("=" * 50)

if __name__ == "__main__":
    diagnose()

注入方式

# 方式1：gdb注入
gdb -p <pid> -ex 'call PyRun_SimpleString("exec(open(\"debug_injector.py\").read())")' -ex 'detach' -ex 'quit'

# 方式2：manhole工具
pip install manhole
# 应用启动时添加
import manhole
manhole.install()
# 连接诊断
python -m manhole <pid>

连接泄漏追踪装饰器

import functools
import weakref
import logging
import traceback

logger = logging.getLogger(__name__)

def track_connections(cls):
    """跟踪类实例生命周期的装饰器"""
    instances = weakref.WeakSet()

    original_init = cls.__init__

    @functools.wraps(original_init)
    def new_init(self, *args, **kwargs):
        original_init(self, *args, **kwargs)
        instances.add(self)
        self._creation_stack = traceback.format_stack()

    cls.__init__ = new_init

    @classmethod
    def get_live_instances(cls):
        return list(instances)

    @classmethod
    def log_leaked(cls, threshold=100):
        live = cls.get_live_instances()
        if len(live) > threshold:
            logger.warning(f"检测到{cls.__name__}泄漏: {len(live)}个存活实例")
            for i, inst in enumerate(live[:5]):
                if hasattr(inst, '_creation_stack'):
                    logger.warning(f"实例{i}创建位置:\n{''.join(inst._creation_stack[-3:])}")

    cls.get_live_instances = get_live_instances
    cls.log_leaked = log_leaked

    return cls

# 使用示例
@track_connections
class DatabaseConnection:
    def __init__(self, dsn):
        self.dsn = dsn
        self.conn = None

    def close(self):
        if self.conn:
            self.conn.close()
            self.conn = None

# 定期监控
async def leak_monitor():
    while True:
        DatabaseConnection.log_leaked(threshold=50)
        await asyncio.sleep(60)

□ 确认Python版本（3.8+的asyncio更稳定）
□ 检查同步/异步代码混用问题（requests是否在async def中？）
□ 验证Session/ClientSession是否复用（避免每次新建）
□ 确认所有池化资源有界（ThreadPoolExecutor、连接池）
□ 检查超时配置完整性（连接、读取、总超时）
□ 验证资源关闭逻辑（try/finally或async with）
□ 确认健康检查端点已部署（Kubernetes场景）
□ 部署Prometheus监控指标
□ 配置Gunicorn/Uvicorn工作模式与参数
□ 进行故障注入测试（模拟下游超时、连接拒绝）

Python网络连接问题的排查需要系统化思维：

1. 诊断先行：先用系统工具和运行时诊断定位问题范围
2. 分层解决：从同步代码→异步代码→服务器配置→监控体系逐层优化
3. 预防为主：通过连接池化、超时配置、资源管理避免问题发生
4. 可观测性：建立完善的监控和告警机制，问题早发现早处理

掌握这套方法论，能够高效解决绝大多数Python网络相关的生产问题。